# -*- coding: utf-8 -*- import urllib import json import xlrd from django.db import IntegrityError from django import forms from django.conf import settings from django.db.models.loading import get_model from theme.models import Theme, Tag from place_exposition.models import PlaceExposition from place_conference.models import PlaceConference from import_xls.models import Log from excel_settings import import_settings from functions.form_check import translit_with_separator from excel_settings import place_exp_sett from import_xls.excel_settings import event_sett from django.utils.translation import ugettext as _ languages = [code for code in settings.LANGUAGES] typical_errors = {'(1048, "Column \'city_id\' cannot be null")':u'Неправильная страна или город', '(1048, "Column \'country_id\' cannot be null")':u'Неправильная страна или город', '(1048, "Column \'data_end\' cannot be null")':u'НЕправильный формат или не заполнена дата окончания', '(1048, "Column \'data_end\' cannot be null")':u'НЕправильный формат или не заполнена дата начала'} def logcall(f, msg): with open(f.file.name, 'a') as logfile: logfile.write(msg.encode('utf8')) class ImportForm(forms.Form): """ abstract form for importing models from excel file to database """ model = None excel_file = forms.FileField(label=_(u'Выберите файл')) language = forms.ChoiceField(label=_(u'Выберите язык'), choices=languages) def save_file(self): data = self.cleaned_data lang = data['language'] f = data['excel_file'] book = xlrd.open_workbook(file_contents=f.read()) sheet = book.sheet_by_index(0) row_list = [sheet.row_values(row_number) for row_number in range(sheet.nrows)] # all field names in excel file (must be in second row) field_names = [name for name in row_list[1]] for row_number, row in enumerate(row_list): # go through all rows in file if row_number > 1: # first two fields are verbose name and name if row[0] != '': # in first column ids try: object = self.model.objects.language(lang).get(id=int(row[0])) except ValueError: object = self.model() object.translate(lang) except self.model.DoesNotExist: object = self.model(id= int(row[0])) object.translate(lang) else: # if id blank - its a new event object = self.model() object.translate(lang) for col_number, cell in enumerate(row): # go through row cells # field name current cell field_name = field_names[col_number] if field_name =='theme': # need save object before saving manytomany field object.save() setting = import_settings.get(field_name) if setting is not None: # if setting exist for this field func = setting.get('func') if func is not None: extra_value = setting.get('extra_values') if extra_value is not None: # if setting has extra value then # it is some field like city, theme, tag # that has relation and can be created # in function we add language(need for relation fields) # and extra value from object (like for city need country) value = func(cell, lang, getattr(object, extra_value)) else: value = func(cell) setattr(object, field_name, value) object.save() class ImportThemeForm(ImportForm): model = Theme def google_address(address): if address: address = address.encode('utf') params = urllib.urlencode({'sensor': 'false', 'address': address}) response = urllib.urlopen("http://maps.google.com/maps/api/geocode/json?%s" % params) response = response.read() results = json.loads(response).get('results') if results: response = {'address' : results[0].get('formatted_address'), 'lat' : results[0]['geometry']['location']['lat'], 'lng' : results[0]['geometry']['location']['lng']} #return response return json.dumps(response) else: return '' return '' # place class ImportPlaceExpositionForm(ImportForm): model = PlaceExposition settings = place_exp_sett def get_row_object(self, row): if row[0] != '': # in first column id try: obj = self.model.objects.language(self.lang).get(id=int(row[0])) except ValueError: obj = self.model() obj.translate(self.lang) except self.model.DoesNotExist: obj = self.model(id= int(row[0])) obj.translate(self.lang) else: # if id blank - its a new object obj = self.model() obj.translate(self.lang) return obj def save_file_debug(self): data = self.cleaned_data lang, self.lang = data['language'], data['language'] f = data['excel_file'] book = xlrd.open_workbook(file_contents=f.read()) sheet = book.sheet_by_index(0) row_list = [sheet.row_values(row_number) for row_number in range(sheet.nrows)] field_names = [name for name in row_list[0]] model = self.model labels = [label for label in row_list[0]] errors = [] #log = Log.custom.create_log_name(f) for row_number, row in enumerate(row_list): if row_number == 0: continue #log_msg = u'[%s] %s: '%(str(row_number), row[2]) obj = self.get_row_object(row) methods = [] # go through row cells for col_number, cell in enumerate(row): # get current label label = labels[col_number] setting = event_sett.get(label) if setting is None: # no label in settings continue if setting.get('method'): # this cell contains data that must be written after creating object if cell != "": methods.append({'func': setting['func'], 'value': cell, 'purpose': setting.get('purpose'), 'field': label}) continue field_name = setting['field'] func = setting.get('func') if func is None: # no function in settings continue extra_value = setting.get('extra_values') if extra_value is not None: # if setting has extra value then # it is some field like city, tag # that has relation and can be created # in function we add language(need for relation fields) # and extra value from object (like for city need country) try: extra = getattr(obj, extra_value) except Exception: continue value = func(cell, 'ru', extra) else: value = func(cell) #try: setattr(obj, field_name, value) #except ValueError: # continue if not obj.url: obj.url = translit_with_separator(obj.name) # try: obj.save() # except IntegrityError, e: # error = str(e) # if typical_errors.get(error): # error = typical_errors[error] # if error.startswith('(1062, "Duplicate entry') and error.endswith('for key \'url\'")'): # # error = u'Событие с таким названием или урлом уже существует' # # errors.append([obj.name, error]) # log_msg += error # logcall(log.log, log_msg + '\n') # continue for method in methods: func = method['func'] if method.get('purpose'): try: func(obj, method['value'], method['purpose']) except: #log_msg += '(%s: Ошибка);'%method['field'] continue else: msg = func(obj, method['value']) if msg: #log_msg += '(%s: %s);'%(method['field'], msg) pass #logcall(log.log, log_msg + '\n') def save_file(self): data = self.cleaned_data lang = data['language'] f = data['excel_file'] book = xlrd.open_workbook(file_contents=f.read()) sheet = book.sheet_by_index(0) row_list = [sheet.row_values(row_number) for row_number in range(sheet.nrows)] model = self.model labels = [label for label in row_list[0]] errors = [] for row_number, row in enumerate(row_list): if row_number == 0: continue if row[0] != '': # in first column id try: obj = self.model.objects.language(lang).get(id=int(row[0])) except ValueError: obj = self.model() obj.translate(lang) except self.model.DoesNotExist: obj = self.model(id= int(row[0])) obj.translate(lang) else: # if id blank - its a new event obj = model() obj.translate(lang) # ---------------- # go through all rows in file methods = [] for col_number, cell in enumerate(row): # go through row cells # field name current cell label = labels[col_number] setting = place_exp_sett.get(label) if setting is None: continue if setting.get('method'): if cell != "": methods.append({'func': setting['func'], 'value': cell, 'purpose': setting.get('purpose')}) continue field_name = setting['field'] func = setting.get('func') if func is None: continue extra_value = setting.get('extra_values') if extra_value is not None: # if setting has extra value then # it is some field like city, theme, tag # that has relation and can be created # in function we add language(need for relation fields) # and extra value from object (like for city need country) try: extra = getattr(obj, extra_value) except Exception: continue value = func(cell, 'ru', extra) else: value = func(cell) try: setattr(obj, field_name, value) except ValueError, e: continue if field_name !='adress': try: setattr(obj, field_name, value) except ValueError: continue else: #gaddress = google_address(value) setattr(obj, 'address', google_address(value)) if not obj.url: obj.url = translit_with_separator(obj.name) try: obj.save() except IntegrityError, e: error = str(e) if typical_errors.get(error): error = typical_errors[error] if error.startswith('(1062, "Duplicate entry') and error.endswith('for key \'url\'")'): error = _(u'Место с таким названием или урлом уже существует') errors.append([obj.name, error]) continue for method in methods: func = method['func'] if method.get('purpose'): try: func(obj, method['value'], method['purpose']) except Exception, e: continue else: func(obj, method['value']) return errors class ImportPlaceConferenceForm(ImportPlaceExpositionForm): model = PlaceConference class ImportEventForm(ImportForm): """ extended form for importing one type of event """ event = forms.ChoiceField(label=_(u'Выберите тип события'), choices=[('exposition.Exposition', _(u'Выставка')), ('conference.Conference', _(u'Конференция')), ('seminar.Seminar', _(u'Семинар')), ('webinar.Webinar', _(u'Вебинар'))]) def save_file_debug(self): data = self.cleaned_data lang = data['language'] f = data['excel_file'] book = xlrd.open_workbook(file_contents=f.read()) sheet = book.sheet_by_index(0) row_list = [sheet.row_values(row_number) for row_number in range(sheet.nrows)] field_names = [name for name in row_list[0]] # model model = get_model(data['event'].split('.')[0], data['event'].split('.')[1]) labels = [label for label in row_list[0]] errors = [] log = Log.custom.create_log_name(f) for row_number, row in enumerate(row_list): if row_number == 0: continue log_msg = u'[%s] %s: '%(str(row_number), row[2]) if row[0] != '': # in first column id try: obj = model.objects.language(lang).get(id=int(row[0])) except ValueError: obj = model() obj.translate(lang) except model.DoesNotExist: obj = model(id= int(row[0])) obj.translate(lang) else: # if id blank - its a new event obj = model() obj.translate(lang) methods = [] for col_number, cell in enumerate(row): # go through row cells # field name current cell label = labels[col_number] setting = event_sett.get(label) if setting is None: continue if setting.get('method'): # this cell contains data that must be written after creating object if cell != "": methods.append({'func': setting['func'], 'value': cell, 'purpose': setting.get('purpose'), 'field': label}) continue field_name = setting['field'] func = setting.get('func') if func is None: continue extra_value = setting.get('extra_values') if extra_value is not None: # if setting has extra value then # it is some field like city, tag # that has relation and can be created # in function we add language(need for relation fields) # and extra value from object (like for city need country) try: extra = getattr(obj, extra_value) except Exception: continue value = func(cell, 'ru', extra) elif setting.get('bitfield'): value = func(obj, cell, setting['label']) else: value = func(cell) if field_name != 'place': try: setattr(obj, field_name, value) except ValueError: continue else: if value is None: setattr(obj, 'place_alt', cell) else: try: setattr(obj, field_name, value) except Exception: setattr(obj, 'place_alt', cell) if not obj.url: obj.url = translit_with_separator(obj.name) obj.is_published = True try: obj.save() except IntegrityError, e: error = str(e) if typical_errors.get(error): error = typical_errors[error] if error.startswith('(1062, "Duplicate entry') and error.endswith('for key \'url\'")'): error = _(u'Событие с таким названием или урлом уже существует') errors.append([obj.name, error]) log_msg += error logcall(log.log, log_msg + '\n') continue for method in methods: func = method['func'] if method.get('purpose'): try: func(obj, method['value'], method['purpose']) except: log_msg += '(%s: Ошибка);'%method['field'] continue else: msg = func(obj, method['value']) if msg: log_msg += '(%s: %s);'%(method['field'], msg) logcall(log.log, log_msg + '\n') return errors def save_file(self): """ save events from excel file in language from form """ data = self.cleaned_data lang = data['language'] f = data['excel_file'] book = xlrd.open_workbook(file_contents=f.read()) sheet = book.sheet_by_index(0) row_list = [sheet.row_values(row_number) for row_number in range(sheet.nrows)] # all field names in excel file (must be in second row) field_names = [name for name in row_list[0]] # model model = get_model(data['event'].split('.')[0], data['event'].split('.')[1]) list_dicts = [] for row_number, row in enumerate(row_list): # go through all rows in file if row_number > 0: # first two fields are verbose name and name if row[0] != '': # in first column ids try: object = self.model.objects.language(lang).get(id=int(row[0])) except ValueError: object = self.model() object.translate(lang) except self.model.DoesNotExist: object = self.model(id= int(row[0])) object.translate(lang) else: # if id blank - its a new event object = model() object.translate(lang) d = {} for col_number, cell in enumerate(row): # through row field_name = field_names[col_number] setting = import_settings.get(field_name) d[setting] = cell list_dicts.append(d) """ # go through row cells # field name current cell field_name = field_names[col_number] if field_name =='theme': # need save object before saving manytomany field #object.save() setting = import_settings.get(field_name) d[setting] = cell if setting is not None: # if setting exist for this field func = setting.get('func') if func is not None: extra_value = setting.get('extra_values') if extra_value is not None: # if setting has extra value then # it is some field like city, theme, tag # that has relation and can be created # in function we add language(need for relation fields) # and extra value from object (like for city need country) if cell: value = func(cell, lang, getattr(object, extra_value)) else: value = None else: value = func(cell) if value: setattr(object, field_name, value) """ #object.save() class ImportTagForm(ImportForm): """ hacked problem with importing theme field in save_file method """ model = Tag #@async def save_file(self): data = self.cleaned_data lang = data['language'] f = data['excel_file'] book = xlrd.open_workbook(file_contents=f.read()) sheet = book.sheet_by_index(0) row_list = [sheet.row_values(row_number) for row_number in range(sheet.nrows)] # all field names in excel file (must be in second row) field_names = [name for name in row_list[1]] for row_number, row in enumerate(row_list): # go through all rows in file if row_number > 1: # first two fields are verbose name and name if row[0] != '': # in first column ids try: object = self.model.objects.language(lang).get(id=int(row[0])) except ValueError: object = self.model() object.translate(lang) except self.model.DoesNotExist: object = self.model(id= int(row[0])) object.translate(lang) else: # if id blank - its a new event object = self.model() object.translate(lang) for col_number, cell in enumerate(row): # go through row cells # field name current cell field_name = field_names[col_number] setting = import_settings.get(field_name) if setting is not None: # if setting exist for this field func = setting.get('func') if func is not None: extra_value = setting.get('extra_values') if extra_value is not None: # if setting has extra value then # it is some field like city, theme, tag # that has relation and can be created # in function we add language(need for relation fields) # and extra value from object (like for city need country) value = func(cell, lang, getattr(object, extra_value)) else: value = func(cell) if field_name != 'theme': setattr(object, field_name, value) else: setattr(object, field_name, value[0]) object.save()