City refactor

remotes/origin/1203
Nazar Kotjuk 10 years ago
parent b8469deb09
commit ce857f4140
  1. 78
      city/management/commands/city_lost_create.py
  2. 22
      city/management/commands/city_slug.py
  3. 30
      city/management/commands/city_update_from_old.py
  4. 26
      city/management/commands/create_hotels.py
  5. 16
      city/management/commands/hotel_photos.py
  6. 15
      city/management/commands/update_hotels_currency.py
  7. 23
      city/management/commands/update_hotels_price.py
  8. 130
      city/models.py
  9. 4
      city/urls.py
  10. 8
      city/views.py

@ -1,78 +0,0 @@
# -*- coding: utf-8 -*-
from django.core.management.base import BaseCommand, CommandError
from city.models import City
from country.models import Country
cities = [
{'id':'-1563952', 'country':'11', 'url':'canberra-australia', 'ru':u'Канберра', 'en':'Canberra'},
]
"""
cities = [
{'id':'-1563952', 'country':'11', 'url':'canberra-australia', 'ru':u'Канберра', 'en':'Canberra'},
{'id':'-1586844', 'country':'11', 'url':'melbourne-australia', 'ru':u'Мельбурн', 'en':'Melbourne'},
{'id':'-1594675', 'country':'11', 'url':'perth-australia', 'ru':u'Перт', 'en':'Perth'},
{'id':'-1555188', 'country':'11', 'url':'adelaide-australia', 'ru':u'Аделаида', 'en':'Adelaide'},
{'id':'-1561728', 'country':'11', 'url':'brisbane-australia', 'ru':u'Брисбен', 'en':'Brisbane'},
{'id':'-1569058', 'country':'11', 'url':'darwin-australia', 'ru':u'Дарвин', 'en':'Darwin'},
{'id':'-1989985', 'country':'10', 'url':'salzburg-austria', 'ru':u'Зальцбург', 'en':'Salzburg'},
{'id':'-1981445', 'country':'10', 'url':'innsbruck-austria', 'ru':u'Инсбрук', 'en':'Innsbruck'},
{'id':'-1982354', 'country':'10', 'url':'klagenfurt-austria', 'ru':u'Клагенфурте', 'en':'Klagenfurt'},
{'id':'-979186', 'country':'8', 'url':'buenos-aires-argentina', 'ru':u'Буэнос-Айрес', 'en':'Buenos Aires'},
{'id':'-1955925', 'country':'17', 'url':'charleroi-belgium', 'ru':u'Шарлеруа', 'en':'Charleroi'},
{'id':'-1953257', 'country':'17', 'url':'antwerp-belgium', 'ru':u'Антверпен', 'en':'Antwerp'},
{'id':'-1955538', 'country':'17', 'url':'brussels-belgium', 'ru':u'Брюссель', 'en':'Brussels'},
{'id':'-1958757', 'country':'17', 'url':'ghent-belgium', 'ru':u'Гент', 'en':'Ghent'},
{'id':'-1963947', 'country':'17', 'url':'leuven-belgium', 'ru':u'Левен', 'en':'Leuven'},
{'id':'-1964016', 'country':'17', 'url':'liege-belgium', 'ru':u'Льеж', 'en':'Liège'},
{'id':'-1965564', 'country':'17', 'url':'mons-belgium', 'ru':u'Монс', 'en':'Mons'},
{'id':'-1956109', 'country':'17', 'url':'ciney-belgium', 'ru':u'Сине', 'en':'Ciney'},
{'id':'-1959925', 'country':'17', 'url':'hasselt-belgium', 'ru':u'Хасселт', 'en':'Hasselt'},
{'id':'-629138', 'country':'27', 'url':'belo-horizonte-brazil', 'ru':u'Белу-Оризонти', 'en':'Belo Horizonte'},
{'id':'-663509', 'country':'27', 'url':'porto-alegre-brazil', 'ru':u'Порту-Алегри', 'en':'Porto Alegre'},
{'id':'-629039', 'country':'27', 'url':'Belem-brazil', 'ru':u'Белем', 'en':'Belém'},
{'id':'-644901', 'country':'27', 'url':'goiania-brazil', 'ru':u'Гояния', 'en':'Goiânia'},
{'id':'-653186', 'country':'27', 'url':'manaus-brazil', 'ru':u'Манаус', 'en':'Manaus'},
{'id':'-657942', 'country':'27', 'url':'olinda-brazil', 'ru':u'Olinda', 'en':'Olinda'},
{'id':'-657510', 'country':'27', 'url':'novo-hamburgo-brazil', 'ru':u'Novo Hamburgo', 'en':'Novo Hamburgo'},
{'id':'-2112243', 'country':'84', 'url':'surat-india', 'ru':u'Сурат', 'en':'Sūrat'},
{'id':'-3102179', 'country':'85', 'url':'basrah-iraq', 'ru':u'Басра', 'en':'Basrah'},
{'id':'-559845', 'country':'33', 'url':'abbotsford-canada', 'ru':u'Abbotsford', 'en':'Abbotsford'},
{'id':'-561990', 'country':'33', 'url':'calgary-canada', 'ru':u'Калгари', 'en':'Calgary'},
{'id':'-571851', 'country':'33', 'url':'quebec-canada', 'ru':u'Квебек', 'en':'Quebec'},
{'id':'-569541', 'country':'33', 'url':'montreal-canada', 'ru':u'Монреаль', 'en':'Montreal'},
{'id':'-570760', 'country':'33', 'url':'ottawa-canada', 'ru':u'Оттава', 'en':'Ottawa'},
{'id':'-567785', 'country':'33', 'url':'laval-canada', 'ru':u'Лаваль', 'en':'Laval'},
{'id':'-564968', 'country':'33', 'url':'fredericton-canada', 'ru':u'Фредериктон', 'en':'Fredericton'},
{'id':'-564064', 'country':'33', 'url':'edmonton-canada', 'ru':u'Эдмонтон', 'en':'Edmonton'},
{'id':'-569498', 'country':'33', 'url':'moncton-canada', 'ru':u'Монктон', 'en':'Moncton'},
{'id':'-562447', 'country':'33', 'url':'charlottetown-canada', 'ru':u'Шарлоттаун', 'en':'Charlottetown'},
{'id':'-2764584', 'country':'147', 'url':'karachi-pakistan', 'ru':u'Карачи', 'en':'Karachi'},
{'id':'-2767043', 'country':'147', 'url':'lahore-pakistan', 'ru':u'Лахор', 'en':'Lahore'},
]
"""
class Command(BaseCommand):
def handle(self, *args, **options):
for city in cities:
#City.objects.get(id=city['id']).delete()
country = Country.objects.get(id=city['country'])
c = City(id=city['id'], country=country, url=city['url'])
c.translate('ru')
c.name = city['ru']
print('pre save %s'%str(c))
c.save()
print('post save %s'%str(c))

@ -1,22 +0,0 @@
# -*- coding: utf-8 -*-
from django.core.management.base import BaseCommand, CommandError
from city.models import City
from country.models import Country
from functions.form_check import translit_with_separator
from django.db import IntegrityError
class Command(BaseCommand):
def handle(self, *args, **options):
qs = City.objects.language('en').filter()
for c in qs:
url = translit_with_separator(c.name.encode('utf8'))
c.url = url
try:
c.save()
except IntegrityError:
continue
print(c.url)
#print(qs.count())

@ -1,30 +0,0 @@
# -*- coding: utf-8 -*-
import MySQLdb
from MySQLdb.cursors import DictCursor
from django.core.management.base import BaseCommand, CommandError
from django.utils import translation
from country.models import City
class Command(BaseCommand):
def handle(self, *args, **options):
db = MySQLdb.connect(host="localhost",
user="kotzilla",
passwd="qazedc",
db="test2",
charset='utf8',
cursorclass=DictCursor)
cursor = db.cursor()
sql = """SELECT title, url, inflect
FROM old_expomap.products_places
WHERE parent_id > 0 """
cursor.execute(sql)
result = cursor.fetchall()
for res in result:
name = res['title']
url = res['url']
inflect = res['inflect']
City.objects.filter(translations__name=name).update(inflect=inflect, old_url=url)
print(name.encode('utf-8'))

@ -12,8 +12,8 @@ HOTEL_URL = 'https://distribution-xml.booking.com/json/bookings.getHotels?'
HOTEL_PHOTO_URL = 'https://distribution-xml.booking.com/json/bookings.getHotelPhotos?'
username = 'expomap'
password = '33xp00m33p'
username = settings.BOOKING_USERNAME
password = settings.BOOKING_PASSWORD
langs = [code for code, name in settings.LANGUAGES]
def create_hotels(hotels, city):
@ -34,8 +34,8 @@ def create_hotels(hotels, city):
tr_model.objects.bulk_create(dj_hotels_translation)
print('city: %s'%str(city))
def main():
def main():
cities = City.objects.select_related('place_expositions', 'place_conferences').\
filter(Q(place_expositions__city__isnull=False) | Q(place_conferences__city__isnull=False)).distinct()
for city in cities:
@ -67,22 +67,4 @@ def main():
class Command(BaseCommand):
def handle(self, *args, **options):
main()
"""
request = urllib2.Request(URL)
base64string = base64.encodestring('%s:%s' % (username, password)).replace('\n', '')
request.add_header("Authorization", "Basic %s" % base64string)
try:
response = urllib2.urlopen(request)
code = response.getcode()
except urllib2.HTTPError, e:
code = e.code
except urllib2.URLError, e:
code = e.code
json_hotels = response.read()
hotels = json.loads(json_hotels)
print(hotels)
"""
main()

@ -4,13 +4,14 @@ from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from city.models import Hotel, City
from functions.files import get_alternative_filename
from concurrent.futures import ThreadPoolExecutor
username = 'expomap'
password = '33xp00m33p'
username = settings.BOOKING_USERNAME
password = settings.BOOKING_PASSWORD
HOTEL_URL = 'https://distribution-xml.booking.com/json/bookings.getHotels?'
HOTEL_PHOTO_URL = 'https://distribution-xml.booking.com/json/bookings.getHotelPhotos?'
from concurrent.futures import ThreadPoolExecutor
def upload(photo):
@ -37,15 +38,6 @@ def download_photos(photos):
result[i[0]] = i[1]
return result
"""
result = {}
for photo in photos:
res = upload(photo)
result[res[0]] = res[1]
return result
"""
def run(hotels):
ids = [str(hotel.id) for hotel in hotels]# comment after testing
url = HOTEL_PHOTO_URL+'hotel_ids=%s'%','.join(ids)

@ -1,11 +1,13 @@
import urllib2, base64
import urllib2
import base64
import json
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
from city.models import Hotel
HOTEL_URL = 'https://distribution-xml.booking.com/json/bookings.getHotels?'
username = 'expomap'
password = '33xp00m33p'
username = settings.BOOKING_USERNAME
password = settings.BOOKING_PASSWORD
def handle_hotels(hotel_ids):
@ -41,11 +43,10 @@ def main():
while hotels_todo > 0:
hotel_ids = [str(item.id) for item in list(Hotel.objects.filter(currency__isnull=True)[:100])]
handle_hotels(hotel_ids)
hotels_todo =Hotel.objects.filter(currency__isnull=True)[:100].count()
hotels_todo = Hotel.objects.filter(currency__isnull=True)[:100].count()
class Command(BaseCommand):
def handle(self, *args, **options):
main()
# fill hotel currencies
main()

@ -1,12 +1,14 @@
import urllib2, base64
import urllib2
import base64
import json
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from django.conf import settings
from city.models import Hotel
URL = 'https://distribution-xml.booking.com/json/bookings.getRooms?'
username = 'expomap'
password = '33xp00m33p'
test_id = '298239'
username = settings.BOOKING_USERNAME
password = settings.BOOKING_PASSWORD
def get_prices(rooms):
min_price = None
@ -28,6 +30,7 @@ def get_prices(rooms):
return min_price, max_price
def handle_hotels(hotel_id):
hotel_ids_str = hotel_id
new_url = URL + 'hotel_ids=%s'%hotel_ids_str
@ -48,21 +51,17 @@ def handle_hotels(hotel_id):
Hotel.objects.filter(id=hotel_id).update(min_price=min_price, max_price=max_price)
print('success')
def main():
hotel_id = test_id
for hotel in Hotel.objects.all():
try:
handle_hotels(str(hotel.id))
except:
continue
#hotels_todo = Hotel.objects.filter(currency__isnull=True)[:100].count()
#while hotels_todo > 0:
#hotel_ids = [str(item.id) for item in list(Hotel.objects.filter(currency__isnull=True)[:100])]
#handle_hotels(hotel_ids)
#hotels_todo = Hotel.objects.filter(currency__isnull=True)[:100].count()
class Command(BaseCommand):
def handle(self, *args, **options):
# update hotels prices
# runs one time in a week
main()

@ -3,147 +3,151 @@ from datetime import date
from django.db import models
from django.db.models.signals import post_save, pre_save
from django.utils import translation
from django.utils.translation import ugettext_lazy as _
from hvad.models import TranslatableModel, TranslatedFields, TranslationManager
from bitfield import BitField
from sorl.thumbnail import ImageField
# models
from directories.models import Iata
from service.models import Service
from exposition.models import Exposition
from place_exposition.models import PlaceExposition
from organiser.models import Organiser
from conference.models import Conference
from seminar.models import Seminar
from webinar.models import Webinar
# custom functions
from functions.db import db_table_exists
from functions.signal_handlers import post_save_handler, pre_save_handler
from functions.models_methods import ExpoManager, CityManager
from service.models import Service
#flags = [str(item.url) for item in Service.objects.all()] if db_table_exists('service_service') else []
#check if table exist and create flags if true
flags = [str(item.url) for item in Service.objects.all()] if db_table_exists('service_service') else []
flags = ['catalog', 'translator', 'participation', 'remote', 'tickets', 'visit', 'buildstand'] if db_table_exists('service_service') else []
class City(TranslatableModel):
"""
Create City model
Uses hvad.TranslatableModel which is child of django.db.models class
City describes cities with translations in the project
all cities with id from booking.com -> id city on booking equal this city objects
"""
objects = ExpoManager()
used = CityManager()
catalog = '/city/'
services = BitField(flags=flags)
url = models.SlugField(unique=True)
#
old_url = models.CharField(max_length=55)
inflect = models.CharField(max_length=255, blank=True)
#relations
country = models.ForeignKey('country.Country', null=True, on_delete=models.PROTECT, related_name='cities')
code_IATA = models.ForeignKey(Iata, blank=True, null=True)
population = models.PositiveIntegerField(blank=True, null=True)
phone_code = models.PositiveIntegerField(blank=True, null=True)
url = models.SlugField(verbose_name=_(u'Url'), unique=True)
logo = models.ImageField(verbose_name='Logo', upload_to='city/logo/', blank=True, max_length=255)
old_url = models.CharField(verbose_name=_(u'Url старой бд'), max_length=55)
# inflect name for russian language. example- в Москве
inflect = models.CharField(verbose_name=_(u'Склонение'), max_length=255, blank=True)
country = models.ForeignKey('country.Country', verbose_name=_(u'Страна'), null=True, on_delete=models.PROTECT,
related_name='cities')
code_IATA = models.ForeignKey(Iata, verbose_name=_(u'IATA'), blank=True, null=True)
population = models.PositiveIntegerField(verbose_name=_(u'Население'), blank=True, null=True)
phone_code = models.PositiveIntegerField(verbose_name=_(u'Тел. код '), blank=True, null=True)
#translated fields
translations = TranslatedFields(
name = models.CharField(max_length=50),
region = models.CharField(max_length=255),
transport = models.TextField(blank=True),
description = models.TextField(blank=True),
famous_places = models.TextField(blank=True),
shoping = models.TextField(blank=True),
name = models.CharField(verbose_name=_(u'Название'), max_length=50),
region = models.CharField(verbose_name=_(u'Регион'), max_length=255),
transport = models.TextField(verbose_name=_(u'Транспорт'), blank=True),
description = models.TextField(verbose_name=_(u'Описание'), blank=True),
famous_places = models.TextField(verbose_name=_(u'Знаменитые места'), blank=True),
shoping = models.TextField(verbose_name=_(u'Магазины'), blank=True),
#-----meta
title = models.CharField(max_length=255),
descriptions = models.CharField(max_length=255),
keywords = models.CharField(max_length=255),
)
# fields saves information about creating and changing model
created = models.DateTimeField(auto_now_add=True)
modified = models.DateTimeField(auto_now=True)
logo = models.ImageField(verbose_name='Logo', upload_to='city/logo/', blank=True, max_length=255)
class Meta:
ordering = ['translations__name']
#class Meta:
#ordering = ['translations__name']
def __unicode__(self):
return self.lazy_translation_getter('name', self.pk)
def get_hotels(self):
"""
returns list of 4 hotels in current city
"""
return list(self.hotels.all()[:4])
def get_events(self):
"""
returns nearest expos in this city
:return:
"""
now = date.today()
return Exposition.objects.filter(data_begin__gte=now, city=self).order_by('data_begin')[:3]
def get_places(self):
return PlaceExposition.objects.filter(city=self)[:3]
def get_organisers(self):
return Organiser.objects.filter(city=self)
def get_permanent_url(self):
return self.catalog+self.url
def events_catalog(self):
return Exposition.catalog+'city-%s'%self.url
def places_catalog(self):
return PlaceExposition.catalog+'city-%s'%self.url
def expositions_number(self):
return Exposition.objects.filter(city=self.id).count()
def conferences_number(self):
return Conference.objects.filter(city=self.id).count()
def seminars_number(self):
return Seminar.objects.filter(city=self.id).count()
def webinars_number(self):
return Webinar.objects.filter(city=self.id).count()
def get_parent(self):
parent = {'text' : self.country.name, 'id': self.country.id, 'name': 'co',
'parent':{'text' : self.country.area.name, 'id': self.country.area.id, 'name': 'area'}}
"""
returns dictionary of parents
uses in search in ajax requests
"""
parent = {'text': self.country.name, 'id': self.country.id, 'name': 'co',
'parent': {
'text': self.country.area.name,
'id': self.country.area.id,
'name': 'area'}
}
return parent
def get_sub_categories(self):
"""
uses in search
city has no children objects, so returns empty list
"""
return []
def get_index_text(self):
"""
returns string of names in all languages for indexing it in search engine
"""
translation.activate('ru')
translations = self.translations.all()
names = ' '.join([tr.name for tr in translations])
return names
class Hotel(TranslatableModel):
url = models.URLField(max_length=255)
city = models.ForeignKey(City, related_name='hotels')
ranking = models.FloatField(blank=True, null=True)
hotel_class = models.CharField(max_length=10, blank=True)
latitude = models.FloatField(blank=True, null=True)
longitude = models.FloatField(blank=True, null=True)
photo = ImageField(upload_to='hotels')
currency = models.CharField(max_length=10)
min_price = models.FloatField(blank=True, null=True)
max_price = models.FloatField(blank=True, null=True)
checked = models.NullBooleanField(blank=True, null=True)
"""
Hotels downloaded from booking.com with booking api
"""
url = models.URLField(verbose_name=_(u'Url'), max_length=255)
city = models.ForeignKey(City, verbose_name=_(u'Город'), related_name='hotels')
ranking = models.FloatField(verbose_name=_(u'Рейтинг'), blank=True, null=True)
hotel_class = models.CharField(verbose_name=_(u'Клас отеля'), max_length=10, blank=True)
latitude = models.FloatField(verbose_name=_(u'Широта'), blank=True, null=True)
longitude = models.FloatField(verbose_name=_(u'Долгота'), blank=True, null=True)
photo = ImageField(verbose_name=_(u'Фото'), upload_to='hotels')
currency = models.CharField(verbose_name=_(u'Валюта'), max_length=10)
min_price = models.FloatField(verbose_name=_(u'Мин. цена'), blank=True, null=True)
max_price = models.FloatField(verbose_name=_(u'Макс. цена'), blank=True, null=True)
checked = models.NullBooleanField(verbose_name=_(u'Проверено'), blank=True, null=True)
translations = TranslatedFields(
name = models.CharField(max_length=255, blank=True),
address = models.CharField(max_length=255, blank=True)
name=models.CharField(verbose_name=_(u'Название'), max_length=255, blank=True),
address=models.CharField(verbose_name=_(u'Адрес'), max_length=255, blank=True)
)

@ -4,6 +4,6 @@ from views import CityView
urlpatterns = patterns('',
url(r'get-city/', 'city.views.get_city'),
url(r'(?P<slug>.*)', CityView.as_view()),
# commented for good times
#url(r'(?P<slug>.*)', CityView.as_view()),
)

@ -5,12 +5,18 @@ from models import City
class CityView(DetailView):
"""
this view is not used yet
"""
model = City
slug_field = 'url'
template_name = 'city/city.html'
template_name = 'client/city/city.html'
def get_city(request):
"""
returns filtered cities in current language in json format
"""
if request.is_ajax():
country = request.GET['country']
term = request.GET['term'].capitalize()

Loading…
Cancel
Save