from django.core.management.base import BaseCommand, CommandError import urllib2, base64, json from country.models import Country from city.models import City from functions.form_check import translit_with_separator import string, random def random_url(): salt = 'randomurl-' r = ''.join(random.choice(string.ascii_lowercase) for _ in range(8)) return salt+r class Command(BaseCommand): # booking data b_username = 'expomap' b_password = '33xp00m33p' def fill_city(self, d): lang = d['languagecode'] id = d['city_id'] country_code = d['countrycode'] name = d['name'] country = Country.objects.filter(country_code=country_code)[0] try: city = City.objects.get(id=id) except City.DoesNotExist: try: url = translit_with_separator(name)+'-%s'%country_code except: url = random_url() city = City(country=country, id=id, url=url) city.translate(lang) city.name = name try: city.save() except Exception, e: with open('/home/kotzilla/Documents/qwer/proj/city/management/commands/errors.txt', 'a') as f: f.write('error: %s\n city: %s\n'%(str(e), str(d))) return try: url = translit_with_separator(name)+'-%s'%country_code except: url = random_url() city.url = url try: city.save() except Exception, e: with open('/home/kotzilla/Documents/qwer/proj/city/management/commands/errors.txt', 'a') as f: f.write('error: %s\n city: %s\n'%(str(e), str(d))) return try: tr = City._meta.translations_model.objects.get(language_code=lang,master__id=getattr(city, 'id')) except: city.translate(lang) city.name = name try: city.save() return except Exception, e: with open('/home/kotzilla/Documents/qwer/proj/city/management/commands/errors.txt', 'a') as f: f.write('error: %s\n city: %s\n'%(str(e), str(d))) return tr.name = name tr.save() return def handle(self, *args, **options): ROWS = 1000 qs = Country.objects.filter() codes = [country.country_code for country in qs] #print(codes[86:]) for code in codes[86:]:# start from 7 cause previous already filled(delete this) offset= 0 run = True while(run): url = 'https://distribution-xml.booking.com/json/bookings.getCities?countrycodes=%s&rows=%s&offset=%s&lanuagecodes=ru,en'%(code ,ROWS, offset) request = urllib2.Request(url) base64string = base64.encodestring('%s:%s' % (self.b_username, self.b_password)).replace('\n', '') request.add_header("Authorization", "Basic %s" % base64string) try: response = urllib2.urlopen(request) except urllib2.HTTPError, e: continue #raise CommandError('request failed. code error: %s'%code) except urllib2.URLError, e: continue #raise CommandError('request failed. code error: %s'%code) if response: try: json_list = response.read() except Exception, e: with open('/home/kotzilla/Documents/qwer/proj/city/management/commands/errors.txt', 'a') as f: f.write('error: %s\n'%str(e)) continue else: with open('/home/kotzilla/Documents/qwer/proj/city/management/commands/errors.txt', 'a') as f: f.write('error: bad request\n') continue try: cities = json.loads(json_list) except: continue for city in cities: if isinstance(city, dict): self.fill_city(city) print('city: %s, country: %s, language: %s'%(city.get('name'), city.get('countrycode'), city.get('languagecode'))) if len(cities) < ROWS: run = False else: offset += ROWS