import pickle, json, urllib2 import urllib2, base64 from django.conf import settings from django.core.management.base import BaseCommand, CommandError from city.models import Hotel, City from functions.files import get_alternative_filename username = 'expomap' password = '33xp00m33p' HOTEL_URL = 'https://distribution-xml.booking.com/json/bookings.getHotels?' HOTEL_PHOTO_URL = 'https://distribution-xml.booking.com/json/bookings.getHotelPhotos?' from concurrent.futures import ThreadPoolExecutor def upload(photo): url = photo['url_max300'] name = url.split('/')[-1] alt_name = get_alternative_filename(settings.MEDIA_ROOT+'hotels/', name) download_to = settings.MEDIA_ROOT+'hotels/'+alt_name try: response = urllib2.urlopen(url, timeout=3) except: return (photo['hotel_id'], '') with open(download_to,'wb') as f: f.write(response.read()) f.close() return (photo['hotel_id'], 'hotels/'+alt_name) def download_photos(photos): result = {} with ThreadPoolExecutor(5) as executor: for i in executor.map(upload, photos): result[i[0]] = i[1] return result """ result = {} for photo in photos: res = upload(photo) result[res[0]] = res[1] return result """ def run(hotels): ids = [str(hotel.id) for hotel in hotels]# comment after testing url = HOTEL_PHOTO_URL+'hotel_ids=%s'%','.join(ids) request = urllib2.Request(url) base64string = base64.encodestring('%s:%s' % (username, password)).replace('\n', '') request.add_header("Authorization", "Basic %s" % base64string) try: response = urllib2.urlopen(request) code = response.getcode() except urllib2.HTTPError, e: code = e.code except urllib2.URLError, e: code = e.code json_photos = response.read() photos = json.loads(json_photos) result = download_photos(photos) for key, value in result.iteritems(): Hotel.objects.filter(id=key).update(photo=value) print(key) def main(): #hotels = Hotel.objects.all() hotels = Hotel.objects.filter(photo='') fr = 0 to = 20 step = 20 while(fr < len(hotels)): cur_hotels = hotels[fr:to] run(cur_hotels) fr += step to += step class Command(BaseCommand): def handle(self, *args, **options): main()