Updated code to keep up with the latest version of extension (3.0.3).

README.md

@@ -3,24 +3,41 @@
 Google provided us with some impressive wallpapers.
 
 * [Chrome OS][Chrome OS link]
-* [Google Maps][Maps link]
+* [Google Maps][Maps link] (code exported from 3.0.3 version)
 
 Now you can use them locally!
 
 ## Usage
 
-Just use Python 2.7.+, no dependencies required.
+### Chrome OS version
 
+Just use Python 3.4+, no dependencies required.
 ```
-$ python2 chrome-os/download-wallpapers.py
-$ python2 maps/download-wallpapers.py
+$ python3 chrome-os/download-wallpapers.py
 ```
 
-Wallpapers will be downloaded to relative `wallpapers` directories,
-such as `chrome-os/wallpapers` and `maps/wallpapers`.
+Wallpapers will be downloaded to `chrome-os/wallpapers`.
+Be wary of download sizes: Chrome OS wallpapers are 200+ MB combined.
 
-Be wary of download sizes: Chrome OS wallpapers are 200+ MB combined and Maps are 1.1+ GB.
+### Maps version
 
+Just use Python 3.4+, no dependencies required. But, if you want but if you want a progress bar to show the percentage of the downloads, consider installing [`tqdm`][tqdm_link] package.
+```
+$ python3 maps/download-wallpapers.py
+```
+
+Wallpapers will be downloaded to `maps/wallpapers`.
+Be wary of download sizes: Maps are 1.8+ GB (up to 3+ GB).
+
+Maps version now can use two different ways to download the image:
+* Official links - links used by the plugin when you use the `Download wallpaper` button (with watermarks).
+* Internal links - links used by the plugin to show you the image in new tab (without watermarks).
+
+Both of this methods provide you with the 1800x1200 images, but the "Official links" method provide you with watermarked images and no optimization, so images will weight more.
+
+## Code doesn't work?
+Check the version of the [Chrome extention][Maps link], if it's bigger than `3.0.3`, then maybe they just updated the API and it's your time to change the code. Look at the code of the extention and extract the links and ids of the images. And don't forget to Pull Request the changes.
 
 [Chrome OS link]: https://chrome.google.com/webstore/detail/chrome-os-wallpapers/dkfibabkihblcenahmcdmfepojcejoan
 [Maps link]: https://chrome.google.com/webstore/detail/earth-view-from-google-ma/bhloflhklmhfpedakmangadcdofhnnoh
+[tqdm_link]: https://pypi.org/project/tqdm/

maps/download-wallpapers.py

@@ -1,7 +1,7 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 '''
-Copyright 2014 Artur Dryomov
+Copyright 2020 Artur Dryomov & Vladislav Kuleykin
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -20,81 +20,138 @@
 import base64
 import json
 import os
-import urllib2
-import urlparse
-
-
-REMOTE_URL = "https://www.gstatic.com/prettyearth/"
+import urllib
+import urllib.request
+
+try:
+    from tqdm import tqdm
+    use_tqdm = True
+    write = tqdm.write
+except ImportError:
+    use_tqdm = False
+    write = print
+
+REMOTE_URLS = [
+    "https://www.gstatic.com/prettyearth/assets/data/v3/{}.json",
+    "https://earthview.withgoogle.com/download/{}.jpg",
+]
 REMOTE_IDS_PATH = "ids.json"
-
 LOCAL_PATH = "wallpapers"
 
+PROMPT_1 = "Download fullsize images? Fullsize images contain watermarks (y/n) "
+PROMPT_2 = "Skip already downloaded wallpapers? (y/n) "
 
-def download_wallpapers():
-    create_wallpapers_path()
-
-    print(":: Downloading Google Maps wallpapers.") 
-
-    for wallpaper_id in get_wallpaper_ids():
-        print(wallpaper_id)
-
-        wallpaper_info = get_wallpaper_info(wallpaper_id)
 
-        wallpaper_path = get_wallpaper_path(wallpaper_id)
-        wallpaper_bytes = get_wallpaper_bytes(wallpaper_info)
-
-        download_wallpaper(wallpaper_path, wallpaper_bytes)
-
-
-def create_wallpapers_path():
+def download_wallpapers():
     wallpapers_path = get_wallpapers_path()
-
-    if not os.path.exists(wallpapers_path):
-        os.makedirs(wallpapers_path)
+    create_directory(wallpapers_path)
+
+    # Choose the url to download from.
+    prompt1_res = input(PROMPT_1).lower() == "y"
+    REMOTE_URL = REMOTE_URLS[prompt1_res]
+
+    # Check the list of already downloaded wallpapers.
+    downloaded_wallpapers = os.listdir(wallpapers_path)
+    downloaded_wallpapers_ids = list(map(lambda x: x.rsplit('.', 1)[0],
+                                    downloaded_wallpapers))
+    # Prompt the user to skip already downloaded wallpapers.
+    wallpapers_ids = set(get_wallpaper_ids())
+    if not wallpapers_ids.isdisjoint(downloaded_wallpapers_ids):
+        if input(PROMPT_2).lower() == 'y':
+            print('Skipping downloaded wallpapers.')
+            wallpapers_ids.difference_update(downloaded_wallpapers_ids)
+
+    # Start the download.
+    print(":: Downloading Google Maps wallpapers.")
+    if use_tqdm:
+        pbar = tqdm.tqdm(total=len(wallpapers_ids))
+    for wallpaper_id in sorted(wallpapers_ids):
+        wallpaper_url = REMOTE_URL.format(wallpaper_id)
+
+        # Download the wallpaper using the chosen URL
+        try:
+            wallpaper_bytes = REQ_FUNCS[prompt1_res](wallpaper_url)
+        except ValueError:
+            write(f"[WARNING] Could not download the wallpaper with id {wallpaper_id}, retrying with other method.")
+            try:
+                wallpaper_url = REMOTE_URLS[not prompt1_res].format(wallpaper_id)
+                wallpaper_bytes = REQ_FUNCS[not prompt1_res](wallpaper_url)
+            except ValueError:
+                write(f"[ERROR] Can't download the wallpaper with id {wallpaper_id} using both methods. Skipping it.")
+                continue
+
+        wallpaper_path = get_wallpaper_path(wallpapers_path, wallpaper_id)
+        save_wallpaper(wallpaper_path, wallpaper_bytes)
+        if use_tqdm:
+            pbar.desc = "Wallpaper id: {}".format(wallpaper_id)
+            pbar.update(1)
+        else:
+            print('[OK] Downloaded wallpaper with id: {}'.format(wallpaper_id))
+
+
+def create_directory(dir_path):
+    """Create the directory in the specified path."""
+    if not os.path.exists(dir_path):
+        os.makedirs(dir_path)
 
 
 def get_wallpapers_path():
+    """Return the absolute path to the wallpapers directory."""
     wallpapers_path = os.path.dirname(os.path.realpath(__file__))
-
     return os.path.abspath(os.path.join(wallpapers_path, LOCAL_PATH))
 
 
 def get_wallpaper_ids():
+    """Load all the wallpaper IDs from the file."""
     with open(get_wallpaper_ids_path()) as wallpaper_ids_file:
         return json.load(wallpaper_ids_file)
 
 
 def get_wallpaper_ids_path():
+    """Return the absolute path to the wallpapers directory."""
     return os.path.join(os.path.dirname(__file__), REMOTE_IDS_PATH)
 
 
-def get_wallpaper_info(wallpaper_id):
-    wallpaper_info_url = get_wallpaper_info_url(wallpaper_id)
+def download_wallpaper_official(wallpaper_url):
+    """Download the wallpaper using the official URL."""
+    response = urllib.request.urlopen(wallpaper_url)
+    if not response.headers.get('Content-Type') in ['image/jpeg', 'application/octet-stream']:
+        raise ValueError("Response should contain image, maybe it's 404 page.")
+    return response.read()
 
-    return json.load(urllib2.urlopen(wallpaper_info_url))["dataUri"]
 
+def get_wallpaper_bytes(wallpaper_info):
+    """Get the wallpapers from the json response."""
+    bytes_start_position = wallpaper_info.index(",") + 1
+    return base64.b64decode(wallpaper_info[bytes_start_position:])
 
-def get_wallpaper_info_url(wallpaper_id):
-    return urlparse.urljoin(REMOTE_URL, "{id}.json".format(id=wallpaper_id))
 
+def download_wallpaper_from_plugin(wallpaper_info_url):
+    """Download the wallpaper as the plugin do."""
+    try:
+        response = urllib.request.urlopen(wallpaper_info_url)
+        json_data = json.loads(response.read())['dataUri']
+    except json.decoder.JSONDecodeError:
+        raise ValueError("Response should contain json data, maybe it's 404 page.")
+    return get_wallpaper_bytes(json_data)
 
-def get_wallpaper_path(wallpaper_id):
-    wallpapers_path = get_wallpapers_path()
-    wallpaper_filename = "{id}.jpg".format(id=wallpaper_id)
 
+def get_wallpaper_path(wallpapers_path, wallpaper_id):
+    """Return the path for the wallpaper with specified id."""
+    wallpaper_filename = "{id}.jpg".format(id=wallpaper_id)
     return os.path.join(wallpapers_path, wallpaper_filename)
 
 
-def get_wallpaper_bytes(wallpaper_info):
-    bytes_start_position = wallpaper_info.index(",") + 1
-
-    return base64.b64decode(wallpaper_info[bytes_start_position:])
-
-
-def download_wallpaper(wallpaper_path, wallpaper_bytes):
+def save_wallpaper(wallpaper_path, wallpaper_bytes):
+    """Save the wallpaper to disk."""
     with open(wallpaper_path, "wb") as wallpaper_file:
         wallpaper_file.write(wallpaper_bytes)
 
 
+REQ_FUNCS = {
+    0: download_wallpaper_from_plugin,
+    1: download_wallpaper_official,
+}
+
 if __name__ == "__main__":
     download_wallpapers()