Add icon pull logic

This commit is contained in:
Marian Steinbach 2018-05-25 19:11:09 +02:00
parent 8a7cf17701
commit 45c4d72326
1 changed files with 58 additions and 7 deletions

View File

@ -8,6 +8,7 @@ from socket import gethostbyname_ex
from urllib.parse import urljoin
from urllib.parse import urlparse
import certifi
import hashlib
import json
import logging
import os
@ -34,7 +35,7 @@ read_timeout = 10
# Git repo for our data
green_directory_repo = 'https://github.com/netzbegruenung/green-directory.git'
# folder in that repo that holds the data
green_direcory_data_path = 'data/countries/de'
green_direcory_data_path = 'data/countries/de/bb'
green_directory_local_path = './cache/green-directory'
result_path = '/out'
@ -62,7 +63,7 @@ def dir_entries():
Iterator over all data files in the cloned green directory
"""
path = os.path.join(green_directory_local_path, green_direcory_data_path)
for root, dirs, files in os.walk(path):
for root, _, files in os.walk(path):
for fname in files:
filepath = os.path.join(root, fname)
@ -135,6 +136,49 @@ def normalize_title(s):
s = s.strip()
return s
def download_icon(icon_url):
"""
Download an icon from the given URL and store it with
a file name of <hash>.<ending>
"""
default_endings = {
"image/x-icon": "ico",
"image/vnd.microsoft.icon": "ico",
"image/png": "png",
"image/jpeg": "jpg",
}
# Download the icon
r = requests.get(icon_url)
r.raise_for_status()
content_hash = hashlib.md5(r.content).hexdigest()
extension = ""
file_name = os.path.basename(icon_url)[-1]
if file_name != "" and "." in file_name:
ext = file_name.split(".")[-1]
if ext != "":
extension = ext
if extension == "":
# derive from content type
t = r.headers.get('content-type')
try:
extension = default_endings[t]
except KeyError:
logging.error("No file ending defined for icon type '%s'" % t)
return None
filename = content_hash + "." + extension.lower()
path = siteicons_path + os.path.sep + filename
with open(path, 'wb') as iconfile:
iconfile.write(r.content)
return filename
def check_responsiveness(url):
"""
Checks
@ -427,12 +471,12 @@ def check_site(entry):
except requests.exceptions.ConnectionError as e:
logging.error(str(e) + " " + check_url)
check['error'] = "connection"
except requests.exceptions.Timeout as e:
logging.error(str(e) + " " + check_url)
check['error'] = "connection_timeout"
except requests.exceptions.ReadTimeout as e:
logging.error(str(e) + " " + check_url)
check['error'] = "read_timeout"
except requests.exceptions.Timeout as e:
logging.error(str(e) + " " + check_url)
check['error'] = "connection_timeout"
except Exception as e:
logging.error(str(e) + " " + check_url)
check['error'] = "unknown"
@ -452,7 +496,14 @@ def check_site(entry):
continue
if c['content']['icon'] is not None:
icons.add(c['content']['icon'])
result['details']['icons'] = sorted(list(icons))
downloaded_icons = set()
for icon_url in icons:
logging.info("Getting icon %s" % icon_url)
try:
downloaded_icons.add(download_icon(icon_url))
except Exception as e:
logging.error("Could not download icon: %s" % e)
result['details']['icons'] = sorted(list(downloaded_icons))
# collect feeds
feeds = set()
@ -632,7 +683,7 @@ def main():
"district": entry.get("district"),
"city": entry.get("city"),
})
except NameError as ne:
except NameError:
logging.error("Error in %s: 'url' key missing (%s)" % (repr_entry(entry), entry['urls'][n]))