mirror of
https://github.com/netzbegruenung/green-spider.git
synced 2024-05-04 01:53:40 +02:00
Merge pull request #41 from ctr49/master
Importiere alle Website-URLs je Eintrag aus green-directory, statt nur einen
This commit is contained in:
commit
b1d0a517bb
16
spider.py
16
spider.py
|
@ -622,16 +622,16 @@ def main():
|
|||
try:
|
||||
if entry['urls'][n]['type'] == "WEBSITE":
|
||||
website_url = entry['urls'][n]['url']
|
||||
if website_url:
|
||||
input_entries.append({
|
||||
"url": website_url,
|
||||
"level": entry.get("level"),
|
||||
"state": entry.get("state"),
|
||||
"district": entry.get("district"),
|
||||
"city": entry.get("city"),
|
||||
})
|
||||
except NameError as ne:
|
||||
logging.error("Error in %s: 'url' key missing (%s)" % (repr_entry(entry), entry['urls'][n]))
|
||||
if website_url:
|
||||
input_entries.append({
|
||||
"url": website_url,
|
||||
"level": entry.get("level"),
|
||||
"state": entry.get("state"),
|
||||
"district": entry.get("district"),
|
||||
"city": entry.get("city"),
|
||||
})
|
||||
|
||||
|
||||
# randomize order, to distribute requests over servers
|
||||
|
|
Loading…
Reference in a new issue