mirror of
https://github.com/netzbegruenung/green-spider-api.git
synced 2024-05-07 19:23:40 +02:00
Adapt CompactResults to use ES
This commit is contained in:
parent
110a3c0ac0
commit
9ea662c582
10
main.py
10
main.py
|
@ -9,13 +9,18 @@ from falcon import media
|
||||||
import jsonhandler
|
import jsonhandler
|
||||||
|
|
||||||
from google.cloud import datastore
|
from google.cloud import datastore
|
||||||
|
from elasticsearch import Elasticsearch
|
||||||
|
|
||||||
credentials_path = getenv('GCLOUD_DATASTORE_CREDENTIALS_PATH')
|
credentials_path = getenv('GCLOUD_DATASTORE_CREDENTIALS_PATH')
|
||||||
datastore_client = datastore.Client.from_service_account_json(credentials_path)
|
datastore_client = datastore.Client.from_service_account_json(credentials_path)
|
||||||
|
|
||||||
|
es = Elasticsearch([{'host': 'elasticsearch', 'port': 9200}])
|
||||||
|
|
||||||
|
es_doc_type = 'result'
|
||||||
spider_results_kind = 'spider-results'
|
spider_results_kind = 'spider-results'
|
||||||
webscreenshots_kind = 'webscreenshot'
|
webscreenshots_kind = 'webscreenshot'
|
||||||
|
|
||||||
|
es_index_name = spider_results_kind
|
||||||
|
|
||||||
def convert_datastore_datetime(field):
|
def convert_datastore_datetime(field):
|
||||||
"""
|
"""
|
||||||
|
@ -43,10 +48,9 @@ def flatten(d, parent_key='', sep='.'):
|
||||||
return dict(items)
|
return dict(items)
|
||||||
|
|
||||||
|
|
||||||
def get_compact_results(client):
|
def get_compact_results():
|
||||||
query = client.query(kind=spider_results_kind,
|
query = client.query(kind=spider_results_kind,
|
||||||
order=['-created'],
|
order=['-created'],
|
||||||
#projection=['created', 'meta', 'score'],
|
|
||||||
)
|
)
|
||||||
|
|
||||||
out = []
|
out = []
|
||||||
|
@ -139,7 +143,7 @@ class CompactResults(object):
|
||||||
"""
|
"""
|
||||||
Returns compact sites overview and score
|
Returns compact sites overview and score
|
||||||
"""
|
"""
|
||||||
out = get_compact_results(datastore_client)
|
out = get_compact_results()
|
||||||
|
|
||||||
maxage = 6 * 60 * 60 # six hours in seconds
|
maxage = 6 * 60 * 60 # six hours in seconds
|
||||||
resp.cache_control = ["max_age=%d" % maxage]
|
resp.cache_control = ["max_age=%d" % maxage]
|
||||||
|
|
Loading…
Reference in a new issue