mirror of
https://github.com/netzbegruenung/green-spider.git
synced 2024-05-05 10:33:39 +02:00
Marian Steinbach
618e29d763
* CLI: remove 'jobs' command, add 'manager' * Add job definition * Move jobs to manage folder * Rename jobs to manager * Add rq and redis dependencies * Add docker-compose YAML * Downgrade to alpine 3.8 * Adjust paths in Dockerfile, remove entrypoint * Rename 'make spiderjobs' to 'make jobs' * Fix docker exectution * Adapt 'make jobs' * Fix metadata scheme * Add docker dependency * Rendomize queue (a bit) * Use latest image, remove debug output * Make docker-compose file downwards-compatible * Use latest instead of dev image tag * Update docker-compose.yaml * Adapt job start script * Fix redis connection in manager * Add support for increasing timeout via environment variable * Adapt load_in_browser to cookies table schema change * Fix execution * Mitigate yaml warning * Bump some dependency versions * Report resource usage stats for each job * checks/load_in_browser: Return DOM size, prevent multiple page loads * Update .dockerignore * Code update * Script update * Update README.md * WIP * WIP commit * Update Dockerfile to alpine:edge and chromium v90 * Update TestCertificateChecker * Set defaults for __init__ function * Detect sunflower theme * Update unit test for new datetime (zero-basing) * Set logging prefs from Chromium in a new way * Move datastore client instantiation As it is not needed for all commands * Change green-directory repository URL * Add git settings for cloning green-directory * Pin alpine version 3.14, fix py3-cryptography * Use plain docker build progress output * Add volumes to 'make test' docker run command * Fix bug * Update example command in README * Update dependencies * Add creation of Kubernetes jobs
68 lines
1.9 KiB
Python
68 lines
1.9 KiB
Python
import config
|
|
|
|
import os
|
|
from datetime import datetime
|
|
import time
|
|
import random
|
|
from pathlib import Path
|
|
|
|
import kubernetes
|
|
|
|
PENDING_LIMIT = 2
|
|
RUNNING_LIMIT = 4
|
|
|
|
INTERVAL = 10 # Seconds
|
|
|
|
def main():
|
|
|
|
# Get jobs
|
|
jobs = list(Path("./k8s-jobs").rglob("*.yaml"))
|
|
random.seed()
|
|
random.shuffle(jobs)
|
|
|
|
kubernetes.config.load_kube_config(context='giantswarm-5jka7')
|
|
v1client = kubernetes.client.CoreV1Api()
|
|
k8sclient = kubernetes.client.ApiClient()
|
|
|
|
start = datetime.utcnow()
|
|
jobs_queued = 0
|
|
|
|
while len(jobs) > 0:
|
|
# Check whether there are pods pending
|
|
pending_pods = v1client.list_pod_for_all_namespaces(
|
|
watch=False,
|
|
field_selector='status.phase=Pending',
|
|
label_selector='app=green-spider')
|
|
pending = list(pending_pods.items)
|
|
|
|
# Get running pods
|
|
running_pods = v1client.list_pod_for_all_namespaces(
|
|
watch=False,
|
|
field_selector='status.phase=Running',
|
|
label_selector='app=green-spider')
|
|
running = list(running_pods.items)
|
|
|
|
now = datetime.utcnow()
|
|
duration = now - start
|
|
|
|
# Add new job to the queue
|
|
if len(pending) < PENDING_LIMIT and len(running) < RUNNING_LIMIT:
|
|
to_be_queued = RUNNING_LIMIT - len(running)
|
|
for _ in range(to_be_queued):
|
|
job_path = jobs.pop(0)
|
|
jobs_queued += 1
|
|
|
|
duration_per_job = duration / jobs_queued
|
|
jobs_remaining = len(jobs)
|
|
|
|
print(f'{jobs_queued} jobs queued in {duration} - {jobs_remaining} jobs (estimated {duration_per_job * jobs_remaining}) remaining at {int(duration_per_job.total_seconds())} seconds per job on average')
|
|
kubernetes.utils.create_from_yaml(k8sclient, job_path)
|
|
os.remove(job_path)
|
|
|
|
time.sleep(INTERVAL)
|
|
|
|
print('No more jobs left. Done.')
|
|
|
|
if __name__ == '__main__':
|
|
main()
|