diff --git a/Dockerfile b/Dockerfile index ec6bb4f..359600a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM python:3.12-slim RUN apt-get update && apt-get install -y --no-install-recommends \ skopeo \ + git \ && rm -rf /var/lib/apt/lists/* WORKDIR /app @@ -9,5 +10,8 @@ COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY check_versions.py . +COPY request_upgrades.py . +COPY web.py . +COPY templates/ templates/ CMD ["python", "check_versions.py"] diff --git a/kubernetes/rbac.yaml b/kubernetes/rbac.yaml index 46c1b13..6156b2a 100644 --- a/kubernetes/rbac.yaml +++ b/kubernetes/rbac.yaml @@ -1,4 +1,5 @@ --- +# ── version-tracker: read-only pod/namespace lister (weekly report) ────────── apiVersion: v1 kind: ServiceAccount metadata: @@ -6,7 +7,6 @@ metadata: namespace: k8s-version-tracker --- -# Cluster-wide read access to list pods in all namespaces apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -29,3 +29,93 @@ subjects: - kind: ServiceAccount name: version-tracker namespace: k8s-version-tracker + +--- +# ── upgrade-requester: read-only pod lister (upgrades go through GitOps/ArgoCD) ─ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: upgrade-requester + namespace: k8s-version-tracker + +--- +# Reuse the same read-only ClusterRole — no direct workload patching needed. +# Image updates are committed to argocd-gitops and ArgoCD syncs automatically. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: upgrade-requester +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: version-tracker-reader +subjects: + - kind: ServiceAccount + name: upgrade-requester + namespace: k8s-version-tracker + +--- +# ── version-tracker-web: dashboard service account ─────────────────────────── +apiVersion: v1 +kind: ServiceAccount +metadata: + name: version-tracker-web + namespace: k8s-version-tracker + +--- +# Cluster-wide read access: pods, namespaces, and workload owners (to map pods → apps) +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: version-tracker-web-reader +rules: + - apiGroups: [""] + resources: ["pods", "namespaces"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["deployments", "replicasets", "statefulsets", "daemonsets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["get", "list", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: version-tracker-web-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: version-tracker-web-reader +subjects: + - kind: ServiceAccount + name: version-tracker-web + namespace: k8s-version-tracker + +--- +# Namespace-scoped: create upgrade Jobs in this namespace only +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: upgrade-trigger + namespace: k8s-version-tracker +rules: + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "list", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: upgrade-trigger + namespace: k8s-version-tracker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: upgrade-trigger +subjects: + - kind: ServiceAccount + name: version-tracker-web + namespace: k8s-version-tracker diff --git a/kubernetes/upgrade-cronjob.yaml b/kubernetes/upgrade-cronjob.yaml index 5187505..4ea196c 100644 --- a/kubernetes/upgrade-cronjob.yaml +++ b/kubernetes/upgrade-cronjob.yaml @@ -42,8 +42,8 @@ spec: - name: GITEA_REPO_URL value: "https://repo.adservio.us/ai_approver/argocd-gitops.git" - name: GITEA_USERNAME - value: "ai_approver" - - name: GITEA_COMMITTER_NAME + value: "ai_approver" + - name: GITEA_COMMITTER_NAME value: "k8s-version-tracker" - name: GITEA_COMMITTER_EMAIL value: "k8s-version-tracker@themosers.club" diff --git a/kubernetes/web-deployment.yaml b/kubernetes/web-deployment.yaml new file mode 100644 index 0000000..873c537 --- /dev/null +++ b/kubernetes/web-deployment.yaml @@ -0,0 +1,67 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: version-tracker-web + namespace: k8s-version-tracker + labels: + app: version-tracker-web +spec: + replicas: 1 + selector: + matchLabels: + app: version-tracker-web + template: + metadata: + labels: + app: version-tracker-web + spec: + serviceAccountName: version-tracker-web + containers: + - name: web + image: registry.storedbox.net/k8s-version-tracker:latest + imagePullPolicy: Always + command: ["python", "web.py"] + ports: + - containerPort: 8080 + name: http + env: + - name: MATTERMOST_WEBHOOK_URL + valueFrom: + secretKeyRef: + name: version-tracker-secrets + key: MATTERMOST_WEBHOOK_URL + - name: APPROVAL_MIDDLEWARE_URL + value: "http://192.168.87.14:8085" + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + readinessProbe: + httpGet: + path: /api/images + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + +--- +apiVersion: v1 +kind: Service +metadata: + name: version-tracker-web + namespace: k8s-version-tracker + labels: + app: version-tracker-web + annotations: + metallb.universe.tf/loadBalancerIPs: 192.168.87.13 +spec: + type: LoadBalancer + selector: + app: version-tracker-web + ports: + - name: http + port: 80 + targetPort: 8080 diff --git a/requirements.txt b/requirements.txt index 6b74992..76c05be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ kubernetes==29.0.0 requests==2.32.3 packaging==24.2 +fastapi==0.115.6 +uvicorn[standard]==0.32.1 +jinja2==3.1.4 diff --git a/templates/base.html b/templates/base.html new file mode 100644 index 0000000..3469a2c --- /dev/null +++ b/templates/base.html @@ -0,0 +1,49 @@ + + + + + + {% block title %}K8s Image Dashboard{% endblock %} + + + + + + + + +
+ {% block content %}{% endblock %} +
+ + + diff --git a/templates/images.html b/templates/images.html new file mode 100644 index 0000000..79b6724 --- /dev/null +++ b/templates/images.html @@ -0,0 +1,213 @@ +{% extends "base.html" %} +{% block title %}Image Inventory — K8s Dashboard{% endblock %} + +{% block content %} +
+ + +
+
+

Image Inventory

+

+ All container images deployed to the cluster, grouped by namespace and application. +

+
+
+ {% if is_checking %} + + + + + + Checking versions… + + {% elif last_updated %} + Updated {{ last_updated }} + {% endif %} + +
+
+ + +
+
+
{{ stats.total }}
+
Total Images
+
+
+
{{ stats.outdated }}
+
Outdated
+
+
+
{{ stats.current }}
+
Up to Date
+
+
+
{{ stats.floating }}
+
Floating Tags
+
+
+
{{ stats.local }}
+
Local / Skipped
+
+
+
{{ stats.pending }}
+
Checking…
+
+
+ + + {% if not namespaces %} +
No cluster data available. Click Refresh to load.
+ {% endif %} + + {% for ns, apps in namespaces.items() %} +
+ + + + + +
+ {% for app_data in apps %} +
+ + + + + +
+ + + + + + + + + + + + {% for c in app_data.containers %} + + + + + + + + {% endfor %} + +
ContainerImageCurrent TagLatest TagStatus
{{ c.name }} + + {% if c.registry != 'docker.io' and c.registry not in ('registry.storedbox.net', '192.168.87.31:5000', '192.168.87.31') %} + {{ c.registry }}/ + {% endif %} + {% if c.is_local %} + {{ c.registry }}/ + {% endif %} + {{ c.image_short.rsplit(':', 1)[0] if ':' in c.image_short else c.image_short }} + + {{ c.tag }} + {{ c.latest_tag or '—' }} + + + {% if c.status == 'outdated' %}⬆ Outdated + {% elif c.status == 'current' %}✓ Current + {% elif c.status == 'floating' %}~ Floating + {% elif c.status == 'local' %}⌂ Local + {% elif c.status == 'skipped' %}– Skipped + {% elif c.status == 'pending' %}… Checking + {% else %}? {{ c.status }} + {% endif %} + +
+
+
+ {% endfor %} +
+
+ {% endfor %} + +
+ + +{% endblock %} diff --git a/templates/upgrade.html b/templates/upgrade.html new file mode 100644 index 0000000..b2a1e2b --- /dev/null +++ b/templates/upgrade.html @@ -0,0 +1,249 @@ +{% extends "base.html" %} +{% block title %}Upgrade — K8s Dashboard{% endblock %} + +{% block content %} +
+ + +
+
+

Image Upgrades

+

+ Review outdated images and trigger the GitOps upgrade workflow. +

+
+ {% if is_checking %} + + + + + + Checking versions… + + {% elif not last_updated %} + + ⚠ No version data — go to Images to start a check + + {% else %} + Data from {{ last_updated }} + {% endif %} +
+ + +
+ + +
+ +
+

+ Outdated Images + {% if outdated_apps %} + + {{ outdated_apps | map(attribute='containers') | sum(start=[]) | list | length }} images + + {% endif %} +

+
+ + {% if not last_updated %} +
+
📋
+
No version data yet
+
+ Visit the Images page to trigger a version check. +
+
+ {% elif not outdated_apps %} +
+
✅
+
All images are up to date
+
Last checked: {{ last_updated }}
+
+ {% else %} + {% for app_data in outdated_apps %} +
+ +
+ + {{ app_data.kind }} + + {{ app_data.app }} + {{ app_data.namespace }} +
+ + + + + + + + + + + + {% for c in app_data.containers %} + + + + + + + {% endfor %} + +
ContainerImageCurrentLatest
{{ c.name }} + {{ c.image_short.rsplit(':', 1)[0] if ':' in c.image_short else c.image_short }} + {{ c.tag }}{{ c.latest_tag or '—' }}
+
+ {% endfor %} + {% endif %} +
+ + +
+ + +
+

Trigger Upgrade

+

+ Creates a Job from the upgrade-requester CronJob. + The workflow checks outdated images, requests Mattermost approval, then commits + updated image tags to the ArgoCD GitOps repo. +

+ + +
+
+ Approval timeout + 60 min +
+
+ Applies changes via + GitOps / ArgoCD +
+
+ Scheduled run + Every Monday 09:00 UTC +
+
+ + + + +
+
+
+ + +
+
+

Recent Jobs

+ +
+ +
+ No upgrade jobs found +
+ +
+ +
+
+
+
+ +
+ + +{% endblock %} diff --git a/web.py b/web.py new file mode 100644 index 0000000..39e6548 --- /dev/null +++ b/web.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +""" +web.py: FastAPI web dashboard for Kubernetes image version tracking. + +Routes: + GET / — image inventory grouped by namespace → app → container + GET /upgrade — upgrade trigger page with outdated images and job history + POST /api/refresh — kick off a background version-check refresh + POST /api/upgrade/trigger — create a Job from the upgrade-requester CronJob + GET /api/upgrade/jobs — recent upgrade job history (JSON) + GET /api/images — full inventory JSON +""" + +import logging +import os +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timezone + +from fastapi import BackgroundTasks, FastAPI, Request +from fastapi.responses import HTMLResponse, JSONResponse +from fastapi.templating import Jinja2Templates +from kubernetes import client, config +from packaging.version import InvalidVersion, Version + +from check_versions import get_latest_version, parse_image_ref + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger(__name__) + +app = FastAPI(title="K8s Image Dashboard") +templates = Jinja2Templates(directory="templates") +_pool = ThreadPoolExecutor(max_workers=20) + +_cache: dict = {"inventory": None, "last_updated": None, "is_checking": False} + + +# ── Kubernetes helpers ──────────────────────────────────────────────────────── + +def _load_k8s() -> None: + try: + config.load_incluster_config() + except config.ConfigException: + config.load_kube_config() + + +def _build_owner_maps() -> tuple[dict, dict]: + apps_v1 = client.AppsV1Api() + batch_v1 = client.BatchV1Api() + + rs_map: dict[str, str] = {} + for rs in apps_v1.list_replica_set_for_all_namespaces().items: + ns = rs.metadata.namespace + for ref in rs.metadata.owner_references or []: + if ref.kind == "Deployment": + rs_map[f"{ns}/{rs.metadata.name}"] = ref.name + + job_map: dict[str, str] = {} + for job in batch_v1.list_job_for_all_namespaces().items: + ns = job.metadata.namespace + for ref in job.metadata.owner_references or []: + if ref.kind == "CronJob": + job_map[f"{ns}/{job.metadata.name}"] = ref.name + + return rs_map, job_map + + +def _resolve_owner(pod, rs_map: dict, job_map: dict) -> tuple[str, str]: + ns = pod.metadata.namespace + for ref in pod.metadata.owner_references or []: + if ref.kind == "ReplicaSet": + deploy = rs_map.get(f"{ns}/{ref.name}") + return (deploy, "Deployment") if deploy else (ref.name, "ReplicaSet") + if ref.kind == "StatefulSet": + return ref.name, "StatefulSet" + if ref.kind == "DaemonSet": + return ref.name, "DaemonSet" + if ref.kind == "Job": + cj = job_map.get(f"{ns}/{ref.name}") + return (cj, "CronJob") if cj else (ref.name, "Job") + return pod.metadata.name, "Pod" + + +def _build_inventory() -> list[dict]: + """Query K8s and build an app-grouped inventory. Version statuses start as 'pending'.""" + _load_k8s() + v1 = client.CoreV1Api() + pods = v1.list_pod_for_all_namespaces(watch=False) + rs_map, job_map = _build_owner_maps() + + apps: dict[tuple, dict] = {} + for pod in pods.items: + ns = pod.metadata.namespace + app_name, kind = _resolve_owner(pod, rs_map, job_map) + key = (ns, app_name) + if key not in apps: + apps[key] = {"namespace": ns, "app": app_name, "kind": kind, "pod_count": 0, "containers": {}} + apps[key]["pod_count"] += 1 + for c in pod.spec.containers or []: + if not c.image or c.name in apps[key]["containers"]: + continue + img = parse_image_ref(c.image) + apps[key]["containers"][c.name] = { + "name": c.name, + "image": c.image, + "image_short": img.display_name, + "tag": img.tag, + "registry": img.registry, + "is_local": img.is_local, + "is_floating": img.is_floating, + "skip_check": img.skip_check, + "latest_tag": None, + "status": "pending", + } + + return sorted( + [{**a, "containers": list(a["containers"].values())} for a in apps.values()], + key=lambda x: (x["namespace"], x["app"]), + ) + + +# ── Version checking ────────────────────────────────────────────────────────── + +def _check_container(container: dict) -> None: + if container["is_local"]: + container["status"] = "local" + return + if container["skip_check"]: + container["status"] = "skipped" + return + if container["is_floating"]: + container["status"] = "floating" + return + img = parse_image_ref(container["image"]) + latest = get_latest_version(img) + container["latest_tag"] = latest + if latest is None: + container["status"] = "unknown" + return + try: + outdated = Version(latest.lstrip("v")) > Version(container["tag"].lstrip("v")) + except InvalidVersion: + outdated = container["tag"] != latest + container["status"] = "outdated" if outdated else "current" + + +def _run_checks(inventory: list[dict]) -> None: + if _cache["is_checking"]: + return + _cache["is_checking"] = True + try: + containers = [c for a in inventory for c in a["containers"]] + futures = {_pool.submit(_check_container, c): c for c in containers} + for f in as_completed(futures): + try: + f.result() + except Exception as e: + logger.warning("Version check error: %s", e) + _cache["last_updated"] = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + logger.info( + "Version check complete: %d outdated, %d current, %d floating", + sum(1 for a in inventory for c in a["containers"] if c["status"] == "outdated"), + sum(1 for a in inventory for c in a["containers"] if c["status"] == "current"), + sum(1 for a in inventory for c in a["containers"] if c["status"] == "floating"), + ) + finally: + _cache["is_checking"] = False + + +def _get_inventory(background_tasks: BackgroundTasks) -> list[dict]: + """Return cached inventory, building it and kicking off version checks if empty.""" + if _cache["inventory"] is None: + inventory = _build_inventory() + _cache["inventory"] = inventory + background_tasks.add_task(_run_checks, inventory) + return _cache["inventory"] + + +# ── Upgrade job helpers ─────────────────────────────────────────────────────── + +def _list_upgrade_jobs() -> list[dict]: + try: + _load_k8s() + batch_v1 = client.BatchV1Api() + jobs = batch_v1.list_namespaced_job( + "k8s-version-tracker", label_selector="app=upgrade-requester" + ) + result = [] + for j in sorted( + jobs.items, + key=lambda x: x.metadata.creation_timestamp or datetime.min, + reverse=True, + )[:5]: + s = j.status + status = "running" if (s.active or 0) else ("succeeded" if (s.succeeded or 0) else "failed") + result.append({ + "name": j.metadata.name, + "created": j.metadata.creation_timestamp.isoformat() if j.metadata.creation_timestamp else "", + "status": status, + "active": s.active or 0, + "succeeded": s.succeeded or 0, + "failed": s.failed or 0, + }) + return result + except Exception as e: + logger.error("Failed to list upgrade jobs: %s", e) + return [] + + +# ── Stats helper ───────────────────────────────────────────────────────────── + +def _stats(inventory: list[dict]) -> dict: + all_c = [c for a in inventory for c in a["containers"]] + return { + "total": len(all_c), + "outdated": sum(1 for c in all_c if c["status"] == "outdated"), + "current": sum(1 for c in all_c if c["status"] == "current"), + "floating": sum(1 for c in all_c if c["status"] == "floating"), + "local": sum(1 for c in all_c if c["status"] in ("local", "skipped")), + "pending": sum(1 for c in all_c if c["status"] in ("pending", "unknown")), + } + + +# ── Routes ──────────────────────────────────────────────────────────────────── + +@app.get("/", response_class=HTMLResponse) +async def images_page(request: Request, background_tasks: BackgroundTasks): + inventory = _get_inventory(background_tasks) + + ns_map: dict[str, list] = {} + for app_data in inventory: + ns_map.setdefault(app_data["namespace"], []).append(app_data) + + return templates.TemplateResponse("images.html", { + "request": request, + "namespaces": dict(sorted(ns_map.items())), + "stats": _stats(inventory), + "last_updated": _cache["last_updated"], + "is_checking": _cache["is_checking"], + }) + + +@app.get("/upgrade", response_class=HTMLResponse) +async def upgrade_page(request: Request): + inventory = _cache["inventory"] or [] + outdated_apps = [ + {**a, "containers": [c for c in a["containers"] if c["status"] == "outdated"]} + for a in inventory + if any(c["status"] == "outdated" for c in a["containers"]) + ] + return templates.TemplateResponse("upgrade.html", { + "request": request, + "outdated_apps": outdated_apps, + "jobs": _list_upgrade_jobs(), + "last_updated": _cache["last_updated"], + "is_checking": _cache["is_checking"], + }) + + +@app.post("/api/refresh") +async def api_refresh(background_tasks: BackgroundTasks): + if _cache["is_checking"]: + return {"status": "already_running"} + inventory = _build_inventory() + _cache["inventory"] = inventory + background_tasks.add_task(_run_checks, inventory) + return {"status": "started"} + + +@app.get("/api/images") +async def api_images(): + return {"inventory": _cache["inventory"] or [], "last_updated": _cache["last_updated"]} + + +@app.post("/api/upgrade/trigger") +async def api_trigger_upgrade(): + try: + _load_k8s() + batch_v1 = client.BatchV1Api() + cj = batch_v1.read_namespaced_cron_job("upgrade-requester", "k8s-version-tracker") + stamp = datetime.now(timezone.utc).strftime("%m%d%H%M") + job_name = f"upgrade-manual-{stamp}" + job = client.V1Job( + api_version="batch/v1", + kind="Job", + metadata=client.V1ObjectMeta( + name=job_name, + namespace="k8s-version-tracker", + labels={"app": "upgrade-requester", "triggered-by": "dashboard"}, + ), + spec=cj.spec.job_template.spec, + ) + batch_v1.create_namespaced_job("k8s-version-tracker", job) + logger.info("Triggered upgrade job: %s", job_name) + return {"status": "created", "job_name": job_name} + except Exception as e: + logger.error("Trigger failed: %s", e) + return JSONResponse(status_code=500, content={"error": str(e)}) + + +@app.get("/api/upgrade/jobs") +async def api_upgrade_jobs(): + return {"jobs": _list_upgrade_jobs()} + + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=8080, log_level="info")