"""Separate manually-started monitoring project; never mutates Contextra Compose.""" import json import os from pathlib import Path import secrets import subprocess ROOT = Path("/usr/local/lib/contextra-maintenance/20260928-r01") CONF = Path("/etc/contextra-maintenance/observability") DATA = Path("/backup/contextra/observability") def execute(args, timeout=60): return subprocess.run(args, check=True, capture_output=True, text=True, timeout=timeout).stdout def prepare(images): if os.geteuid() != 0 or not Path("/backup").is_mount(): raise RuntimeError("Root and mounted /backup required") if CONF.exists() and list(CONF.iterdir()): raise RuntimeError("Existing monitoring config requires review") CONF.mkdir(parents=True, mode=0o755, exist_ok=True) DATA.mkdir(parents=True, exist_ok=True, mode=0o755) for name, uid in (("prometheus", 65534), ("grafana", 472), ("textfile", 1001)): directory = DATA / name directory.mkdir(mode=0o750, exist_ok=True) os.chown(directory, uid, uid) (DATA / "textfile").chmod(0o755) ids = {x["tag"].split(":")[0]: x.get("engine_image_id", x["image_id"]) for x in images["images"]} for value in ids.values(): execute(["docker", "image", "inspect", value]) password = CONF / "grafana-password" password.write_text(secrets.token_urlsafe(24) + "\n") password.chmod(0o600) os.chown(password, 472, 472) (CONF / "provisioning/datasources").mkdir(parents=True) (CONF / "provisioning/dashboards").mkdir() (CONF / "dashboards").mkdir() # JSON is valid YAML; no additional YAML package is required. (CONF / "provisioning/datasources/prometheus.yaml").write_text(json.dumps({"apiVersion": 1, "datasources": [{"name": "Prometheus", "uid": "contextra-local", "type": "prometheus", "url": "http://127.0.0.1:39090", "access": "proxy", "isDefault": True, "editable": False}]})) (CONF / "provisioning/dashboards/local.yaml").write_text(json.dumps({"apiVersion": 1, "providers": [{"name": "Contextra", "type": "file", "disableDeletion": True, "options": {"path": "/etc/grafana/dashboards"}}]})) panels = [] queries = [ ("CPU 전체 논리코어 평균 사용률", '100 * (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[1m])))', "percent"), ("GPU 사용률", "contextra_gpu_utilization_percent", "percent"), ("RAM 사용률", "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", "percent"), ("GPU 메모리 사용량", "contextra_gpu_memory_used_mib", "mebibytes"), ("Filesystem 여유 공간", 'node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"}', "bytes"), ("Triton 요청률", "sum by (model) (rate(nv_inference_request_success[1m]))", "reqps"), ("Triton Queue 평균", "sum by (model) (rate(nv_inference_queue_duration_us[1m])) / clamp_min(sum by (model) (rate(nv_inference_request_success[1m])), 0.001) / 1000", "ms"), ("Triton Compute 평균", "sum by (model) (rate(nv_inference_compute_infer_duration_us[1m])) / clamp_min(sum by (model) (rate(nv_inference_request_success[1m])), 0.001) / 1000", "ms"), ("채널 실행 상태", "contextra_stream_state", "short"), ("관측 대상 응답", "up", "short"), ] for i, (title, expr, unit) in enumerate(queries): panels.append(dict(id=i+1, type="timeseries", title=title, gridPos=dict(x=(i % 2)*12, y=(i//2)*8, w=12, h=8), datasource=dict(type="prometheus", uid="contextra-local"), targets=[dict(refId="A", expr=expr)], fieldConfig=dict(defaults=dict(unit=unit), overrides=[]))) (CONF / "dashboards/contextra.json").write_text(json.dumps(dict(uid="contextra-offline", title="Contextra 서버 관측", schemaVersion=39, version=1, timezone="browser", refresh="15s", time={"from":"now-30m","to":"now"}, panels=panels), ensure_ascii=False)) limits = dict(restart="no", network_mode="host", security_opt=["no-new-privileges:true"]) compose = {"name": "contextra-observe", "services": { "node": dict(limits, image=ids["prom/node-exporter"], cpus=.25, mem_limit="128m", pid="host", read_only=True, cap_drop=["ALL"], volumes=["/:/host:ro,rslave", str(DATA / "textfile")+":/textfile:ro"], command=["--web.listen-address=127.0.0.1:19100", "--path.rootfs=/host", "--collector.textfile.directory=/textfile"]), "prometheus": dict(limits, image=ids["prom/prometheus"], cpus=1, mem_limit="1536m", volumes=[str(CONF / "prometheus.json")+":/etc/prometheus/prometheus.json:ro",str(DATA / "prometheus")+":/prometheus"], command=["--config.file=/etc/prometheus/prometheus.json", "--web.listen-address=127.0.0.1:39090", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d", "--storage.tsdb.retention.size=10GB"]), "grafana": dict(limits, image=ids["grafana/grafana"], cpus=1, mem_limit="768m", environment={"GF_SERVER_HTTP_ADDR":"127.0.0.1", "GF_SERVER_HTTP_PORT":"33000", "GF_SECURITY_ADMIN_PASSWORD__FILE":"/run/grafana-password", "GF_USERS_ALLOW_SIGN_UP":"false", "GF_ANALYTICS_REPORTING_ENABLED":"false", "GF_ANALYTICS_CHECK_FOR_UPDATES":"false", "GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES":"false"}, volumes=[str(DATA / "grafana")+":/var/lib/grafana", str(password)+":/run/grafana-password:ro", str(CONF / "provisioning")+":/etc/grafana/provisioning:ro", str(CONF / "dashboards")+":/etc/grafana/dashboards:ro"]), }} compose["services"]["grafana"]["environment"].update({"GF_PLUGINS_PREINSTALL_DISABLED":"true", "GF_PLUGINS_PREINSTALL_AUTO_UPDATE":"false"}) (CONF / "compose.json").write_text(json.dumps(compose, indent=2)) refresh_targets() print("Observability installed; autostart OFF; no existing Runtime changed") def refresh_targets(): configs = [dict(job_name="host", static_configs=[dict(targets=["127.0.0.1:19100"])])] containers = execute(["docker", "ps", "--filter", "label=com.docker.compose.project=contextra-icerti", "--filter", "label=com.docker.compose.service=analytics", "--format", "{{.ID}}"]).split() if containers: info = json.loads(execute(["docker", "inspect", containers[0]]))[0] addresses = [n["IPAddress"] for n in info["NetworkSettings"]["Networks"].values() if n.get("IPAddress")] if addresses: ip = addresses[0] configs.extend([dict(job_name="contextra", static_configs=[dict(targets=[ip+":5000"])]), dict(job_name="triton", static_configs=[dict(targets=[ip+":8002"])])]) value = {"global": {"scrape_interval":"15s", "evaluation_interval":"15s", "scrape_timeout":"8s"}, "scrape_configs": configs} temporary = CONF / "prometheus.next.json" temporary.write_text(json.dumps(value, indent=2)) temporary.replace(CONF / "prometheus.json") def action(name): if os.geteuid() != 0: raise RuntimeError("Use sudo ct observe") if name == "start": if not Path("/backup").is_mount(): raise RuntimeError("/backup not mounted") refresh_targets() execute(["systemctl", "start", "contextra-gpu-metrics.service"]) print(execute(["docker", "compose", "-f", str(CONF / "compose.json"), "up", "-d", "--force-recreate", "--pull", "never", "--no-build"], 120)) elif name == "stop": print(execute(["docker", "compose", "-f", str(CONF / "compose.json"), "stop", "--timeout", "30"], 90)) execute(["systemctl", "stop", "contextra-gpu-metrics.service"]) elif name == "status": print(execute(["docker", "compose", "-f", str(CONF / "compose.json"), "ps"])) else: raise ValueError("start/stop/status only")