307 lines
14 KiB
Python
307 lines
14 KiB
Python
# /opt/docker/dev/service_finder/backend/app/workers/monitor_dashboard2.0.py
|
|
# docker exec sf_api python -m app.workers.monitor_dashboard
|
|
import asyncio
|
|
import os
|
|
import httpx
|
|
import pynvml
|
|
import psutil
|
|
import subprocess
|
|
from datetime import datetime, timedelta
|
|
from sqlalchemy import text
|
|
from sqlalchemy.ext.asyncio import create_async_engine
|
|
from rich.console import Console
|
|
from rich.table import Table
|
|
from rich.panel import Panel
|
|
from rich.live import Live
|
|
from rich.layout import Layout
|
|
from rich.text import Text
|
|
from app.core.config import settings
|
|
|
|
console = Console()
|
|
|
|
# Magyar fordítási szótár a státuszokhoz
|
|
STATUS_TRANSLATIONS = {
|
|
'published': 'Véglegesítve (Publikált)',
|
|
'awaiting_ai_synthesis': 'AI Szintézisre Vár',
|
|
'manual_review_needed': 'Kézi Javítás Szükséges',
|
|
'unverified': 'Ellenőrizetlen (Nyers)',
|
|
'research_in_progress': 'Kutatás Folyamatban',
|
|
'ai_synthesis_in_progress': 'AI Szintézis Alatt',
|
|
'gold_enriched': 'Aranyosított (Végleges)',
|
|
'pending': 'Függőben',
|
|
'processing': 'Feldolgozás alatt'
|
|
}
|
|
|
|
try:
|
|
pynvml.nvmlInit()
|
|
gpu_available = True
|
|
except Exception:
|
|
gpu_available = False
|
|
|
|
def get_gpu_via_nvidia_smi():
|
|
"""GPU adatok lekérése nvidia-smi parancs segítségével"""
|
|
try:
|
|
output = subprocess.check_output(
|
|
['nvidia-smi', '--query-gpu=utilization.gpu,memory.used,memory.total,temperature.gpu',
|
|
'--format=csv,noheader,nounits'],
|
|
text=True
|
|
).strip()
|
|
if output:
|
|
# Több GPU esetén csak az elsőt vesszük
|
|
lines = output.split('\n')
|
|
first_line = lines[0]
|
|
values = [v.strip() for v in first_line.split(',')]
|
|
if len(values) >= 4:
|
|
gpu_util = int(values[0]) # %
|
|
mem_used = int(values[1]) # MiB
|
|
mem_total = int(values[2]) # MiB
|
|
temp = int(values[3]) # °C
|
|
return {
|
|
"load": gpu_util,
|
|
"vram_used": mem_used,
|
|
"vram_total": mem_total,
|
|
"temp": temp,
|
|
"source": "nvidia-smi"
|
|
}
|
|
except (subprocess.CalledProcessError, FileNotFoundError, ValueError, IndexError):
|
|
pass
|
|
return None
|
|
|
|
def get_gpu_content():
|
|
"""GPU adatok generálása a panelhez a megadott bolondbiztos megoldással"""
|
|
try:
|
|
gpu_raw = subprocess.check_output(
|
|
['nvidia-smi', '--query-gpu=name,utilization.gpu,memory.used,memory.total,temperature.gpu', '--format=csv,noheader,nounits'],
|
|
encoding='utf-8'
|
|
).strip().split(', ')
|
|
gpu_content = f"NVIDIA {gpu_raw[0]}\nTerhelés: {gpu_raw[1]}%\nVRAM: {gpu_raw[2]} MB / {gpu_raw[3]} MB\nHőmérséklet: {gpu_raw[4]} °C"
|
|
except Exception as e:
|
|
gpu_content = f"GPU adatok olvasása sikertelen: {str(e)}"
|
|
return gpu_content
|
|
|
|
async def get_hardware_stats():
|
|
stats = {
|
|
"cpu_usage": psutil.cpu_percent(interval=None),
|
|
"ram_total": psutil.virtual_memory().total // 1024**2,
|
|
"ram_used": psutil.virtual_memory().used // 1024**2,
|
|
"ram_perc": psutil.virtual_memory().percent,
|
|
"gpu": None,
|
|
"gpu_content": get_gpu_content()
|
|
}
|
|
|
|
# Először próbáljuk a pynvml-t
|
|
gpu_data = None
|
|
if gpu_available:
|
|
try:
|
|
handle = pynvml.nvmlDeviceGetHandleByIndex(0)
|
|
gpu_data = {
|
|
"name": pynvml.nvmlDeviceGetName(handle),
|
|
"temp": pynvml.nvmlDeviceGetTemperature(handle, pynvml.NVML_TEMPERATURE_GPU),
|
|
"load": pynvml.nvmlDeviceGetUtilizationRates(handle).gpu,
|
|
"vram_total": pynvml.nvmlDeviceGetMemoryInfo(handle).total // 1024**2,
|
|
"vram_used": pynvml.nvmlDeviceGetMemoryInfo(handle).used // 1024**2,
|
|
"power": pynvml.nvmlDeviceGetPowerUsage(handle) / 1000,
|
|
"source": "pynvml"
|
|
}
|
|
except:
|
|
gpu_data = None
|
|
|
|
# Ha nincs pynvml adat, próbáljuk az nvidia-smi-t
|
|
if not gpu_data:
|
|
gpu_data = get_gpu_via_nvidia_smi()
|
|
if gpu_data:
|
|
gpu_data["name"] = "NVIDIA GPU (via nvidia-smi)"
|
|
|
|
stats["gpu"] = gpu_data
|
|
return stats
|
|
|
|
async def get_ollama_models():
|
|
try:
|
|
async with httpx.AsyncClient(timeout=2.0) as client:
|
|
resp = await client.get("http://ollama:11434/api/ps")
|
|
if resp.status_code == 200:
|
|
return [m['name'] for m in resp.json().get("models", [])]
|
|
except: return ["Ollama API Offline"]
|
|
return []
|
|
|
|
async def get_stats(engine):
|
|
async with engine.connect() as conn:
|
|
# 1. Sebesség adatok (Golyóálló COALESCE használatával)
|
|
hr_rate = (await conn.execute(text("SELECT COALESCE(count(*), 0) FROM vehicle.vehicle_model_definitions WHERE status = 'gold_enriched' AND updated_at > NOW() - INTERVAL '1 hour'"))).scalar()
|
|
day_rate = (await conn.execute(text("SELECT COALESCE(count(*), 0) FROM vehicle.vehicle_model_definitions WHERE status = 'gold_enriched' AND updated_at > NOW() - INTERVAL '24 hours'"))).scalar()
|
|
|
|
# 2. Pipeline (R1, R2, R3, R4) - Külön lekérdezések a biztonságért
|
|
r1 = (await conn.execute(text("SELECT count(*) FROM vehicle.catalog_discovery WHERE status = 'pending'"))).scalar()
|
|
r2 = (await conn.execute(text("SELECT count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'unverified'"))).scalar()
|
|
r3 = (await conn.execute(text("SELECT count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'awaiting_ai_synthesis'"))).scalar()
|
|
r4 = (await conn.execute(text("SELECT count(*) FROM vehicle.vehicle_model_definitions WHERE status = 'gold_enriched'"))).scalar()
|
|
r_counts = (r1, r2, r3, r4)
|
|
|
|
# 3. TOP 7 Márka a végleges (Robot 1+) táblában
|
|
top_makes = (await conn.execute(text("SELECT make, count(*) as qty FROM vehicle.vehicle_model_definitions GROUP BY make ORDER BY qty DESC LIMIT 7"))).fetchall()
|
|
|
|
# 4. AKTIVITÁS (Utolsó beszúrások)
|
|
res_r4 = (await conn.execute(text("SELECT make, marketing_name FROM vehicle.vehicle_model_definitions WHERE status = 'gold_enriched' ORDER BY updated_at DESC LIMIT 5"))).fetchall()
|
|
res_r3 = (await conn.execute(text("SELECT make, marketing_name FROM vehicle.vehicle_model_definitions WHERE status = 'ai_synthesis_in_progress' ORDER BY updated_at DESC LIMIT 5"))).fetchall()
|
|
|
|
# JAVÍTÁS: A Discovery táblában "model" az oszlop neve, nem "marketing_name"!
|
|
res_r12 = (await conn.execute(text("SELECT make, model FROM vehicle.catalog_discovery WHERE status = 'processing' ORDER BY id DESC LIMIT 5"))).fetchall()
|
|
|
|
# 5. Új adatbázis statisztikák
|
|
published_count = (await conn.execute(text("SELECT COUNT(*) FROM vehicle.vehicle_model_definitions WHERE status = 'published'"))).scalar()
|
|
manual_review_needed_count = (await conn.execute(text("SELECT COUNT(*) FROM vehicle.vehicle_model_definitions WHERE status = 'unverified'"))).scalar()
|
|
|
|
status_distribution = (await conn.execute(text("SELECT status, COUNT(*) as count FROM vehicle.vehicle_model_definitions GROUP BY status ORDER BY count DESC"))).fetchall()
|
|
|
|
make_distribution = (await conn.execute(text("SELECT make, COUNT(*) as count FROM vehicle.vehicle_model_definitions WHERE status = 'published' GROUP BY make ORDER BY count DESC LIMIT 15"))).fetchall()
|
|
|
|
# 6. Kézi javításra várók listája (Top 15)
|
|
manual_review_list = (await conn.execute(text(
|
|
"SELECT make, marketing_name, COUNT(*) as count FROM vehicle.vehicle_model_definitions WHERE status = 'manual_review_needed' GROUP BY make, marketing_name ORDER BY count DESC LIMIT 15"
|
|
))).fetchall()
|
|
|
|
hw = await get_hardware_stats()
|
|
ai = await get_ollama_models()
|
|
|
|
return (hr_rate, day_rate), r_counts, top_makes, (res_r4, res_r3, res_r12), hw, ai, (published_count, manual_review_needed_count, status_distribution, make_distribution, manual_review_list)
|
|
|
|
def make_layout() -> Layout:
|
|
layout = Layout()
|
|
layout.split_column(
|
|
Layout(name="header", size=3),
|
|
Layout(name="main", ratio=1),
|
|
Layout(name="hardware", size=6),
|
|
Layout(name="footer", size=3)
|
|
)
|
|
layout["main"].split_row(
|
|
Layout(name="left", ratio=1),
|
|
Layout(name="middle", ratio=1),
|
|
Layout(name="right", ratio=2)
|
|
)
|
|
layout["left"].split_column(Layout(name="robot_stats"), Layout(name="inventory"))
|
|
layout["middle"].split_column(Layout(name="db_left"), Layout(name="db_right"))
|
|
layout["right"].split_column(
|
|
Layout(name="live_ops", ratio=1),
|
|
Layout(name="manual_review", ratio=1)
|
|
)
|
|
return layout
|
|
|
|
def translate_status(status):
|
|
"""Státusz fordítása angolról magyarra"""
|
|
return STATUS_TRANSLATIONS.get(status, status)
|
|
|
|
def update_dashboard(layout, data, error_msg=""):
|
|
rates, r_counts, top_makes, live_data, hw, ai_models, db_stats = data
|
|
r4_list, r3_list, r12_list = live_data
|
|
published_count, manual_review_needed_count, status_distribution, make_distribution, manual_review_list = db_stats
|
|
|
|
local_time = datetime.now() + timedelta(hours=1)
|
|
|
|
layout["header"].update(Panel(
|
|
f"🛰️ SENTINEL IRÁNYÍTÓKÖZPONT | [bold yellow]{local_time.strftime('%Y-%m-%d %H:%M:%S')}[/] | R4 (Arany): [green]{rates[0]}[/] /óra — [cyan]{rates[1]}[/] /nap | Összes feldolgozott: [bold green]{published_count:,}[/]",
|
|
style="bold white on blue"
|
|
))
|
|
|
|
robot_table = Table(title="🤖 Robot Pipeline Állapot", expand=True, border_style="cyan")
|
|
robot_table.add_column("Robot", style="bold")
|
|
robot_table.add_column("Várakozik", justify="right")
|
|
robot_table.add_row("R1-Hunter (Nyers gyűjtés)", f"{r_counts[0]:,} db")
|
|
robot_table.add_row("R2-Researcher (Webes kutatás)", f"{r_counts[1]:,} db")
|
|
robot_table.add_row("R3-Alchemist (AI Szintézis)", f"{r_counts[2]:,} db")
|
|
robot_table.add_row("R4-Validator (Várakozó Arany)", f"[green]{r_counts[3]:,}[/] db")
|
|
layout["robot_stats"].update(robot_table)
|
|
|
|
brand_table = Table(title="🚜 Bányászott Márkák (Top 7)", expand=True, border_style="magenta")
|
|
brand_table.add_column("Márka", style="yellow")
|
|
brand_table.add_column("Darabszám", justify="right")
|
|
for m, q in top_makes: brand_table.add_row(str(m), str(q))
|
|
layout["inventory"].update(brand_table)
|
|
|
|
ops_table = Table(title="⚡ Aktuális Folyamatok", expand=True, border_style="green")
|
|
ops_table.add_column("Robot", width=15)
|
|
ops_table.add_column("Márka / Típus")
|
|
|
|
for r in r4_list: ops_table.add_row("[gold1]R4-ARANY[/]", f"{r[0]} {r[1] or ''}")
|
|
if r4_list: ops_table.add_section()
|
|
|
|
for r in r3_list: ops_table.add_row("[medium_purple1]R3-AI[/]", f"{r[0]} {r[1] or ''}")
|
|
if r3_list: ops_table.add_section()
|
|
|
|
for r in r12_list: ops_table.add_row("[sky_blue1]R1-HUNTER[/]", f"{r[0]} {r[1] or ''}")
|
|
layout["live_ops"].update(ops_table)
|
|
|
|
hw_layout = Layout()
|
|
hw_layout.split_row(
|
|
Layout(name="sys"),
|
|
Layout(name="gpu_ai_column")
|
|
)
|
|
hw_layout["gpu_ai_column"].split_column(
|
|
Layout(name="gpu"),
|
|
Layout(name="ai")
|
|
)
|
|
|
|
sys_info = f"[bold]CPU:[/]\t[bright_blue]{hw['cpu_usage']}%[/]\n[bold]RAM:[/]\t[bright_magenta]{hw['ram_perc']}%[/] ({hw['ram_used']}/{hw['ram_total']}MB)"
|
|
hw_layout["sys"].update(Panel(sys_info, title="💻 Rendszer", border_style="bright_blue"))
|
|
|
|
# GPU adatok a get_gpu_content() által generált szöveggel
|
|
gpu_info = hw.get("gpu_content", "GPU adatok nem elérhetők")
|
|
hw_layout["gpu"].update(Panel(gpu_info, title="🔌 GPU Adatok", border_style="orange3"))
|
|
|
|
ai_info = "\n".join([f"🧠 {m}" for m in ai_models]) if ai_models else "Nincs betöltve modell."
|
|
hw_layout["ai"].update(Panel(ai_info, title="🤖 Ollama VRAM", border_style="plum1"))
|
|
|
|
layout["hardware"].update(hw_layout)
|
|
|
|
summary_text = f"[bold green]Véglegesített: {published_count:,}[/] | [bold yellow]Kézi ellenőrzés: {manual_review_needed_count:,}[/]"
|
|
|
|
status_table = Table(title="📈 Státusz eloszlás", expand=True, border_style="magenta")
|
|
status_table.add_column("Státusz", style="bold")
|
|
status_table.add_column("Mennyiség", justify="right")
|
|
for status, count in status_distribution:
|
|
translated = translate_status(status)
|
|
status_table.add_row(translated, f"{count:,}")
|
|
layout["db_left"].update(Panel(status_table, title="📊 Státuszok", border_style="magenta"))
|
|
|
|
make_table = Table(title="🚗 Márkák (véglegesített)", expand=True, border_style="green")
|
|
make_table.add_column("Márka", style="yellow")
|
|
make_table.add_column("Véglegesített DB", justify="right")
|
|
for make, count in make_distribution:
|
|
make_table.add_row(str(make), f"{count:,}")
|
|
layout["db_right"].update(Panel(make_table, title="🏆 Top Márkák", border_style="green"))
|
|
|
|
manual_table = Table(title="🛠️ Kézi Javításra Várók (Top 15)", expand=True, border_style="yellow")
|
|
manual_table.add_column("Márka", style="bold")
|
|
manual_table.add_column("Modell", style="cyan")
|
|
manual_table.add_column("Darabszám", justify="right")
|
|
for make, model, count in manual_review_list:
|
|
manual_table.add_row(str(make), str(model) if model else "N/A", f"{count:,}")
|
|
layout["manual_review"].update(Panel(manual_table, title="🛠️ Kézi Javításra Várók", border_style="yellow"))
|
|
|
|
footer_text = f"Sentinel v2.6 | Kernel: Stabil | R1 Pörög: {r_counts[0]} várakozik"
|
|
if error_msg: footer_text = f"[red bold]HIBA: {error_msg}[/]"
|
|
layout["footer"].update(Panel(footer_text, style="italic grey50"))
|
|
|
|
async def main():
|
|
engine = create_async_engine(settings.DATABASE_URL)
|
|
layout = make_layout()
|
|
with Live(layout, refresh_per_second=2, screen=True):
|
|
while True:
|
|
try:
|
|
data = await get_stats(engine)
|
|
update_dashboard(layout, data)
|
|
except Exception as e:
|
|
# JAVÍTVA: A db_stats tuple most már 5 elemű, ahogy az update_dashboard várja!
|
|
fallback_data = (
|
|
(0, 0), # rates
|
|
(0, 0, 0, 0), # r_counts
|
|
[], # top_makes
|
|
([], [], []), # live_data
|
|
{"cpu_usage": 0, "ram_perc": 0, "ram_used": 0, "ram_total": 0, "gpu": None, "gpu_content": "Várakozás..."}, # hw
|
|
[], # ai
|
|
(0, 0, [], [], []) # db_stats -> 5 ELEM!
|
|
)
|
|
update_dashboard(layout, fallback_data, str(e))
|
|
await asyncio.sleep(0.5)
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main()) |