import asyncio import httpx import logging import json import os import datetime from sqlalchemy import text from app.db.session import SessionLocal from app.models.asset import AssetCatalog logging.basicConfig(level=logging.INFO) logger = logging.getLogger("Robot-v1.0.13-Global-Hunter") class CatalogMaster: """ Master Hunter Robot v1.0.13 - Global Hunter Edition - Holland (RDW), Brit (DVLA) és Amerikai (NHTSA) adatbázis integráció. - Ratio-Filter: Kiszűri a 0.19-es kW/kg arányszámokat. - Multi-field Power Discovery: Minden lehetséges mezőből kinyeri a kW-ot. - Dinamikus évjárat kezelés a duplikációk ellen. """ # API Végpontok RDW_MAIN = "https://opendata.rdw.nl/resource/m9d7-ebf2.json" RDW_FUEL = "https://opendata.rdw.nl/resource/8ys7-d773.json" RDW_AXLE = "https://opendata.rdw.nl/resource/3huj-srit.json" RDW_BODY = "https://opendata.rdw.nl/resource/vezc-m2t6.json" UK_DVLA = "https://driver-vehicle-licensing.api.gov.uk/vehicle-enquiry/v1/vehicles" US_NHTSA = "https://vpic.nhtsa.dot.gov/api/vehicles/DecodeVinValuesBatch/" RDW_TOKEN = os.getenv("RDW_APP_TOKEN") UK_API_KEY = os.getenv("UK_DVLA_API_KEY") HEADERS_RDW = {"X-App-Token": RDW_TOKEN} if RDW_TOKEN else {} HEADERS_UK = {"x-api-key": UK_API_KEY, "Content-Type": "application/json"} if UK_API_KEY else {} CATEGORY_MAP = { "Personenauto": "car", "Motorfiets": "motorcycle", "Bedrijfsauto": "truck", "Vrachtwagen": "truck", "Opleggertrekker": "truck", "Bus": "bus", "Aanhangwagen": "trailer", "Oplegger": "trailer", "Landbouw- of bosbouwtrekker": "agricultural", "camper": "camper" } @classmethod def clean_kw(cls, val): """Speciális kW tisztító: ignorálja az 1.0 alatti arányszámokat.""" try: if val is None: return None f_val = float(str(val).replace(',', '.')) if 0 < f_val < 1.0: return None # Ez csak arányszám (kW/kg) v = int(f_val) return v if v > 0 else None except (ValueError, TypeError): return None @classmethod def clean_int(cls, val): """Általános egész szám tisztító.""" try: if val is None: return None return int(float(str(val).replace(',', '.'))) except (ValueError, TypeError): return None @classmethod async def fetch_api(cls, url, params=None, headers=None, method="GET", json_data=None): """Univerzális API hívó sebességkorlátozással.""" async with httpx.AsyncClient(headers=headers, follow_redirects=True) as client: try: await asyncio.sleep(1.2) # Biztonsági késleltetés if method == "POST": resp = await client.post(url, json=json_data, timeout=30) else: resp = await client.get(url, params=params, timeout=30) return resp.json() if resp.status_code in [200, 201] else [] except Exception as e: logger.error(f"❌ API Hiba ({url}): {e}") return [] @classmethod async def get_deep_tech(cls, plate, main_kw=None, vin=None): """Nemzetközi dúsítás: Holland -> Brit -> Amerikai sorrendben.""" res = {"kw": cls.clean_kw(main_kw), "fuel": "Unknown", "axles": None, "body": "Standard", "euro": None} # 1. HOLLAND (RDW) DÚSÍTÁS fuel_data = await cls.fetch_api(cls.RDW_FUEL, {"kenteken": plate}, headers=cls.HEADERS_RDW) if fuel_data: f0 = fuel_data[0] if not res["kw"]: res["kw"] = cls.clean_kw(f0.get("nettomaximumvermogen") or f0.get("netto_maximum_vermogen")) res["fuel"] = f0.get("brandstof_omschrijving", "Unknown") res["euro"] = f0.get("uitlaatemissieniveau") # 2. BRIT (DVLA) ELLENŐRZÉS (Ha van UK kulcs és még hiányzik adat) if cls.UK_API_KEY and (not res["kw"] or not res["euro"]): uk_data = await cls.fetch_api(cls.UK_DVLA, method="POST", json_data={"registrationNumber": plate}, headers=cls.HEADERS_UK) if uk_data: res["kw"] = res["kw"] or cls.clean_kw(uk_data.get("engineCapacity")) # Brit adatok finomítása res["euro"] = res["euro"] or uk_data.get("euroStatus") # 3. AMERIKAI (NHTSA) KUTATÁS (Ha van alvázszám) if vin and len(vin) == 17: us_data = await cls.fetch_api(cls.US_NHTSA, params={"format": "json", "data": vin}) if us_data and "Results" in us_data: # Az amerikai adatbázisból kinyerjük a lóerőt (HP), ha a kW még mindig nincs meg hp = us_data["Results"][0].get("EngineHP") if hp and not res["kw"]: res["kw"] = int(float(hp) * 0.7457) # HP -> kW konverzió # RDW Extra adatok (Tengely, Karosszéria) axle = await cls.fetch_api(cls.RDW_AXLE, {"kenteken": plate}, headers=cls.HEADERS_RDW) if axle: res["axles"] = cls.clean_int(axle[0].get("aantal_assen")) body = await cls.fetch_api(cls.RDW_BODY, {"kenteken": plate}, headers=cls.HEADERS_RDW) if body: res["body"] = body[0].get("carrosserie_omschrijving", "Standard") return res @classmethod async def process_make(cls, db, task_id, make_name): logger.info(f"🚀 >>> {make_name} GlobalHunter v1.0.13 INDUL...") offset, limit, total_saved = 0, 1000, 0 unique_variants = {} while True: params = {"merk": make_name.upper(), "$limit": limit, "$offset": offset} main_data = await cls.fetch_api(cls.RDW_MAIN, params, headers=cls.HEADERS_RDW) if not main_data: break for item in main_data: plate = item.get("kenteken") if not plate: continue model = str(item.get("handelsbenaming", "Unknown")).upper() ccm = cls.clean_int(item.get("cilinderinhoud")) weight = cls.clean_int(item.get("massa_ledig_voertuig") or item.get("massa_rijklaar")) kw_candidate = item.get("netto_maximum_vermogen") or item.get("vermogen_massarijklaar") raw_date = item.get("datum_eerste_toelating") prod_year = int(str(raw_date)[:4]) if raw_date else 2024 v_class = cls.CATEGORY_MAP.get(item.get("voertuigsoort"), "other") if "kampeerwagen" in str(item.get("inrichting", "")).lower(): v_class = "camper" # Variáns kulcs: Modell + CCM + Súly + kW + Év = Egyedi technikai ujjlenyomat variant_key = f"{model}-{ccm}-{weight}-{v_class}-{kw_candidate}-{prod_year}" if variant_key not in unique_variants: unique_variants[variant_key] = { "model": model, "ccm": ccm, "weight": weight, "v_class": v_class, "plate": plate, "main_kw": kw_candidate, "prod_year": prod_year, "vin": item.get("vin") # Ha az RDW-ben benne van a VIN } if len(main_data) < limit or offset > 90000: break offset += limit logger.info(f"📊 {len(unique_variants)} egyedi variáns kutatása indul...") for key, v in unique_variants.items(): deep = await cls.get_deep_tech(v["plate"], main_kw=v["main_kw"], vin=v["vin"]) try: db_item = AssetCatalog( make=make_name.upper(), model=v["model"], vehicle_class=v["v_class"], fuel_type=deep["fuel"], power_kw=deep["kw"], engine_capacity=v["ccm"], max_weight_kg=v["weight"], axle_count=deep["axles"], body_type=deep["body"], year_from=v["prod_year"], euro_class=deep["euro"], factory_data={ "source": "GlobalHunter-v1.0.13", "sample_plate": v["plate"], "enriched_at": str(datetime.datetime.now()) } ) db.add(db_item) await db.commit() total_saved += 1 if total_saved % 50 == 0: logger.info(f"✅ {total_saved} variáns elmentve.") except Exception: await db.rollback() continue await db.execute(text("UPDATE data.catalog_discovery SET status = 'processed' WHERE id = :id"), {"id": task_id}) await db.commit() logger.info(f"🏁 {make_name} KÉSZ. {total_saved} rekord rögzítve.") @classmethod async def run(cls): logger.info("🤖 Robot 1.0.13 (Global Hunter) ONLINE") while True: async with SessionLocal() as db: res = await db.execute(text("SELECT id, make FROM data.catalog_discovery WHERE status = 'pending' LIMIT 1")) task = res.fetchone() if task: await cls.process_make(db, task[0], task[1]) else: logger.info("😴 Várólista üres. Alvás 60 mp...") await asyncio.sleep(60) await asyncio.sleep(1) if __name__ == "__main__": asyncio.run(CatalogMaster.run())