diff --git a/__pycache__/get_rpc_config_auto.cpython-310.pyc b/__pycache__/get_rpc_config_auto.cpython-310.pyc index 586788b..4b0576e 100644 Binary files a/__pycache__/get_rpc_config_auto.cpython-310.pyc and b/__pycache__/get_rpc_config_auto.cpython-310.pyc differ diff --git a/get_rpc_config_auto.py b/get_rpc_config_auto.py index d325440..3743b57 100644 --- a/get_rpc_config_auto.py +++ b/get_rpc_config_auto.py @@ -3,7 +3,6 @@ import time from playwright.sync_api import sync_playwright def analyze_gwt_response(response_text): - """Finds potential coordinates to validate response data.""" candidates = [] try: if response_text.startswith("//OK"): @@ -17,71 +16,82 @@ def analyze_gwt_response(response_text): if abs(val1) > 100000 and abs(val2) > 100000: candidates.append((val1, val2)) if len(candidates) > 5: break - except: - pass + except: pass return candidates -def get_fresh_config(map_url): +def fetch_live_data(map_url): """ - Launches headless browser to scrape headers, body, AND cookies. + Uses a real browser to fetch data AND capture the raw request details. """ - print(f"--- Auto-Repair: Launching Browser for {map_url} ---") - - captured_request = None - captured_cookies = [] + print(f"--- Browser Fetch: {map_url} ---") + data_result = None + captured_headers = None + captured_cookies = None + captured_body = None # <--- New: Capture raw body with sync_playwright() as p: - browser = p.chromium.launch(headless=True) - # Create a persistent context to ensure cookies are tracked - context = browser.new_context() + browser = p.chromium.launch(headless=True, args=['--disable-blink-features=AutomationControlled']) + context = browser.new_context( + user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' + ) page = context.new_page() def handle_request(request): - nonlocal captured_request + nonlocal captured_headers, captured_body if ".rpc" in request.url and request.method == "POST": + # Capture the request details blindly before we even know if it works + if "getCombinedOutageDetails" in request.post_data or "getOutages" in request.post_data: + captured_headers = request.headers + captured_body = request.post_data + + def handle_response(response): + nonlocal data_result + if ".rpc" in response.url and response.request.method == "POST": try: - if "getCombinedOutageDetails" in request.post_data or "getOutages" in request.post_data: - captured_request = { - 'url': request.url, - 'headers': request.headers, - 'body': request.post_data - } - except: - pass + if "getCombinedOutageDetails" in response.request.post_data or "getOutages" in response.request.post_data: + text = response.text() + if text.startswith("//OK"): + data_result = json.loads(text[4:]) + print(" [+] Captured Data via Browser") + except: pass page.on("request", handle_request) + page.on("response", handle_response) try: - page.goto(map_url, wait_until="networkidle", timeout=45000) - time.sleep(5) - - # Capture cookies from the browser context + page.goto(map_url, wait_until="networkidle", timeout=60000) + for _ in range(10): + if data_result: break + time.sleep(1) captured_cookies = context.cookies() - except Exception as e: - print(f"Auto-Repair Browser Error: {e}") + print(f"Browser Fetch Error: {e}") finally: browser.close() - if captured_request: - req_headers = captured_request['headers'] - # Clean headers (keep specific GWT ones, discard dynamic browser ones that requests handles) - clean_headers = { - 'content-type': req_headers.get('content-type', 'text/x-gwt-rpc; charset=UTF-8'), - 'x-gwt-module-base': req_headers.get('x-gwt-module-base'), - 'x-gwt-permutation': req_headers.get('x-gwt-permutation'), - 'Referer': map_url - } - + return data_result, captured_headers, captured_cookies, captured_body + +def get_fresh_config(map_url): + data, headers, cookies, body = fetch_live_data(map_url) + + if headers and body: + # Minimal cleaning: Only remove headers that 'requests' MUST generate itself + # This keeps all custom NISC/GWT headers safe. + forbidden = {'content-length', 'host', 'connection', 'cookie', 'accept-encoding'} + clean_headers = {k: v for k, v in headers.items() if k.lower() not in forbidden} + return { 'headers': clean_headers, - 'body': captured_request['body'], - 'url': captured_request['url'], - 'cookies': captured_cookies # <--- Return cookies + 'body': body, # Save exact body + 'url': headers.get('url', map_url.replace('.html', '') + '/GWT.rpc'), # Best guess URL if missing + 'cookies': cookies, + 'user_agent': headers.get('user-agent') } - return None if __name__ == "__main__": url = input("Enter Map URL: ") - print(get_fresh_config(url)) \ No newline at end of file + res = get_fresh_config(url) + if res: + print("Success! Captured Body length:", len(res['body'])) + print("Captured Headers:", res['headers'].keys()) \ No newline at end of file diff --git a/newpower2.py b/newpower2.py index fec1e06..bdba0ce 100644 --- a/newpower2.py +++ b/newpower2.py @@ -47,16 +47,13 @@ def update_provider_config(provider_name, new_settings): updated = False for p in providers: if p.get('name') == provider_name: - if 'headers' in new_settings: - p['headers'] = new_settings['headers'] - if 'body' in new_settings: - p['body'] = new_settings['body'] - if 'url' in new_settings: - p['url'] = new_settings['url'] + if 'headers' in new_settings: p['headers'] = new_settings['headers'] + if 'body' in new_settings: p['body'] = new_settings['body'] + if 'url' in new_settings: p['url'] = new_settings['url'] + if 'cookies' in new_settings: p['cookies'] = new_settings['cookies'] - # <--- NEW: Save Cookies - if 'cookies' in new_settings: - p['cookies'] = new_settings['cookies'] + # <--- NEW: Save User-Agent + if 'user_agent' in new_settings: p['user_agent'] = new_settings['user_agent'] p['last_auto_update'] = datetime.now(timezone.utc).isoformat() updated = True @@ -67,7 +64,6 @@ def update_provider_config(provider_name, new_settings): return True return False - # --- DATABASE --- class PowerDB: def __init__(self, config): @@ -244,9 +240,11 @@ class GwtRpcProvider(BaseProvider): self.state_filter = config.get('state_filter') self.map_url = config.get('map_url') - # 1. Base Headers + # 1. Set User-Agent (Dynamic > Default) + # We try to use the one from config if available (captured from actual browser) + ua = config.get('user_agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36') self.session.headers.update({ - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'User-Agent': ua, 'Accept': '*/*', 'Sec-Fetch-Site': 'same-origin' }) @@ -254,16 +252,12 @@ class GwtRpcProvider(BaseProvider): parsed_url = urlparse(config.get('url')) self.session.headers.update({'Origin': f"{parsed_url.scheme}://{parsed_url.netloc}"}) - # 2. Load Cookies (if available, but don't rely solely on them) + # Load Cookies if config.get('cookies'): for cookie in config['cookies']: - # Handle expiry mapping if needed, or ignore errors try: self.session.cookies.set( - cookie['name'], - cookie['value'], - domain=cookie['domain'], - path=cookie['path'] + cookie['name'], cookie['value'], domain=cookie['domain'], path=cookie['path'] ) except: pass @@ -273,7 +267,6 @@ class GwtRpcProvider(BaseProvider): 'KY': {'lat_min': 36.4, 'lat_max': 39.2, 'lon_min': -89.6, 'lon_max': -81.9}, 'IA': {'lat_min': 40.3, 'lat_max': 43.6, 'lon_min': -96.7, 'lon_max': -90.1} } - if config.get('epsg'): try: self.transformer = Transformer.from_crs(f"EPSG:{config['epsg']}", "EPSG:4326", always_xy=True) @@ -282,101 +275,127 @@ class GwtRpcProvider(BaseProvider): def attempt_auto_repair(self): if not self.map_url: return False - # --- Cooldown Check --- - last_update = self.config.get('last_auto_update') - if last_update: - try: - last_dt = datetime.fromisoformat(last_update) - if last_dt.tzinfo is None: last_dt = last_dt.replace(tzinfo=timezone.utc) - if datetime.now(timezone.utc) - last_dt < timedelta(hours=AUTO_UPDATE_COOLDOWN_HOURS): - logger.info(f"Skipping auto-repair for {self.name} (Cooldown active).") - return False - except ValueError: pass + # ... (Cooldown check - keep as is) ... logger.info(f"Attempting Auto-Repair for {self.name}...") try: + # We expect 4 return values now new_settings = get_rpc_config_auto.get_fresh_config(self.map_url) + if new_settings: logger.info(f"Repair successful! Updating {self.name}.") - # Update In-Memory Config (CRITICAL: prevents loop) - current_time = datetime.now(timezone.utc).isoformat() - self.config['headers'] = new_settings['headers'] - self.config['body'] = new_settings['body'] - self.config['url'] = new_settings['url'] - self.config['cookies'] = new_settings.get('cookies', []) - self.config['last_auto_update'] = current_time + # Update In-Memory + self.config.update(new_settings) + self.config['last_auto_update'] = datetime.now(timezone.utc).isoformat() - # Force updates to session - # We clear cookies to ensure we don't mix old/new session logic + # Update Session Cookies self.session.cookies.clear() if new_settings.get('cookies'): - for cookie in new_settings['cookies']: - self.session.cookies.set(cookie['name'], cookie['value'], domain=cookie['domain'], path=cookie['path']) + for c in new_settings['cookies']: + self.session.cookies.set(c['name'], c['value'], domain=c['domain'], path=c['path']) + + # Update Session UA + if new_settings.get('user_agent'): + self.session.headers.update({'User-Agent': new_settings['user_agent']}) + # Persist to disk update_provider_config(self.name, new_settings) return True except Exception as e: logger.error(f"Auto-repair failed: {e}") return False - + def fetch(self, is_retry=False): url = self.config.get('url') headers = self.config.get('headers', {}) body = self.config.get('body') - if not url or not body: return [] + if not url: return [] + # --- STRATEGY A: Standard Requests (Fast) --- try: - # 3. Dynamic Origin Update parsed_url = urlparse(url) origin = f"{parsed_url.scheme}://{parsed_url.netloc}" - self.session.headers.update({'Origin': origin}) - # 4. ALWAYS PRIME SESSION (Fixes the regression) - # Even if we have cookies, they might be stale or missing JSESSIONID. - # Hitting the page refreshes the jar. - prime_url = headers.get('Referer') or headers.get('x-gwt-module-base') or origin - if prime_url: - try: - self.session.get(prime_url, verify=False, timeout=10) + # Priority: Configured Referer > Module Base > Origin + correct_referer = headers.get('Referer') or headers.get('x-gwt-module-base') or origin + + ua = headers.get('User-Agent', self.session.headers['User-Agent']) + if "Headless" in ua: + ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' + + self.session.headers.update({ + 'Origin': origin, + 'Referer': correct_referer, + 'User-Agent': ua + }) + + if self.map_url and not self.config.get('cookies'): + try: self.session.get(correct_referer, verify=False, timeout=10) except: pass req_headers = headers.copy() if 'Content-Type' not in req_headers: req_headers['Content-Type'] = 'text/x-gwt-rpc; charset=UTF-8' + req_headers['Referer'] = correct_referer + req_headers['User-Agent'] = ua + # Debug log (Optional - disable if too noisy) + # logger.info(f"Sending Headers: {json.dumps(req_headers, indent=2)}") + resp = self.session.post(url, headers=req_headers, data=body, verify=False) - # 5. Error Handling & Retry - failed = False - if "//EX" in resp.text: failed = True - if resp.status_code == 500: failed = True - - if failed: - logger.error(f"GWT Failure for {self.name} (Status: {resp.status_code}).") + # --- STRATEGY B: Browser Fallback & Self-Heal --- + if resp.status_code == 500 or "//EX" in resp.text: + logger.warning(f"Standard fetch failed for {self.name} (Status: {resp.status_code}). Switching to Browser Fetch.") - # Check recursion limit - if is_retry: - logger.error(f"Retry failed for {self.name}. Aborting.") - return [] + if self.map_url: + # 1. Fetch data AND credentials via Browser + data, valid_headers, valid_cookies, valid_body = get_rpc_config_auto.fetch_live_data(self.map_url) + + if data: + logger.info(f"Browser success! Self-healing {self.name} configuration...") + + # --- HEADER CLEANING FIX --- + # Instead of selecting specific headers, we exclude known transport headers. + # This preserves custom headers like 'coop.nisc.outagewebmap.configname' + excluded = { + 'content-length', 'host', 'connection', 'cookie', 'accept-encoding', + 'sec-ch-ua', 'sec-ch-ua-mobile', 'sec-ch-ua-platform', 'origin' + } + + clean_headers = {} + for k, v in valid_headers.items(): + if k.lower() not in excluded: + clean_headers[k] = v + + # Ensure we force the correct Referer for next time + clean_headers['Referer'] = self.map_url - if self.attempt_auto_repair(): - logger.info("Retrying fetch with new settings...") - return self.fetch(is_retry=True) - else: - return [] + # 3. Save to JSON so next run is FAST + new_settings = { + 'headers': clean_headers, + 'cookies': valid_cookies, + 'body': valid_body, + 'user_agent': valid_headers.get('user-agent') + } + update_provider_config(self.name, new_settings) + + return self._extract_outages(data) + + logger.error(f"Browser Fetch failed for {self.name}.") + return [] if not resp.ok: return [] text = resp.text if text.startswith('//OK'): text = text[4:] return self._extract_outages(json.loads(text)) + except Exception as e: logger.error(f"Fetch error {self.name}: {e}") - return [] - - # ... Keep _extract_outages and _is_valid as is ... + return [] def _extract_outages(self, data_list): results = [] if not self.transformer: return [] @@ -429,7 +448,6 @@ class GwtRpcProvider(BaseProvider): return b['lat_min'] <= lat <= b['lat_max'] and b['lon_min'] <= lon <= b['lon_max'] - # --- REGISTRY --- PROVIDER_REGISTRY = { 'kubra': KubraProvider, diff --git a/providers.json b/providers.json index 1e86aae..b51a148 100644 --- a/providers.json +++ b/providers.json @@ -104,20 +104,23 @@ "epsg": 3735, "state_filter": "OH", "headers": { - "content-type": "text/x-gwt-rpc; charset=UTF-8", - "x-gwt-module-base": "https://weci.ebill.coop/woViewer/MapWiseWeb/", "x-gwt-permutation": "92F322F8E48548F604D2E1BE43DB1F13", + "x-gwt-module-base": "https://weci.ebill.coop/woViewer/MapWiseWeb/", + "referer": "https://weci.ebill.coop/woViewer/mapviewer.html?config=Outage+Web+Map", + "coop.nisc.outagewebmap.configname": "Outage Web Map", + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "content-type": "text/x-gwt-rpc; charset=UTF-8", "Referer": "https://weci.ebill.coop/woViewer/mapviewer.html?config=Outage+Web+Map" }, "body": "7|0|4|https://weci.ebill.coop/woViewer/MapWiseWeb/|612278413EC26C34D54A3907AA0CDFD8|coop.nisc.oms.webmap.services.RpcCombinedOutageDetailsService|getCombinedOutageDetails|1|2|3|4|0|", - "last_auto_update": "2025-12-07T03:24:46.435173+00:00", + "last_auto_update": "2025-12-07T03:56:27.722877+00:00", "cookies": [ { "name": "__utma", - "value": "105963909.535514741.1765077881.1765077881.1765077881.1", + "value": "105963909.1267880890.1765079787.1765079787.1765079787.1", "domain": ".weci.ebill.coop", "path": "/", - "expires": 1799637880.601006, + "expires": 1799639786.874286, "httpOnly": false, "secure": false, "sameSite": "Lax" @@ -134,10 +137,10 @@ }, { "name": "__utmz", - "value": "105963909.1765077881.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)", + "value": "105963909.1765079787.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)", "domain": ".weci.ebill.coop", "path": "/", - "expires": 1780845880, + "expires": 1780847786, "httpOnly": false, "secure": false, "sameSite": "Lax" @@ -147,7 +150,7 @@ "value": "1", "domain": ".weci.ebill.coop", "path": "/", - "expires": 1765078480, + "expires": 1765080386, "httpOnly": false, "secure": false, "sameSite": "Lax" @@ -157,37 +160,37 @@ "value": "1", "domain": ".weci.ebill.coop", "path": "/", - "expires": 1765078480, + "expires": 1765080386, "httpOnly": false, "secure": false, "sameSite": "Lax" }, { "name": "__utmb", - "value": "105963909.2.10.1765077881", + "value": "105963909.2.10.1765079787", "domain": ".weci.ebill.coop", "path": "/", - "expires": 1765079680, + "expires": 1765081586, "httpOnly": false, "secure": false, "sameSite": "Lax" }, { "name": "__utma", - "value": "105963909.535514741.1765077881.1765077881.1765077881.1", + "value": "105963909.1267880890.1765079787.1765079787.1765079787.1", "domain": "weci.ebill.coop", "path": "/", - "expires": 1799637880.601622, + "expires": 1799639786.87497, "httpOnly": false, "secure": false, "sameSite": "Lax" }, { "name": "__utmb", - "value": "105963909.3.9.1765077881", + "value": "105963909.3.9.1765079787", "domain": "weci.ebill.coop", "path": "/", - "expires": 1765079680, + "expires": 1765081586, "httpOnly": false, "secure": false, "sameSite": "Lax" @@ -204,14 +207,15 @@ }, { "name": "__utmz", - "value": "105963909.1765077881.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)", + "value": "105963909.1765079787.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)", "domain": "weci.ebill.coop", "path": "/", - "expires": 1780845880, + "expires": 1780847786, "httpOnly": false, "secure": false, "sameSite": "Lax" } - ] + ], + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" } ] \ No newline at end of file