stoat/test
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed rpc, separated providers

Browse Source
This commit is contained in:
John Peck
2025-12-07 03:57:20 +00:00
parent 32638df3a9
commit f9f0fc5093
4 changed files with 163 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
__pycache__/get_rpc_config_auto.cpython-310.pyc
View File

Binary file not shown.

94
get_rpc_config_auto.py
View File

@@ -3,7 +3,6 @@ import time
from playwright.sync_api import sync_playwright
def analyze_gwt_response(response_text):
"""Finds potential coordinates to validate response data."""
candidates = []
try:
if response_text.startswith("//OK"):
@@ -17,71 +16,82 @@ def analyze_gwt_response(response_text):
if abs(val1) > 100000 and abs(val2) > 100000:
candidates.append((val1, val2))
if len(candidates) > 5: break
except:
pass
except: pass
return candidates
def get_fresh_config(map_url):
def fetch_live_data(map_url):
"""
Launches headless browser to scrape headers, body, AND cookies.
Uses a real browser to fetch data AND capture the raw request details.
"""
print(f"--- Auto-Repair: Launching Browser for {map_url} ---")
captured_request = None
captured_cookies = []
print(f"--- Browser Fetch: {map_url} ---")
data_result = None
captured_headers = None
captured_cookies = None
captured_body = None # <--- New: Capture raw body
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
# Create a persistent context to ensure cookies are tracked
context = browser.new_context()
browser = p.chromium.launch(headless=True, args=['--disable-blink-features=AutomationControlled'])
context = browser.new_context(
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
)
page = context.new_page()
def handle_request(request):
nonlocal captured_request
nonlocal captured_headers, captured_body
if ".rpc" in request.url and request.method == "POST":
# Capture the request details blindly before we even know if it works
if "getCombinedOutageDetails" in request.post_data or "getOutages" in request.post_data:
captured_headers = request.headers
captured_body = request.post_data
def handle_response(response):
nonlocal data_result
if ".rpc" in response.url and response.request.method == "POST":
try:
if "getCombinedOutageDetails" in request.post_data or "getOutages" in request.post_data:
captured_request = {
'url': request.url,
'headers': request.headers,
'body': request.post_data
}
except:
pass
if "getCombinedOutageDetails" in response.request.post_data or "getOutages" in response.request.post_data:
text = response.text()
if text.startswith("//OK"):
data_result = json.loads(text[4:])
print(" [+] Captured Data via Browser")
except: pass
page.on("request", handle_request)
page.on("response", handle_response)
try:
page.goto(map_url, wait_until="networkidle", timeout=45000)
time.sleep(5)
# Capture cookies from the browser context
page.goto(map_url, wait_until="networkidle", timeout=60000)
for _ in range(10):
if data_result: break
time.sleep(1)
captured_cookies = context.cookies()
except Exception as e:
print(f"Auto-Repair Browser Error: {e}")
print(f"Browser Fetch Error: {e}")
finally:
browser.close()
if captured_request:
req_headers = captured_request['headers']
# Clean headers (keep specific GWT ones, discard dynamic browser ones that requests handles)
clean_headers = {
'content-type': req_headers.get('content-type', 'text/x-gwt-rpc; charset=UTF-8'),
'x-gwt-module-base': req_headers.get('x-gwt-module-base'),
'x-gwt-permutation': req_headers.get('x-gwt-permutation'),
'Referer': map_url
}
return data_result, captured_headers, captured_cookies, captured_body
def get_fresh_config(map_url):
data, headers, cookies, body = fetch_live_data(map_url)
if headers and body:
# Minimal cleaning: Only remove headers that 'requests' MUST generate itself
# This keeps all custom NISC/GWT headers safe.
forbidden = {'content-length', 'host', 'connection', 'cookie', 'accept-encoding'}
clean_headers = {k: v for k, v in headers.items() if k.lower() not in forbidden}
return {
'headers': clean_headers,
'body': captured_request['body'],
'url': captured_request['url'],
'cookies': captured_cookies # <--- Return cookies
'body': body, # Save exact body
'url': headers.get('url', map_url.replace('.html', '') + '/GWT.rpc'), # Best guess URL if missing
'cookies': cookies,
'user_agent': headers.get('user-agent')
}
return None
if __name__ == "__main__":
url = input("Enter Map URL: ")
print(get_fresh_config(url))
res = get_fresh_config(url)
if res:
print("Success! Captured Body length:", len(res['body']))
print("Captured Headers:", res['headers'].keys())

160
newpower2.py
View File

@@ -47,16 +47,13 @@ def update_provider_config(provider_name, new_settings):
updated = False
for p in providers:
if p.get('name') == provider_name:
if 'headers' in new_settings:
p['headers'] = new_settings['headers']
if 'body' in new_settings:
p['body'] = new_settings['body']
if 'url' in new_settings:
p['url'] = new_settings['url']
if 'headers' in new_settings: p['headers'] = new_settings['headers']
if 'body' in new_settings: p['body'] = new_settings['body']
if 'url' in new_settings: p['url'] = new_settings['url']
if 'cookies' in new_settings: p['cookies'] = new_settings['cookies']
# <--- NEW: Save Cookies
if 'cookies' in new_settings:
p['cookies'] = new_settings['cookies']
# <--- NEW: Save User-Agent
if 'user_agent' in new_settings: p['user_agent'] = new_settings['user_agent']
p['last_auto_update'] = datetime.now(timezone.utc).isoformat()
updated = True
@@ -67,7 +64,6 @@ def update_provider_config(provider_name, new_settings):
return True
return False
# --- DATABASE ---
class PowerDB:
def __init__(self, config):
@@ -244,9 +240,11 @@ class GwtRpcProvider(BaseProvider):
self.state_filter = config.get('state_filter')
self.map_url = config.get('map_url')
# 1. Base Headers
# 1. Set User-Agent (Dynamic > Default)
# We try to use the one from config if available (captured from actual browser)
ua = config.get('user_agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')
self.session.headers.update({
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'User-Agent': ua,
'Accept': '*/*',
'Sec-Fetch-Site': 'same-origin'
})
@@ -254,16 +252,12 @@ class GwtRpcProvider(BaseProvider):
parsed_url = urlparse(config.get('url'))
self.session.headers.update({'Origin': f"{parsed_url.scheme}://{parsed_url.netloc}"})
# 2. Load Cookies (if available, but don't rely solely on them)
# Load Cookies
if config.get('cookies'):
for cookie in config['cookies']:
# Handle expiry mapping if needed, or ignore errors
try:
self.session.cookies.set(
cookie['name'],
cookie['value'],
domain=cookie['domain'],
path=cookie['path']
cookie['name'], cookie['value'], domain=cookie['domain'], path=cookie['path']
)
except: pass
@@ -273,7 +267,6 @@ class GwtRpcProvider(BaseProvider):
'KY': {'lat_min': 36.4, 'lat_max': 39.2, 'lon_min': -89.6, 'lon_max': -81.9},
'IA': {'lat_min': 40.3, 'lat_max': 43.6, 'lon_min': -96.7, 'lon_max': -90.1}
}
if config.get('epsg'):
try:
self.transformer = Transformer.from_crs(f"EPSG:{config['epsg']}", "EPSG:4326", always_xy=True)
@@ -282,101 +275,127 @@ class GwtRpcProvider(BaseProvider):
def attempt_auto_repair(self):
if not self.map_url: return False
# --- Cooldown Check ---
last_update = self.config.get('last_auto_update')
if last_update:
try:
last_dt = datetime.fromisoformat(last_update)
if last_dt.tzinfo is None: last_dt = last_dt.replace(tzinfo=timezone.utc)
if datetime.now(timezone.utc) - last_dt < timedelta(hours=AUTO_UPDATE_COOLDOWN_HOURS):
logger.info(f"Skipping auto-repair for {self.name} (Cooldown active).")
return False
except ValueError: pass
# ... (Cooldown check - keep as is) ...
logger.info(f"Attempting Auto-Repair for {self.name}...")
try:
# We expect 4 return values now
new_settings = get_rpc_config_auto.get_fresh_config(self.map_url)
if new_settings:
logger.info(f"Repair successful! Updating {self.name}.")
# Update In-Memory Config (CRITICAL: prevents loop)
current_time = datetime.now(timezone.utc).isoformat()
self.config['headers'] = new_settings['headers']
self.config['body'] = new_settings['body']
self.config['url'] = new_settings['url']
self.config['cookies'] = new_settings.get('cookies', [])
self.config['last_auto_update'] = current_time
# Update In-Memory
self.config.update(new_settings)
self.config['last_auto_update'] = datetime.now(timezone.utc).isoformat()
# Force updates to session
# We clear cookies to ensure we don't mix old/new session logic
# Update Session Cookies
self.session.cookies.clear()
if new_settings.get('cookies'):
for cookie in new_settings['cookies']:
self.session.cookies.set(cookie['name'], cookie['value'], domain=cookie['domain'], path=cookie['path'])
for c in new_settings['cookies']:
self.session.cookies.set(c['name'], c['value'], domain=c['domain'], path=c['path'])
# Update Session UA
if new_settings.get('user_agent'):
self.session.headers.update({'User-Agent': new_settings['user_agent']})
# Persist to disk
update_provider_config(self.name, new_settings)
return True
except Exception as e:
logger.error(f"Auto-repair failed: {e}")
return False
def fetch(self, is_retry=False):
url = self.config.get('url')
headers = self.config.get('headers', {})
body = self.config.get('body')
if not url or not body: return []
if not url: return []
# --- STRATEGY A: Standard Requests (Fast) ---
try:
# 3. Dynamic Origin Update
parsed_url = urlparse(url)
origin = f"{parsed_url.scheme}://{parsed_url.netloc}"
self.session.headers.update({'Origin': origin})
# 4. ALWAYS PRIME SESSION (Fixes the regression)
# Even if we have cookies, they might be stale or missing JSESSIONID.
# Hitting the page refreshes the jar.
prime_url = headers.get('Referer') or headers.get('x-gwt-module-base') or origin
if prime_url:
try:
self.session.get(prime_url, verify=False, timeout=10)
# Priority: Configured Referer > Module Base > Origin
correct_referer = headers.get('Referer') or headers.get('x-gwt-module-base') or origin
ua = headers.get('User-Agent', self.session.headers['User-Agent'])
if "Headless" in ua:
ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
self.session.headers.update({
'Origin': origin,
'Referer': correct_referer,
'User-Agent': ua
})
if self.map_url and not self.config.get('cookies'):
try: self.session.get(correct_referer, verify=False, timeout=10)
except: pass
req_headers = headers.copy()
if 'Content-Type' not in req_headers: req_headers['Content-Type'] = 'text/x-gwt-rpc; charset=UTF-8'
req_headers['Referer'] = correct_referer
req_headers['User-Agent'] = ua
# Debug log (Optional - disable if too noisy)
# logger.info(f"Sending Headers: {json.dumps(req_headers, indent=2)}")
resp = self.session.post(url, headers=req_headers, data=body, verify=False)
# 5. Error Handling & Retry
failed = False
if "//EX" in resp.text: failed = True
if resp.status_code == 500: failed = True
if failed:
logger.error(f"GWT Failure for {self.name} (Status: {resp.status_code}).")
# --- STRATEGY B: Browser Fallback & Self-Heal ---
if resp.status_code == 500 or "//EX" in resp.text:
logger.warning(f"Standard fetch failed for {self.name} (Status: {resp.status_code}). Switching to Browser Fetch.")
# Check recursion limit
if is_retry:
logger.error(f"Retry failed for {self.name}. Aborting.")
return []
if self.map_url:
# 1. Fetch data AND credentials via Browser
data, valid_headers, valid_cookies, valid_body = get_rpc_config_auto.fetch_live_data(self.map_url)
if data:
logger.info(f"Browser success! Self-healing {self.name} configuration...")
# --- HEADER CLEANING FIX ---
# Instead of selecting specific headers, we exclude known transport headers.
# This preserves custom headers like 'coop.nisc.outagewebmap.configname'
excluded = {
'content-length', 'host', 'connection', 'cookie', 'accept-encoding',
'sec-ch-ua', 'sec-ch-ua-mobile', 'sec-ch-ua-platform', 'origin'
}
clean_headers = {}
for k, v in valid_headers.items():
if k.lower() not in excluded:
clean_headers[k] = v
# Ensure we force the correct Referer for next time
clean_headers['Referer'] = self.map_url
if self.attempt_auto_repair():
logger.info("Retrying fetch with new settings...")
return self.fetch(is_retry=True)
else:
return []
# 3. Save to JSON so next run is FAST
new_settings = {
'headers': clean_headers,
'cookies': valid_cookies,
'body': valid_body,
'user_agent': valid_headers.get('user-agent')
}
update_provider_config(self.name, new_settings)
return self._extract_outages(data)
logger.error(f"Browser Fetch failed for {self.name}.")
return []
if not resp.ok: return []
text = resp.text
if text.startswith('//OK'): text = text[4:]
return self._extract_outages(json.loads(text))
except Exception as e:
logger.error(f"Fetch error {self.name}: {e}")
return []
# ... Keep _extract_outages and _is_valid as is ...
return []
def _extract_outages(self, data_list):
results = []
if not self.transformer: return []
@@ -429,7 +448,6 @@ class GwtRpcProvider(BaseProvider):
return b['lat_min'] <= lat <= b['lat_max'] and b['lon_min'] <= lon <= b['lon_max']
# --- REGISTRY ---
PROVIDER_REGISTRY = {
'kubra': KubraProvider,

40
providers.json
View File

@@ -104,20 +104,23 @@
"epsg": 3735,
"state_filter": "OH",
"headers": {
"content-type": "text/x-gwt-rpc; charset=UTF-8",
"x-gwt-module-base": "https://weci.ebill.coop/woViewer/MapWiseWeb/",
"x-gwt-permutation": "92F322F8E48548F604D2E1BE43DB1F13",
"x-gwt-module-base": "https://weci.ebill.coop/woViewer/MapWiseWeb/",
"referer": "https://weci.ebill.coop/woViewer/mapviewer.html?config=Outage+Web+Map",
"coop.nisc.outagewebmap.configname": "Outage Web Map",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"content-type": "text/x-gwt-rpc; charset=UTF-8",
"Referer": "https://weci.ebill.coop/woViewer/mapviewer.html?config=Outage+Web+Map"
},
"body": "7|0|4|https://weci.ebill.coop/woViewer/MapWiseWeb/|612278413EC26C34D54A3907AA0CDFD8|coop.nisc.oms.webmap.services.RpcCombinedOutageDetailsService|getCombinedOutageDetails|1|2|3|4|0|",
"last_auto_update": "2025-12-07T03:24:46.435173+00:00",
"last_auto_update": "2025-12-07T03:56:27.722877+00:00",
"cookies": [
{
"name": "__utma",
"value": "105963909.535514741.1765077881.1765077881.1765077881.1",
"value": "105963909.1267880890.1765079787.1765079787.1765079787.1",
"domain": ".weci.ebill.coop",
"path": "/",
"expires": 1799637880.601006,
"expires": 1799639786.874286,
"httpOnly": false,
"secure": false,
"sameSite": "Lax"
@@ -134,10 +137,10 @@
},
{
"name": "__utmz",
"value": "105963909.1765077881.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)",
"value": "105963909.1765079787.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)",
"domain": ".weci.ebill.coop",
"path": "/",
"expires": 1780845880,
"expires": 1780847786,
"httpOnly": false,
"secure": false,
"sameSite": "Lax"
@@ -147,7 +150,7 @@
"value": "1",
"domain": ".weci.ebill.coop",
"path": "/",
"expires": 1765078480,
"expires": 1765080386,
"httpOnly": false,
"secure": false,
"sameSite": "Lax"
@@ -157,37 +160,37 @@
"value": "1",
"domain": ".weci.ebill.coop",
"path": "/",
"expires": 1765078480,
"expires": 1765080386,
"httpOnly": false,
"secure": false,
"sameSite": "Lax"
},
{
"name": "__utmb",
"value": "105963909.2.10.1765077881",
"value": "105963909.2.10.1765079787",
"domain": ".weci.ebill.coop",
"path": "/",
"expires": 1765079680,
"expires": 1765081586,
"httpOnly": false,
"secure": false,
"sameSite": "Lax"
},
{
"name": "__utma",
"value": "105963909.535514741.1765077881.1765077881.1765077881.1",
"value": "105963909.1267880890.1765079787.1765079787.1765079787.1",
"domain": "weci.ebill.coop",
"path": "/",
"expires": 1799637880.601622,
"expires": 1799639786.87497,
"httpOnly": false,
"secure": false,
"sameSite": "Lax"
},
{
"name": "__utmb",
"value": "105963909.3.9.1765077881",
"value": "105963909.3.9.1765079787",
"domain": "weci.ebill.coop",
"path": "/",
"expires": 1765079680,
"expires": 1765081586,
"httpOnly": false,
"secure": false,
"sameSite": "Lax"
@@ -204,14 +207,15 @@
},
{
"name": "__utmz",
"value": "105963909.1765077881.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)",
"value": "105963909.1765079787.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)",
"domain": "weci.ebill.coop",
"path": "/",
"expires": 1780845880,
"expires": 1780847786,
"httpOnly": false,
"secure": false,
"sameSite": "Lax"
}
]
],
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
}
]