From facc2a486ebe2b4ae28c92d4a8a4fb4da1f6cc64 Mon Sep 17 00:00:00 2001 From: John Peck Date: Sun, 7 Dec 2025 12:17:53 +0000 Subject: [PATCH] checkpoint rpc county fixed --- newpower.py | 491 +++++++++++++++++++++++++++++++++++++++++++++++- newpower2.py | 5 +- outage_data.txt | 1 + powercounty.py | 6 +- providers.json | 88 ++++----- test.py | 163 ++++++++++++++++ 6 files changed, 695 insertions(+), 59 deletions(-) create mode 100644 outage_data.txt create mode 100644 test.py diff --git a/newpower.py b/newpower.py index 021a0c7..f017776 100644 --- a/newpower.py +++ b/newpower.py @@ -1,6 +1,489 @@ -#!/var/www/html/power/venv/bin/python +import requests +import json +import psycopg2 +import logging +import os +import re +from datetime import datetime, timezone, timedelta +from abc import ABC, abstractmethod +from urllib.parse import urlparse +from requests.packages.urllib3.exceptions import InsecureRequestWarning -# Add Harrison REA -# Add WashingtonElectric -# Add Buckeye +# Import the helper module for auto-repair +import get_rpc_config_auto +requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + +# --- LOGGING --- +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[logging.FileHandler('newpower_county.log'), logging.StreamHandler()] +) +logger = logging.getLogger(__name__) + +# --- CONFIG --- +DB_CONFIG = {'host': 'localhost', 'database': 'nws', 'user': 'nws', 'password': 'nws'} +CONFIG_FILE = 'providers.json' +AUTO_UPDATE_COOLDOWN_HOURS = 4 # Only try to repair once every 4 hours + +# --- CONFIG MANAGEMENT --- +def load_providers(): + if not os.path.exists(CONFIG_FILE): + logger.error(f"{CONFIG_FILE} not found!") + return [] + with open(CONFIG_FILE, 'r') as f: + # Filter for providers that have 'county_url' or 'county_meta_url' + all_providers = json.load(f) + return [p for p in all_providers if p.get('county_type')] + +def save_providers(providers): + with open(CONFIG_FILE, 'w') as f: + json.dump(providers, f, indent=4, default=str) + logger.info("Configuration saved to providers.json") + +def update_provider_config(provider_name, new_settings): + """Updates a specific provider in the JSON file safely""" + # This needs to read the raw file, not the filtered one + with open(CONFIG_FILE, 'r') as f: + providers = json.load(f) + + updated = False + for p in providers: + if p.get('name') == provider_name: + for key in ['headers', 'body', 'url', 'cookies', 'user_agent']: + if key in new_settings: + p[key] = new_settings[key] + p['last_auto_update'] = datetime.now(timezone.utc).isoformat() + updated = True + break + + if updated: + save_providers(providers) + return True + return False + +# --- DATABASE --- +class CountyPowerDB: + def __init__(self, config): + self.conn = psycopg2.connect(**config) + self.conn.autocommit = True + def close(self): + self.conn.close() + + def upsert_and_zero_outages(self, company_name, outage_data): + """ + Atomically updates outage information for a given company. + 1. UPSERTS counties with active outages, updating their counts. + 2. SETS outage count to 0 for any other county from that company that was not in the active list. + """ + current_fetch_time = datetime.now(timezone.utc) + + # Prepare data for counties with active outages + active_outage_values = [] + reported_counties = [] + for item in outage_data: + if all(k in item for k in ['county', 'state', 'company']): + val = ( + item['county'], item['state'], item['company'], + item.get('outages'), item.get('served'), current_fetch_time + ) + active_outage_values.append(val) + reported_counties.append(item['county']) + + with self.conn.cursor() as cursor: + # Step 1: UPSERT active outages + if active_outage_values: + upsert_sql = """ + INSERT INTO newcountyoutages (county, state, company, outages, served, fetch_time) + VALUES (%s, %s, %s, %s, %s, %s) + ON CONFLICT (county, state, company) DO UPDATE SET + outages = EXCLUDED.outages, + served = COALESCE(EXCLUDED.served, newcountyoutages.served), + fetch_time = EXCLUDED.fetch_time; + """ + cursor.executemany(upsert_sql, active_outage_values) + logger.info(f"Upserted {len(active_outage_values)} active outage records for {company_name}.") + + # Step 2: Set outages to 0 for any other county from this company + # This correctly creates point-in-time zero records by updating the fetch_time. + zero_out_sql = """ + UPDATE newcountyoutages + SET outages = 0, fetch_time = %s + WHERE company = %s AND county NOT IN %s; + """ + # Ensure reported_counties is not empty to avoid "IN (NULL)" + if not reported_counties: + reported_counties.append("NO_COUNTIES_REPORTED_DUMMY_VALUE") + + cursor.execute(zero_out_sql, (current_fetch_time, company_name, tuple(reported_counties))) + logger.info(f"Zeroed out {cursor.rowcount} resolved outage records for {company_name}.") + + def run_post_processing(self): + logger.info("Running post-processing for county data...") + with self.conn.cursor() as cursor: + cursor.execute(""" + UPDATE newcountyoutages + SET cwa = c.cwa + FROM public.county c + WHERE c.countyname = newcountyoutages.county + AND c.state = newcountyoutages.state + AND newcountyoutages.cwa IS NULL + """) + cursor.execute("DELETE FROM newcountyoutages WHERE fetch_time < NOW() - INTERVAL '30 days'") + logger.info("County post-processing complete.") + +# --- PROVIDERS --- + +class BaseCountyProvider(ABC): + def __init__(self, config, session): + self.config = config + self.session = session + self.name = config.get('name', 'Unknown') + + @abstractmethod + def fetch(self): + pass + +class SimpleCountyJsonProvider(BaseCountyProvider): + def fetch(self): + url = self.config.get('county_url') + state = self.config.get('state_filter') + try: + resp = self.session.get(url, verify=False) + if not resp.ok: return [] + data = resp.json() + results = [] + for boundary_group in data: + for item in boundary_group.get('boundaries', []): + results.append({ + 'outages': item.get('customersOutNow'), + 'served': item.get('customersServed'), + 'county': item.get('name'), + 'state': state, + 'company': self.name + }) + return results + except Exception as e: + logger.error(f"Error fetching {self.name}: {e}") + return [] + +class KubraCountyProvider(BaseCountyProvider): + def fetch(self): + meta_url = self.config.get('county_meta_url') + report_url_suffix = self.config.get('county_report_suffix') + + try: + # 1. Get hexes from meta_url + meta_resp = self.session.get(meta_url) + meta_data = meta_resp.json() + path = meta_data.get('data', {}).get('cluster_interval_generation_data') + if not path: return [] + + # 2. Construct final report URL + # The old script's logic reveals the path is composed of a base, + # the second hex from the metadata path, and the report suffix. + # Example path from meta: data/e2ae0326-9912-436a-9355-eb2687e798b1 + path_parts = path.split('/') # e.g., ['data', 'hex1', 'hex2', 'hex3'] + if len(path_parts) < 4: + logger.error(f"Invalid metadata path format for {self.name}: {path}") + return [] + + # This is the single, correct URL format used by the original script. + # It uses the fourth element (index 3) from the metadata path. + report_url = f"https://kubra.io/data/{path_parts[3]}{report_url_suffix}" + + # 3. Fetch and process report + report_resp = self.session.get(report_url) + if not report_resp.ok or not report_resp.text: + logger.info(f"No county report data available for {self.name} at this time.") + return [] + + report_data = report_resp.json() + return self._normalize(report_data) + except json.JSONDecodeError: + logger.warning(f"Could not decode JSON from county report for {self.name}. The report may be empty or invalid.") + return [] + except requests.exceptions.RequestException as e: + logger.error(f"Error fetching Kubra county data for {self.name}: {e}") + return [] + + def _normalize(self, data): + results = [] + primary_areas = data.get("file_data", {}).get("areas", []) + if not primary_areas: return [] + + first_item_key = primary_areas[0].get("key") + + if first_item_key == "state": + for state_area in primary_areas: + for county in state_area.get("areas", []): + if county.get("key") == "county": + results.append(self._extract_info(county)) + elif first_item_key == "county": + for county in primary_areas: + if county.get("key") == "county": + results.append(self._extract_info(county)) + return results + + def _extract_info(self, county_item): + return { + 'outages': county_item.get('cust_a', {}).get('val'), + 'served': county_item.get('cust_s'), + 'county': county_item.get('name', '').capitalize(), + 'state': county_item.get('state') or self.config.get('state_filter'), + 'company': self.name + } + +class NiscCountyProvider(BaseCountyProvider): + """ Handles county data from NISC-hosted cloud sources. """ + def fetch(self): + url = self.config.get('county_url') + state = self.config.get('state_filter') + try: + resp = self.session.get(url, verify=False) + if not resp.ok: return [] + data = resp.json() + results = [] + # The structure is typically a list containing one object with a 'boundaries' key + for boundary_group in data: + for item in boundary_group.get('boundaries', []): + results.append({ + 'outages': item.get('customersOutNow'), + 'served': item.get('customersServed'), + 'county': item.get('name'), + 'state': state, + 'company': self.name + }) + return results + except Exception as e: + logger.error(f"Error fetching NISC county data for {self.name}: {e}") + return [] + +class GwtRpcCountyProvider(BaseCountyProvider): + """ Handles county data from GWT-RPC sources. """ + def __init__(self, config, session): + super().__init__(config, session) + self.state_filter = config.get('state_filter') + self.map_url = config.get('map_url') + + # Set up session headers and cookies from config + self.session.headers.update({ + 'User-Agent': config.get('user_agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'), + 'Accept': '*/*', + 'Sec-Fetch-Site': 'same-origin' + }) + if config.get('cookies'): + for cookie in config['cookies']: + self.session.cookies.set(cookie['name'], cookie['value'], domain=cookie['domain'], path=cookie['path']) + + def attempt_auto_repair(self): + if not self.map_url: return False + last_update = self.config.get('last_auto_update') + if last_update: + try: + last_dt = datetime.fromisoformat(last_update) + if last_dt.tzinfo is None: last_dt = last_dt.replace(tzinfo=timezone.utc) + if datetime.now(timezone.utc) - last_dt < timedelta(hours=AUTO_UPDATE_COOLDOWN_HOURS): + logger.info(f"Skipping auto-repair for {self.name} (Cooldown active).") + return False + except ValueError: pass + + logger.info(f"Attempting Auto-Repair for {self.name}...") + try: + _, valid_headers, valid_cookies, valid_body = get_rpc_config_auto.fetch_live_data(self.map_url) + if valid_headers and valid_body: + logger.info(f"Repair successful! Updating {self.name}.") + excluded = {'content-length', 'host', 'connection', 'cookie', 'accept-encoding', 'sec-ch-ua', 'sec-ch-ua-mobile', 'sec-ch-ua-platform', 'origin'} + clean_headers = {k: v for k, v in valid_headers.items() if k.lower() not in excluded} + clean_headers['Referer'] = self.map_url + + # Update in-memory config for the current run + self.config.update({ + 'headers': clean_headers, 'body': valid_body, 'cookies': valid_cookies, + 'user_agent': valid_headers.get('user-agent'), + 'last_auto_update': datetime.now(timezone.utc).isoformat() + }) + # Update session for the current run + self.session.cookies.clear() + for cookie in valid_cookies: + self.session.cookies.set(cookie['name'], cookie['value'], domain=cookie['domain'], path=cookie['path']) + + # Save to disk for next time + update_provider_config(self.name, self.config) + return True + except Exception as e: + logger.error(f"Auto-repair failed: {e}") + return False + + def fetch(self, is_retry=False): + url = self.config.get('url') + headers = self.config.get('headers', {}) + body = self.config.get('body') + if not url or not body: return [] + + try: + parsed_url = urlparse(url) + origin = f"{parsed_url.scheme}://{parsed_url.netloc}" + correct_referer = headers.get('Referer') or headers.get('x-gwt-module-base') or origin + + req_headers = headers.copy() + req_headers['Content-Type'] = 'text/x-gwt-rpc; charset=UTF-8' + req_headers['Referer'] = correct_referer + + resp = self.session.post(url, headers=req_headers, data=body, verify=False) + + if "//EX" in resp.text or resp.status_code == 500: + logger.error(f"GWT Failure for {self.name} (County Fetch).") + if is_retry: return [] + if self.attempt_auto_repair(): + logger.info("Retrying county fetch with new settings...") + # After repair, self.config is updated, so we can just call fetch again. + return self.fetch(is_retry=True) + return [] + + if not resp.ok: return [] + text = resp.text.replace('//OK', '') + return self._extract_county_summary(json.loads(text)) + except Exception as e: + logger.error(f"County fetch error for {self.name}: {e}") + return [] + + def _extract_county_summary(self, data_list): + """ + Decodes a GWT-RPC payload to extract outage data for Counties. + This logic is adapted from test.py. + """ + try: + # 1. Separate Stream and String Table + string_table = None + stream_raw = [] + for item in data_list: + if isinstance(item, list): + string_table = item + break + else: + stream_raw.append(item) + + if not string_table: + logger.error(f"String table not found in payload for {self.name}.") + return [] + + # 2. Normalize the Stream + stream = [] + for token in stream_raw: + if isinstance(token, int): + stream.append(token) + elif isinstance(token, float): + stream.append(int(token)) + elif isinstance(token, str): + try: + stream.append(int(float(token))) + except ValueError: + pass # Ignore non-numeric strings + + # 3. Decode Logic + REGION_SIG = "cc.nisc.oms.clientandserver.v2.pojo.Region/3192921568" + INTEGER_SIG = "java.lang.Integer/3438268394" + CATEGORY_KEY = "County" + + def get_index(val): + try: return string_table.index(val) + 1 + except ValueError: return 0 + + region_type_id = get_index(REGION_SIG) + integer_type_id = get_index(INTEGER_SIG) + county_type_id = get_index(CATEGORY_KEY) + + if region_type_id == 0: + logger.error(f"Region type signature not found for {self.name}.") + return [] + + results = [] + i = 0 + stream_len = len(stream) + + while i < stream_len: + if stream[i] == region_type_id: + try: + p = i + 1 + + served = 0 + val1 = stream[p] + p += 1 + if p < stream_len and stream[p] == integer_type_id: + served = val1 + p += 1 + + out = 0 + val2 = stream[p] + p += 1 + if p < stream_len and stream[p] == integer_type_id: + out = val2 + p += 1 + + name_idx = stream[p] + p += 1 + cat_idx = stream[p] + + if cat_idx == county_type_id: + name = "Unknown" + if 0 < name_idx <= len(string_table): + name = string_table[name_idx - 1] + + results.append({ + 'county': name, 'state': self.state_filter, + 'company': self.name, 'outages': out, 'served': served + }) + except IndexError: + pass + i += 1 + return results + except (ValueError, IndexError, TypeError) as e: + logger.error(f"Could not parse county summary for {self.name}: {e}") + return [] + +# --- REGISTRY --- +PROVIDER_REGISTRY = { + 'kubra_county': KubraCountyProvider, + 'simple_county_json': SimpleCountyJsonProvider, + 'nisc_hosted_county': NiscCountyProvider, + 'gwt_rpc_county': GwtRpcCountyProvider, +} + +# --- MAIN --- +def main(): + S = requests.Session() + S.headers.update({'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'}) + + db = CountyPowerDB(DB_CONFIG) + + logger.info("Starting County Power Scraper...") + providers = load_providers() + + for config in providers: + p_type = config.get('county_type') + p_name = config.get('name') + + ProviderClass = PROVIDER_REGISTRY.get(p_type) + if ProviderClass: + try: + provider = ProviderClass(config, S) + logger.info(f"Fetching county data for {p_name}...") + outages = provider.fetch() + logger.info(f"Found {len(outages)} active outage records for {p_name}.") + + # Process this company's data in a single transaction + db.upsert_and_zero_outages(p_name, outages) + + except Exception as e: + logger.error(f"Error processing {p_name}: {e}") + elif p_type: + logger.warning(f"Unknown provider type '{p_type}' for {p_name}") + + db.run_post_processing() + db.close() + logger.info("County scraping complete.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/newpower2.py b/newpower2.py index 95f25be..054a5c2 100644 --- a/newpower2.py +++ b/newpower2.py @@ -36,7 +36,9 @@ def load_providers(): logger.error(f"{CONFIG_FILE} not found!") return [] with open(CONFIG_FILE, 'r') as f: - return json.load(f) + # Filter for providers that have a 'type' for point-based scraping + all_providers = json.load(f) + return [p for p in all_providers if p.get('type')] def save_providers(providers): with open(CONFIG_FILE, 'w') as f: @@ -549,6 +551,7 @@ PROVIDER_REGISTRY = { } # --- MAIN --- +# --- MAIN (Point Scraper) --- def main(): S = requests.Session() S.verify = False diff --git a/outage_data.txt b/outage_data.txt new file mode 100644 index 0000000..eb7af4a --- /dev/null +++ b/outage_data.txt @@ -0,0 +1 @@ +//OK["Zr4qUVs",301239.12957,141,1603180.59877,141,0,0,0,0,-183,0,0,-182,0,0,0,0,0,150,0,"Zr0UCp6",137,0,0,-10,0,0,0,0,0,0,-180,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,"Zr1MjZg",137,0,0,0,0,0,0,0,0,136,135,134,131,133,132,0,0,128,0,131,130,150,4,302633.61627,141,1601198.37167,141,0,0,0,0,-216,0,0,-182,0,0,0,0,0,149,0,"Zrz0NGg",137,0,0,57,10,0,0,0,0,0,0,-180,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,"Zr0tp3A",137,0,0,0,0,0,0,0,0,136,135,134,131,133,132,0,144,144,143,1,128,0,131,130,149,4,304161.88947,141,1598555.84507,141,0,0,0,0,-183,0,0,-182,0,0,0,0,0,148,0,"Zrz4yEg",137,0,0,-10,0,0,0,0,0,0,-180,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,"Zr0xFlg",137,0,0,0,0,0,0,0,0,136,135,134,131,133,132,0,144,144,143,1,128,0,131,130,148,4,302482.86147,141,1602243.43187,141,0,0,0,0,-183,0,0,-182,0,0,0,0,0,147,0,"Zrz4utz",137,0,0,-10,0,0,0,0,0,0,-180,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,"Zr0xFlg",137,0,0,0,0,0,0,0,0,136,135,134,131,133,132,0,144,144,143,1,128,0,131,130,147,4,303872.94357,141,1600459.13517,141,0,0,0,0,3,140,0,0,-182,0,0,0,0,0,146,0,"Zrz0QeB",137,0,0,165,10,0,0,0,0,0,0,-180,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,"Zr039Cg",137,0,0,0,0,0,0,0,0,136,135,134,131,133,132,0,144,144,143,1,128,0,131,130,146,4,0.0,141,0.0,141,0,0,0,0,6,140,0,0,-182,0,0,0,0,0,145,0,"Zrz2flg",137,0,0,-10,0,0,0,0,0,0,-180,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,"Zr0xFlg",137,0,0,0,0,0,0,0,0,136,135,134,131,133,132,0,144,144,143,1,128,0,131,130,145,4,305521.98897,141,1598866.21207,141,0,0,0,0,-183,0,0,-182,0,0,0,0,0,142,0,"Zrz2CSg",137,0,0,-10,0,0,0,0,0,0,-180,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,"Zr0tp3A",137,0,0,0,0,0,0,0,0,136,135,134,131,133,132,0,144,144,143,1,128,0,131,130,142,4,298175.80987,141,1603235.82047,141,0,0,0,0,5,140,0,0,0,139,0,0,0,0,0,129,0,"Zrz6nQg",137,0,0,-10,0,0,0,0,0,0,0,138,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,"Zr00hUA",137,0,0,0,0,0,0,0,0,136,135,134,131,133,132,0,0,128,0,131,130,129,4,8,3,0,128,0,3,0,7184,10,228,10,120,4,120,4,8,4,5,4,2,3,0,-10,0,0,7,824,10,-7,127,120,7,32,10,-7,126,120,7,849,10,227,10,125,120,7,315,10,-7,124,120,7,4840,10,-7,123,120,7,14,10,-7,122,120,7,172,10,-7,121,120,7,8,6,120,4,-10,-10,119,8,7,-10,-7,118,8,7,-10,-7,117,8,7,-10,-7,116,8,7,-71,-7,115,8,7,19,10,-7,114,8,7,-10,-7,113,8,7,-10,-10,112,8,7,-10,-7,111,8,7,-10,-7,110,8,7,-10,-7,109,8,7,-71,-7,108,8,7,-10,-10,107,8,7,-14,-7,106,8,7,233,10,-7,105,8,7,-10,-7,104,8,7,-10,-7,103,8,7,1476,10,-7,102,8,7,-10,-7,101,8,7,-10,-7,100,8,7,-10,-7,99,8,7,-10,-7,98,8,7,-71,-7,97,8,7,-10,-7,96,8,7,271,10,-7,95,8,7,-34,-7,94,8,7,-10,-10,93,8,7,-23,-7,92,8,7,-10,-10,91,8,7,-10,-7,90,8,7,-10,-7,89,8,7,-10,-7,88,8,7,962,10,-7,87,8,7,-10,-7,86,8,7,103,10,-7,85,8,7,-10,-7,84,8,7,-10,-7,83,8,7,-10,-7,82,8,7,181,10,-7,81,8,7,-10,-10,80,8,7,-10,-7,79,8,7,-10,-7,78,8,7,-10,-7,77,8,7,-48,-7,76,8,7,-10,-7,75,8,7,-10,-7,74,8,7,-71,-7,73,8,7,-10,-7,72,8,7,-10,-7,71,8,7,-10,-7,70,8,7,-10,-7,69,8,7,-10,-7,68,8,7,-10,-7,67,8,7,-14,-10,66,8,7,-10,-7,65,8,7,-10,-10,64,8,7,-10,-7,63,8,7,11,10,-7,0,8,7,-71,-7,62,8,7,-71,-7,61,8,7,-10,-10,60,8,7,-10,-7,59,8,7,951,10,6,10,58,8,7,-10,-7,57,8,7,-34,3,10,56,8,7,-10,-7,55,8,7,-10,-7,54,8,7,578,10,-7,53,8,7,58,10,-7,52,8,7,-14,-7,51,8,7,-10,-7,50,8,7,-10,-7,49,8,7,-10,-7,48,8,7,473,10,-7,47,8,7,-10,-7,46,8,7,-10,-7,45,8,7,-34,-7,44,8,7,-10,-10,43,8,7,-10,-7,42,8,7,-10,-10,41,8,7,-14,-7,40,8,7,1101,10,-34,39,8,7,8,10,-7,38,8,7,-10,-7,37,8,7,369,10,193,10,36,8,7,-10,-7,35,8,7,65,10,-7,34,8,7,-10,-7,33,8,7,-34,-7,32,8,7,-14,-10,31,8,7,-14,-14,30,8,7,-10,-10,29,8,7,4,10,-7,28,8,7,252,10,-7,27,8,7,21,10,-10,26,8,7,-14,-7,25,8,7,-10,-10,24,8,7,7,10,-7,23,8,7,-10,-7,22,8,7,5,10,-7,21,8,7,-10,-7,20,8,7,-10,-7,19,8,7,-10,-10,18,8,7,-10,-7,17,8,7,60,10,-7,16,8,7,-10,-10,15,8,7,-14,2,10,14,8,7,-10,-7,13,8,7,-10,-10,12,8,7,1,10,-7,11,8,7,84,10,0,10,9,8,7,111,6,5,4,2,3,2,1,["coop.nisc.oms.webmap.client.cache.CombinedOutageDetails/2939392267","cc.nisc.oms.clientandserver.v2.pojo.ConsumersSummary/2114508074","java.util.HashMap/1797211028","java.lang.String/2004016611","zipCode","java.util.LinkedHashSet/95640124","cc.nisc.oms.clientandserver.v2.pojo.Region/3192921568","Zip Code","26416","java.lang.Integer/3438268394","26555","26260","26337","26377","15120","26302","25309","19454","26237","25301","26369","21204","26323","26320","44231","26422","26408","26463","47591","26362","26105","26104","25560","26452","15220","26456","08215","26582","26301","26438","22701","26034","44410","26271","25303","25213","26385","21061","60631","33905","19312","26554","26378","26461","44718","26415","24315","26426","26062","27104","26257","26404","26070","26155","44880","26170","08251","02116","22071","26436","26187","23663","26568","44312","92593","19103","43460","45750","16046","44232","26386","25314","25832","20711","26332","25526","26201","26276","32231","19041","43017","26238","25177","26236","26431","43085","26505","21740","25276","26159","25336","26330","21541","26361","26448","26101","43946","26508","25777","15236","44688","25401","44646","26451","75025","15522","43123","21060","44224","County","BARBOUR","MARION","HARRISON","LEWIS","DODDRIDGE","TAYLOR","UPSHUR","java.util.ArrayList/4159755760","158577","cc.nisc.oms.clientandserver.v2.pojo.Outage/3707048994","","cc.nisc.oms.clientandserver.v2.pojo.Cause/4238436236","cc.nisc.oms.clientandserver.v2.pojo.CauseGroup/2252923322","0","99","UNKNOWN ","java.lang.Long/4227064769","java.lang.Boolean/476441737","cc.nisc.oms.clientandserver.v2.pojo.OutageStatus/3497953849","cc.nisc.oms.clientandserver.v2.pojo.OutageType/4066452034","java.lang.Double/858496421","158569","cc.nisc.oms.clientandserver.v2.pojo.Crew/3956537207","crew","158571","158582","158574","158575","158572","158583"],0,7] \ No newline at end of file diff --git a/powercounty.py b/powercounty.py index dbab0b1..1aa0fc2 100644 --- a/powercounty.py +++ b/powercounty.py @@ -1,4 +1,7 @@ # powercounty.py +# +# THIS SCRIPT IS DEPRECATED. +# Please use newpower.py, which uses the providers.json config file. import logging import requests @@ -590,4 +593,5 @@ def main(): if __name__ == '__main__': - main() + # main() + logger.warning("This script (powercounty.py) is deprecated. Please use newpower.py instead.") diff --git a/providers.json b/providers.json index d5c73dd..a8c5b32 100644 --- a/providers.json +++ b/providers.json @@ -4,81 +4,58 @@ "type": "kubra", "meta_url": "https://kubra.io/stormcenter/api/v1/stormcenters/6674f49e-0236-4ed8-a40a-b31747557ab7/views/8cfe790f-59f3-4ce3-a73f-a9642227411f/currentState?preview=false", "layer": "cluster-2", - "quadkeys": [ - "0320001", - "0320003", - "0320010", - "0320011", - "0320012", - "0320013", - "0320021", - "0320030", - "0320031", - "0320100", - "0320102", - "0320120" - ] + "quadkeys": [ "0320001", "0320003", "0320010", "0320011", "0320012", "0320013", "0320021", "0320030", "0320031", "0320100", "0320102", "0320120" ], + "county_type": "kubra_county", + "county_meta_url": "https://kubra.io/stormcenter/api/v1/stormcenters/6674f49e-0236-4ed8-a40a-b31747557ab7/views/8cfe790f-59f3-4ce3-a73f-a9642227411f/currentState?preview=false", + "county_report_suffix": "/public/reports/7929429f-635d-4761-b6c7-78f646cef3c2_report.json" }, { "name": "AEP-OH", "type": "kubra", "meta_url": "https://kubra.io/stormcenter/api/v1/stormcenters/9c0735d8-b721-4dce-b80b-558e98ce1083/views/9b2feb80-69f8-4035-925e-f2acbcf1728e/currentState?preview=false", "layer": "cluster-1", - "quadkeys": [ - "0320013", - "0320010", - "0320011", - "0320012", - "0320003", - "0320001", - "0302322", - "0302233", - "0302232", - "0302223", - "0320102", - "0320100" - ] + "quadkeys": [ "0320013", "0320010", "0320011", "0320012", "0320003", "0320001", "0302322", "0302233", "0302232", "0302223", "0320102", "0320100" ], + "county_type": "kubra_county", + "county_meta_url": "https://kubra.io/stormcenter/api/v1/stormcenters/9c0735d8-b721-4dce-b80b-558e98ce1083/views/9b2feb80-69f8-4035-925e-f2acbcf1728e/currentState?preview=false", + "county_report_suffix": "/public/reports/1bc6bd19-2315-4548-980a-6df73b93b355_report.json" }, { "name": "AEP-KY", "type": "kubra", "meta_url": "https://kubra.io/stormcenter/api/v1/stormcenters/23dcd38e-2573-4e20-a463-959b11cae011/views/60f31606-5702-4a1e-a74c-08d866b7a6fa/currentState?preview=false", "layer": "cluster-2", - "quadkeys": [ - "0320031", - "0320030", - "0320021", - "0320013", - "0320012", - "0320011", - "0320010", - "0320003", - "0320001" - ] + "quadkeys": [ "0320031", "0320030", "0320021", "0320013", "0320012", "0320011", "0320010", "0320003", "0320001" ], + "county_type": "kubra_county", + "county_meta_url": "https://kubra.io/stormcenter/api/v1/stormcenters/23dcd38e-2573-4e20-a463-959b11cae011/views/60f31606-5702-4a1e-a74c-08d866b7a6fa/currentState?preview=false", + "county_report_suffix": "/public/reports/03da18cc-51e0-4662-98e5-e9d49dcf08c8_report.json", + "state_filter": "KY" }, { "name": "FirstEnergy", "type": "kubra", "meta_url": "https://kubra.io/stormcenter/api/v1/stormcenters/6c715f0e-bbec-465f-98cc-0b81623744be/views/5ed3ddf1-3a6f-4cfd-8957-eba54b5baaad/currentState?preview=false", "layer": "cluster-4", - "quadkeys": [ - "030223", - "030232", - "032001", - "032003", - "032010", - "032012" - ] + "quadkeys": [ "030223", "030232", "032001", "032003", "032010", "032012" ], + "county_type": "kubra_county", + "county_meta_url": "https://kubra.io/stormcenter/api/v1/stormcenters/6c715f0e-bbec-465f-98cc-0b81623744be/views/5ed3ddf1-3a6f-4cfd-8957-eba54b5baaad/currentState?preview=false", + "county_report_suffix": "/public/reports/8c3b0b30-c9e8-4e8f-8b0d-999c568bb085_report.json", + "state_filter": "WV" }, { "name": "SouthCentral", - "type": "simple_json", - "url": "https://outage.southcentralpower.com/data/outages.json" + "type": "simple_json", + "url": "https://outage.southcentralpower.com/data/outages.json", + "county_type": "simple_county_json", + "county_url": "https://outage.southcentralpower.com/data/boundaries.json", + "state_filter": "OH" }, { "name": "Grayson", - "type": "simple_json", - "url": "https://outages.graysonrecc.com/data/outages.json" + "type": "simple_json", + "url": "https://outages.graysonrecc.com/data/outages.json", + "county_type": "simple_county_json", + "county_url": "https://outages.graysonrecc.com/data/boundaries.json", + "state_filter": "KY" }, { "name": "Harrison REA", @@ -94,7 +71,8 @@ "Referer": "https://harrisonrea.ebill.coop/maps/external/OutageWebMap/" }, "body": "7|0|4|https://harrisonrea.ebill.coop/maps/external/OutageWebMap/maps/|612278413EC26C34D54A3907AA0CDFD8|coop.nisc.oms.webmap.services.RpcCombinedOutageDetailsService|getCombinedOutageDetails|1|2|3|4|0|", - "last_auto_update": "2025-12-07T03:09:05.982825+00:00" + "last_auto_update": "2025-12-07T03:09:05.982825+00:00", + "county_type": "gwt_rpc_county" }, { "name": "WashingtonElectric", @@ -113,6 +91,7 @@ "Referer": "https://weci.ebill.coop/woViewer/mapviewer.html?config=Outage+Web+Map" }, "body": "7|0|4|https://weci.ebill.coop/woViewer/MapWiseWeb/|612278413EC26C34D54A3907AA0CDFD8|coop.nisc.oms.webmap.services.RpcCombinedOutageDetailsService|getCombinedOutageDetails|1|2|3|4|0|", + "county_type": "gwt_rpc_county", "last_auto_update": "2025-12-07T03:56:27.722877+00:00", "cookies": [ { @@ -222,6 +201,9 @@ "name": "BuckeyeREC", "type": "nisc_hosted", "url": "https://outagemap-data.cloud.coop/buckeyerec/Hosted_Outage_Map/summary.json", - "proj_string": "+proj=lcc +lat_1=40.03333333333333 +lat_2=38.73333333333333 +lat_0=38.73333333333333 +lon_0=-82.648 +x_0=0 +y_0=0 +ellps=GRS80 +datum=NAD83 +to_meter=0.3048006096012192 +no_defs" + "proj_string": "+proj=lcc +lat_1=40.03333333333333 +lat_2=38.73333333333333 +lat_0=38.73333333333333 +lon_0=-82.648 +x_0=0 +y_0=0 +ellps=GRS80 +datum=NAD83 +to_meter=0.3048006096012192 +no_defs", + "county_type": "nisc_hosted_county", + "county_url": "https://outagemap-data.cloud.coop/buckeyerec/Hosted_Outage_Map/boundaries.json", + "state_filter": "OH" } ] \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 0000000..1e69a6a --- /dev/null +++ b/test.py @@ -0,0 +1,163 @@ +import json +import os + +def decode_gwt_rpc(payload): + """ + Decodes a GWT-RPC payload to extract outage data for Counties. + """ + # 1. Clean the payload + # GWT responses often start with //OK. We strip that. + if payload.startswith("//OK"): + payload = payload[4:] + + # 2. Parse the FULL payload as JSON + # The GWT payload is structurally a JSON array: [stream_data..., [string_table], flags...] + try: + full_data = json.loads(payload) + except json.JSONDecodeError as e: + print(f"Error parsing payload JSON: {e}") + return None + + # 3. Separate Stream and String Table + # The String Table is a list of strings located near the end of the main array. + # The "Stream" is everything before that string table. + + string_table = None + stream_raw = [] + + # Iterate through the parsed array to find the string table (which is a list) + for item in full_data: + if isinstance(item, list): + string_table = item + # Once we find the table, we assume the rest are flags and stop adding to stream + break + else: + stream_raw.append(item) + + if not string_table: + print("Error: String table not found in payload.") + return None + + # 4. Normalize the Stream + # The decoder logic relies on integers (1-based indices). + # The raw stream might contain floats or strings that we need to cast or filter. + stream = [] + for token in stream_raw: + if isinstance(token, int): + stream.append(token) + elif isinstance(token, float): + stream.append(int(token)) + elif isinstance(token, str): + # Sometimes numeric values are sent as strings in the stream + try: + stream.append(int(float(token))) + except ValueError: + # If it's a non-numeric string token (like a cache ID), ignore it + pass + + # 5. Decode Logic + try: + # Define the signatures we are looking for in the String Table + REGION_SIG = "cc.nisc.oms.clientandserver.v2.pojo.Region/3192921568" + INTEGER_SIG = "java.lang.Integer/3438268394" + CATEGORY_KEY = "County" + + # Helper to find 1-based index + def get_index(val): + try: return string_table.index(val) + 1 + except ValueError: return 0 + + region_type_id = get_index(REGION_SIG) + integer_type_id = get_index(INTEGER_SIG) + county_type_id = get_index(CATEGORY_KEY) + + if region_type_id == 0: + print("Error: Region type signature not found in string table.") + # Debug: Print first few strings to verify if signatures changed + # print("Available strings:", string_table[:10]) + return None + + results = [] + i = 0 + stream_len = len(stream) + + # Iterate through the stream looking for Region objects + while i < stream_len: + if stream[i] == region_type_id: + try: + # We found a Region. The next few integers define its properties. + # Pointer 'p' is relative to current index 'i' + p = i + 1 + + # --- Field 1: Total Served --- + # Logic: Value is valid if followed by Integer Type ID + served = 0 + val1 = stream[p] + p += 1 + if p < stream_len and stream[p] == integer_type_id: + served = val1 + p += 1 # Skip type ID + + # --- Field 2: Number Out --- + out = 0 + val2 = stream[p] + p += 1 + if p < stream_len and stream[p] == integer_type_id: + out = val2 + p += 1 # Skip type ID + + # --- Field 3: Name Index --- + name_idx = stream[p] + p += 1 + + # --- Field 4: Category Index --- + cat_idx = stream[p] + + # Check if this is a County + if cat_idx == county_type_id: + name = "Unknown" + if 0 < name_idx <= len(string_table): + name = string_table[name_idx - 1] + + percent = 0.0 + if served > 0: + percent = (out / served) * 100 + + results.append({ + "county": name, + "served": served, + "out": out, + "percent": percent + }) + + except IndexError: + pass + i += 1 + + return results + + except Exception as e: + print(f"Error during stream traversal: {e}") + return None + +if __name__ == "__main__": + filename = "outage_data.txt" + if os.path.exists(filename): + with open(filename, "r", encoding="utf-8") as f: + raw_content = f.read().strip() + + data = decode_gwt_rpc(raw_content) + + if data: + # Sort A-Z + data.sort(key=lambda x: x['county']) + + print(f"{'County':<20} | {'Served':>8} | {'Out':>8} | {'Percent':>8}") + print("-" * 55) + for row in data: + print(f"{row['county']:<20} | {row['served']:>8} | {row['out']:>8} | {row['percent']:>7.2f}%") + else: + print("No data found.") + else: + print(f"File '{filename}' not found. Please create it and paste the payload.") + \ No newline at end of file