import requests import polyline import json import psycopg2 import psycopg2.extensions from datetime import datetime, timezone from geojson import Point, Feature, FeatureCollection, dump import pandas as pd from requests.packages.urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings(InsecureRequestWarning) # TODO # Add Buckeye REC https://buckeyerec.outagemap.coop/ conn = psycopg2.connect(host='localhost', database='nws', user='nws', password='nws') cursor = conn.cursor() proxies = {"http":"http://nws:nws@localhost:9000"} aepwvnew = ['0320001','0320003','0320010','0320011','0320012','0320013','0320021','0320030','0320031','0320100','0320102','0320120'] aepohnew = ['0320013','0320010','0320011','0320012','0320003','0320001','0302322','0302233','0302232','0302223','0320102','0320100'] aepkynew = ['0320031','0320030','0320021','0320013','0320012','0320011','0320010','0320003','0320001'] firstenergy = ['030223','030232','032001','032003','032010','032012'] dominionva = ['0320121','0320120','0300103','0320102','0320101','0320100','0320031','0320013','0320011'] baltimore = ['0320011','0320100','0320101','0320013','0320102','0320103'] pepco = ['03201002','03201003','03201020','03201021'] aepohmeta = "http://outagemap.aepohio.com.s3.amazonaws.com/resources/data/external/interval_generation_data/metadata.json" aepwvmeta = "http://outagemap.appalachianpower.com.s3.amazonaws.com/resources/data/external/interval_generation_data/metadata.json" aepwvkubra = "https://kubra.io/stormcenter/api/v1/stormcenters/6674f49e-0236-4ed8-a40a-b31747557ab7/views/8cfe790f-59f3-4ce3-a73f-a9642227411f/currentState?preview=false" aepohkubra = 'https://kubra.io/stormcenter/api/v1/stormcenters/9c0735d8-b721-4dce-b80b-558e98ce1083/views/9b2feb80-69f8-4035-925e-f2acbcf1728e/currentState?preview=false' aepkykubra = 'https://kubra.io/stormcenter/api/v1/stormcenters/23dcd38e-2573-4e20-a463-959b11cae011/views/60f31606-5702-4a1e-a74c-08d866b7a6fa/currentState?preview=false' #firstpowerwvmeta = "https://s3.amazonaws.com/outages.sc4.firstenergycorp.com/resources/data/pa/interval_generation_data/metadata.json" aepkymeta = 'http://outagemap.kentuckypower.com.s3.amazonaws.com/resources/data/external/interval_generation_data/metadata.json' domvameta = 'https://outagemap.dominionenergy.com/resources/data/external/interval_generation_data/metadata.json' wvfemeta = 'https://kubra.io/stormcenter/api/v1/stormcenters/6c715f0e-bbec-465f-98cc-0b81623744be/views/5ed3ddf1-3a6f-4cfd-8957-eba54b5baaad/currentState?preview=false' aepwvcluster = 'cluster-2' aepohcluster = 'cluster-1' aepkycluster = 'cluster-2' aepwvbase = "http://outagemap.appalachianpower.com.s3.amazonaws.com/resources/data/external/interval_generation_data/" aepbasewv = 'https://kubra.io/cluster-data/' aepohbase = "http://outagemap.aepohio.com.s3.amazonaws.com/resources/data/external/interval_generation_data/" #firstpowerwvbase = "https://s3.amazonaws.com/outages.sc4.firstenergycorp.com/resources/data/mdwv/interval_generation_data/" aepkybase = 'http://outagemap.kentuckypower.com.s3.amazonaws.com/resources/data/external/interval_generation_data/' domvabase = 'https://outagemap.dominionenergy.com/resources/data/external/interval_generation_data/' graysonrecc = 'https://outages.graysonrecc.com/data/outages.json' kubrabase = 'https://kubra.io/cluster-data/' firstenergybase = 'https://kubra.io/cluster-data/' firstenergycluster = 'cluster-4' firstenergyhex1 = 'f5f94943-5df4-4752-a0a7-8ef4baded880' firstenergyhex2 = 'e2986f8a-5a69-4d2f-821c-e5db03932b68' southcentraljson = 'https://outage.southcentralpower.com/data/outages.json' allcountyoutages = [] allkubraoutages = [] allaepkubracoutages = [] def remove_external_curly_braces(s): try: p = s[0] return p except Error as e: print('error in curly ' + e) return s def get_kubra_hexes(url): outage = S.get(url) try: tempdata = json.loads(outage.text) bothhex = tempdata.get('data').get('cluster_interval_generation_data') hexes = bothhex.split('/') returndata = (hexes[2],hexes[3]) return returndata except Exception as e: print(e) def kubra(baseurl,cluster,namearray,meta): try: data = get_kubra_hexes(meta) #print(data) hex1, hex2 = get_kubra_hexes(meta) newnamearray = [] for i in namearray: dir = str(i) dir = dir[-3:] dir = str(dir[::-1]) url = baseurl + dir + '/' + hex1 + '/' + hex2 + '/public/' + cluster + '/' + i + '.json' outage = S.get(url) if outage.headers.get('Content-Type').startswith('application/json'): tempdata = json.loads(outage.text) for j in tempdata['file_data']: outageinfo = None try: outageinfo = j.get('desc').get('cluster') except: continue if outageinfo == True: for k in range(4): newnamearray.append(str(i)+ str(k)) if outageinfo == False: allkubraoutages.append(j) newnamearray = list(dict.fromkeys(newnamearray)) if len(newnamearray) > 0: kubra(baseurl,cluster,newnamearray,meta) except Exception as e: print(e) def kubra_aep(baseurl,cluster,namearray,meta): data = get_kubra_hexes(meta) #print(data) hex1, hex2 = get_kubra_hexes(meta) newnamearray = [] for i in namearray: dir = str(i) dir = dir[-3:] dir = str(dir[::-1]) url = baseurl + dir + '/' + hex1 + '/' + hex2 + '/public/' + cluster + '/' + i + '.json' outage = S.get(url) if outage.headers.get('Content-Type').startswith('application/json'): tempdata = json.loads(outage.text) for j in tempdata['file_data']: outageinfo = None try: outageinfo = j.get('desc').get('cluster') except: continue if outageinfo == True: for k in range(4): newnamearray.append(str(i)+ str(k)) if outageinfo == False: allaepkubracoutages.append(j) #allkubraoutages.append(j) newnamearray = list(dict.fromkeys(newnamearray)) if len(newnamearray) > 0: kubra_aep(baseurl,cluster,newnamearray,meta) def insertkubra(data): for j in data: try: custa = j.get('desc').get('cust_a').get('val') except: continue pointgeom = j.get('geom').get('p') if len(pointgeom) == 1: pointlatlon = polyline.decode(pointgeom[0]) lat = pointlatlon[0][0] lon = pointlatlon[0][1] else: continue areageom = j.get('geom').get('a') if areageom != None: areageom = remove_external_curly_braces(areageom) else: areageom = None cause = j.get('desc').get('cause').get('EN-US') #cause = j.get('desc').get('cause') start = j.get('desc').get('start_time') if start != None: try: start = datetime.strptime(start,"%Y-%m-%dT%H:%M:%S%z") except ValueError: start = datetime.strptime(start,"%Y-%m-%dT%H:%M%z") etr = j.get('desc').get('etr') if etr == 'ETR-NULL' or etr == 'ETR-EXP': etr = None if etr != None: try: etr = datetime.strptime(etr,"%Y-%m-%dT%H:%M:%S%z") except ValueError: etr = datetime.strptime(etr,"%Y-%m-%dT%H:%M%z") incid = j.get('desc').get('inc_id') crew_status = j.get('desc').get('crew_status').get('EN-US') current_timestamp = str(datetime.utcnow()) sql = "INSERT INTO power (lat,lon,pointgeom,areageom,start,cause,outagen,crew_status,incidentid,peakoutage,etr,derivedstart,lastchange,active) values (%s,%s, %s, %s, %s, %s, %s, %s, %s ,%s, %s, %s, %s, %s) on conflict (pointgeom) do update set (outagen, cause, start, etr, crew_status,lastchange) = (%s, %s, %s, %s, %s, %s)" vals = (lat,lon,pointgeom,areageom, start, cause, custa, crew_status, incid, custa,etr,current_timestamp, current_timestamp,'True',custa, cause, start, etr, crew_status,current_timestamp) val = (sql,vals) # print(val) cursor.execute(sql,vals) conn.commit() def insert_kubra_aep(data): for j in data: try: custa = j.get('desc').get('cust_a').get('val') except: continue pointgeom = j.get('geom').get('p') if len(pointgeom) == 1: pointlatlon = polyline.decode(pointgeom[0]) lat = pointlatlon[0][0] lon = pointlatlon[0][1] else: continue areageom = j.get('geom').get('a') if areageom != None: areageom = remove_external_curly_braces(areageom) else: areageom = None cause = "Pending Investigation" # Default to Pending if no cause is found cause_dict = j.get('desc').get('cause') if cause_dict: cause = cause_dict.get('EN-US') #cause = j.get('desc').get('cause') start = j.get('desc').get('start_time') if start != None: try: start = datetime.strptime(start,"%Y-%m-%dT%H:%M:%S%z") except ValueError: start = datetime.strptime(start,"%Y-%m-%dT%H:%M%z") etr = j.get('desc').get('etr') if etr == 'ETR-NULL' or etr == 'ETR-EXP': etr = None if etr != None: try: etr = datetime.strptime(etr,"%Y-%m-%dT%H:%M:%S%z") except ValueError: etr = datetime.strptime(etr,"%Y-%m-%dT%H:%M%z") incid = j.get('desc').get('inc_id') crew_status = j.get('desc').get('crew_status').get('EN-US') current_timestamp = str(datetime.utcnow()) sql = "INSERT INTO power (lat,lon,pointgeom,areageom,start,cause,outagen,crew_status,incidentid,peakoutage,etr,derivedstart,lastchange,active) values (%s,%s, %s, %s, %s, %s, %s, %s, %s ,%s, %s, %s, %s, %s) on conflict (pointgeom) do update set (outagen, cause, start, etr, crew_status,lastchange) = (%s, %s, %s, %s, %s, %s)" vals = (lat,lon,pointgeom,areageom, start, cause, custa, crew_status, incid, custa,etr,current_timestamp, current_timestamp,'True',custa, cause, start, etr, crew_status,current_timestamp) val = (sql,vals) #print(val) cursor.execute(sql,vals) conn.commit() def remove_dupes(l): b = [] for i in range(0, len(l)): if l[i] not in l[i+1:]: b.append(l[i]) return b def newaep(meta,namearray,baseurl): newnamearray = [] metainfo = json.loads(S.get(meta).text) metadir = metainfo['directory'] for i in namearray: url = baseurl + metadir + '/outages/' + i + '.json' outage = S.get(url) if outage.headers.get('Content-Type').startswith('application/octet-stream'): tempdata = json.loads(outage.text) for j in tempdata['file_data']: outageinfo = None try: outageinfo = j.get('title') except: continue if outageinfo == 'Area Outage': for k in range(4): newnamearray.append(str(i)+ str(k)) if outageinfo == 'Outage Information': allkubraoutages.append(j) newnamearray = list(dict.fromkeys(newnamearray)) if len(newnamearray) > 0: newaep(meta,newnamearray,baseurl) def check_bad_offset(offset): try: if ":" == offset[-3:-2]: offset = offset[:-3]+offset[-2:] return offset except: return offset def fix_bad_timestamp(timestamp): parsed_timestamp = pd.to_datetime(timestamp) return parsed_timestamp S = requests.Session() S.verify = False def southcentral(): Sp = requests.Session() Sp.verify = False #Sp.proxies.update(proxies) temp = Sp.get(southcentraljson).text outageinfo = json.loads(temp) # print(outageinfo) if len(outageinfo) != 0: for i in outageinfo: id = i.get('outageRecID') lat = i.get('outagePoint').get('lat') lon = i.get('outagePoint').get('lng') start = i.get('outageStartTime') end = i.get('outageEndTime') cause = i.get('cause') initial = i.get('customersOutInitially') now = i.get('customersOutNow') change = i.get('outageModifiedTime') crew = i.get('outageWorkStatus') # change = check_bad_offset(change) # start = check_bad_offset(start) # end = check_bad_offset(end) if start != None: start = fix_bad_timestamp(start) if end != None: end = fix_bad_timestamp(end) if change != None: change = fix_bad_timestamp(change) # change = datetime.strptime(change,"%Y-%m-%dT%H:%M:%S%f%z") current_timestamp = str(datetime.utcnow()) sql = "INSERT INTO power (lat,lon,start,derivedstart,cause,outagen,crew_status,peakoutage,pointgeom,lastchange,active) values (%s,%s,%s, %s, %s, %s, %s, %s, %s, %s ,%s) on conflict (pointgeom) do update set (outagen, cause, start, crew_status,lastchange) = (%s, %s, %s, %s, %s)" vals = (lat,lon, start, current_timestamp, cause, now, crew, initial,id,change,'True',now,cause,start,crew,change) val = (sql,vals) cursor.execute(sql,vals) conn.commit() def grayson(): outageinfo = json.loads(S.get(graysonrecc).text) if len(outageinfo) != 0: for i in outageinfo: id = i.get('outageRecID') lat = i.get('outagePoint').get('lat') lon = i.get('outagePoint').get('lng') start = i.get('outageStartTime') end = i.get('outageEndTime') cause = i.get('cause') initial = i.get('customersOutInitially') now = i.get('customersOutNow') change = i.get('outageModifiedTime') crew = i.get('outageWorkStatus') # change = check_bad_offset(change) # start = check_bad_offset(start) # end = check_bad_offset(end) if start != None: start = fix_bad_timestamp(start) if end != None: end = fix_bad_timestamp(end) if change != None: change = fix_bad_timestamp(change) # change = datetime.strptime(change,"%Y-%m-%dT%H:%M:%S%f%z") current_timestamp = str(datetime.utcnow()) sql = "INSERT INTO power (lat,lon,start,derivedstart,cause,outagen,crew_status,peakoutage,pointgeom,lastchange,active) values (%s,%s,%s, %s, %s, %s, %s, %s, %s, %s ,%s) on conflict (pointgeom) do update set (outagen, cause, start, crew_status,lastchange) = (%s, %s, %s, %s, %s)" vals = (lat,lon, start, current_timestamp, cause, now, crew, initial,id,change,'True',now,cause,start,crew,change) val = (sql,vals) cursor.execute(sql,vals) conn.commit() def check_outages(meta,namearray,baseurl): metainfo = json.loads(S.get(meta).text) metadir = metainfo['directory'] for i in namearray: url = baseurl + metadir + '/outages/' + i + '.json' outage = S.get(url) if outage.headers.get('Content-Type').startswith('application/octet-stream'): tempdata = json.loads(outage.text) for j in tempdata['file_data']: id = j.get('id') try: custa = j.get('desc').get('cust_a').get('val') except: continue pointgeom = j.get('geom').get('p') if len(pointgeom) == 1: pointlatlon = polyline.decode(pointgeom[0]) lat = pointlatlon[0][0] lon = pointlatlon[0][1] else: continue areageom = j.get('geom').get('a') if areageom != None: areageom = remove_external_curly_braces(areageom) else: areageom = None cause = j.get('desc').get('cause') start = j.get('desc').get('start') if start != None: start = datetime.strptime(start,"%Y-%m-%dT%H:%M:%S%f%z") etr = j.get('desc').get('etr') if etr == 'ETR-NULL' or etr == 'ETR-EXP': etr = None if etr != None: etr = datetime.strptime(etr,"%Y-%m-%dT%H:%M:%S%f%z") incid = j.get('desc').get('inc_id') crew_status = j.get('desc').get('crew_status') current_timestamp = str(datetime.utcnow()) sql = "INSERT INTO power (lat,lon,pointgeom,areageom,start,cause,outagen,crew_status,incidentid,peakoutage,etr,genericid,derivedstart,lastchange,active) values (%s,%s,%s, %s, %s, %s, %s, %s, %s, %s ,%s, %s, %s, %s, %s) on conflict (pointgeom) do update set (outagen, cause, start, etr, crew_status,lastchange) = (%s, %s, %s, %s, %s, %s)" vals = (lat,lon,pointgeom,areageom, start, cause, custa, crew_status, incid, custa,etr,id,current_timestamp, current_timestamp,'True',custa, cause, start, etr, crew_status,current_timestamp) val = (sql,vals) cursor.execute(sql,vals) conn.commit() try: southcentral() except Exception as e: print(e) try: grayson() except Exception as e: print(e) #try: # newaep(aepwvmeta,aepwvnew,aepwvbase) #except Exception as e: # print(e) #try: # newaep(aepohmeta,aepohnew,aepohbase) #except Exception as e: # print(e) #try: # newaep(aepkymeta,aepkynew,aepkybase) #except Exception as e: # print(e) try: kubra_aep(kubrabase,aepwvcluster,aepwvnew,aepwvkubra) except Exception as e: print(e) try: kubra_aep(kubrabase,aepohcluster,aepohnew,aepohkubra) except Exception as e: print(e) try: kubra_aep(kubrabase,aepkycluster,aepkynew,aepkykubra) except Exception as e: print(e) try: #newaep(firstpowerwvmeta,firstpowerwvnew,firstpowerwvbase) kubra(firstenergybase,firstenergycluster,firstenergy,wvfemeta) except Exception as e: print(e) #try: # newaep(domvameta,dominionva,domvabase) #except Exception as e: # print(e) #kubra(kubrabase,aepwvcluster,aepwvnew,aepwvmeta) nodupe = remove_dupes(allcountyoutages) nodupekubra = remove_dupes(allkubraoutages) nodupeaepkubra = remove_dupes(allaepkubracoutages) #print(nodupe) def insertaep(data): for j in data: try: custa = j.get('desc').get('cust_a').get('val') except: continue pointgeom = j.get('geom').get('p') if len(pointgeom) == 1: pointlatlon = polyline.decode(pointgeom[0]) lat = pointlatlon[0][0] lon = pointlatlon[0][1] else: continue areageom = j.get('geom').get('a') if areageom != None: areageom = remove_external_curly_braces(areageom) else: areageom = None cause = j.get('desc').get('cause') start = j.get('desc').get('start') if start != None: start = datetime.strptime(start,"%Y-%m-%dT%H:%M:%S%f%z") etr = j.get('desc').get('etr') if etr == 'ETR-NULL' or etr == 'ETR-EXP': etr = None if etr != None: etr = datetime.strptime(etr,"%Y-%m-%dT%H:%M:%S%f%z") incid = j.get('desc').get('inc_id') crew_status = j.get('desc').get('crew_status') current_timestamp = str(datetime.utcnow()) #sql = "INSERT INTO power (lat,lon,pointgeom,areageom,start,cause,outagen,crew_status,incidentid,peakoutage,etr,derivedstart,lastchange,active) values (%s,%s, %s, %s, %s, %s, %s, %s, %s ,%s, %s, %s, %s, %s) on conflict (pointgeom) do update set (outagen, cause, start, etr, crew_status,lastchange) = (%s, %s, %s, %s, %s, %s)" vals = (lat,lon,pointgeom,areageom, start, cause, custa, crew_status, incid, custa,etr,current_timestamp, current_timestamp,'True',custa, cause, start, etr, crew_status,current_timestamp) #val = (sql,vals) print(vals) #cursor.execute(sql,vals) #conn.commit() if len(nodupe) > 0: insertaep(nodupe) if len(nodupekubra) > 0: insertkubra(nodupekubra) if len(nodupeaepkubra) > 0: insert_kubra_aep(nodupeaepkubra) cursor.execute('UPDATE public.power SET realgeom = ST_SetSRID(ST_MakePoint(lon, lat), 4326) where (lat is not null and lon is not null and realgeom is null)') cursor.execute('UPDATE public.power SET peakoutage = outagen where outagen > peakoutage') cursor.execute('update public.power set county = county.countyname from public.county where ST_contains(county.geom,power.realgeom) and power.county is null') cursor.execute('update public.power set cwa = fzone.cwa from public.fzone where ST_contains(fzone.geom,power.realgeom) and power.cwa is null') cursor.execute('update public.power set state = county.state from public.county where ST_contains(county.geom,power.realgeom) and power.state is null') cursor.execute('update public.power set startguess = least(start,derivedstart)') cursor.execute('update public.power set realareageom = st_linefromencodedpolyline(areageom) where areageom is not null and realareageom is null') cursor.execute("update power set pointgeom = NULL where lastchange < now() - interval '2 hours'") conn.commit() cursor.execute("update power set active = true where lastchange > now() - interval '30 minutes'") cursor.execute("update power set active = false where lastchange < now() - interval '30 minutes'") conn.commit() #cursor.execute("delete from power where cwa != 'RLX'") cursor.execute("delete from power where lastchange < now () - interval '365 days'") conn.commit() #print(allkubraoutages) cursor.close() conn.close()