test/ohcam.py

import time
import requests
import json
import psycopg2
import psycopg2.extensions
from psycopg2.extras import Json, execute_values 
import re

conn = psycopg2.connect(host='127.0.0.1', database='nws', user='nws', password='nws')
cursor = conn.cursor()

ohcams = 'https://api.ohgo.com/roadmarkers/cameras?pointData={%22lowLongitude%22:-83.85946954181879,%22highLongitude%22:-77.00743340900629,%22lowLatitude%22:37.434082799235426,%22highLatitude%22:40.20012906764399,%22routeDirection%22:%22%22,%22routeName%22:%22%22}'

response = json.loads(requests.get(ohcams).text)
tings = []
for p in response:
 tings = [p['Cameras'][0]['Direction'], p['Cameras'][0]['LargeURL'], p['Latitude'], p['Longitude'],p['Description'],'true','normalproxy','10','https://ohgo.com']
 sql = 'INSERT INTO cams (aspect, url, lat, lon, description, active, method, interval,source) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING;'
 cursor.execute(sql,tings)
 conn.commit()




wvids = 'https://wv511.org/wsvc/gmap.asmx/buildCamerasJSONjs'

wvheaders = {'Connection': 'keep-alive',
             'Accept': 'application/json',
             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36',
             'X-HTTP-Method-Override': 'GET',
             'Referer': 'http://wv511.org/',
             'Accept-Language': 'en-US,en;q=0.9',
             'Cookie': 'mapExtent=-86.271172363286%3A35.832853816094%3A-74.92229541016%3A41.956945104642'}

r = requests.get(wvids,headers=wvheaders)
x = r.text
potato = re.findall(r'var camera_data = { "count": 118,(.*?}])?', x, re.DOTALL | re.MULTILINE)
potato = re.findall(r' "cams":.(.*?}])?', x, re.DOTALL | re.MULTILINE)
potatoer = json.loads(potato[0])


wvcams = []


#print(potatoer)
for i in potatoer:
  try:
    lat = i['start_lat']
    lon = i['start_lng']
    url = str('https://sfstest.roadsummary.com/rtplive/' + str(i['md5'])+'/playlist.m3u8')
    description = re.findall(r'<div id="camDescription">(.*?)<span?',i['description'])[0]
    tings = [description,lat,lon,url,'rtsp','true','10','http://wv511.org']
    sql = 'INSERT INTO cams (description, lat, lon, url, method, active, interval,source) VALUES (%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING;'
    cursor.execute(sql,tings)
    conn.commit()
  except Exception as e:
    print(e,url)








#Bounding box of CWA

x101 = 137
x102 = 143
y101 = 194
y102 = 200








def bloomsky():
    
  (xlat,xlon,nlat,nlon) = (40.000897, -77.513573, 36.749991, -83.098124)
  cursor.execute("UPDATE cams set bloomsky = url where method = 'bloomsky' and bloomsky is null")
  conn.commit()
  cursor.execute("SELECT camid, bloomsky from cams where method = 'bloomsky' and active = True")
  bloomcams = cursor.fetchall()
  bloomurl = 'https://map.bloomsky.com/data/device/'
  bloomheaders = {'Connection': 'keep-alive',
                        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36',
                        'Content-Type': 'application/json; charset=UTF-8',
                        'Accept': '*/*',
                        'Origin': 'https://map.bloomsky.com',
                        'Sec-Fetch-Site': 'same-origin',
                        'Sec-Fetch-Mode': 'cors',
                        'Sec-Fetch-Dest': 'empty',
                        'X-HTTP-Method-Override': 'POST',
                        'Host': 'map.bloomsky.com',
                        'Referer': 'https://map.bloomsky.com',
                        'Accept-Language': 'en-US,en;q=0.9'}
                        #'Cookie': 'AWSELB=A38D8DDD18933E21FE3ADA5953BE34121339A3A57D7398DD0E6A3F21299C92AD55655B00C912D9CF7F381931286A93623AAED145BC6DAEC803C16F67CC4FA4015E6CF710F2; AWSELBCORS=A38D8DDD18933E21FE3ADA5953BE34121339A3A57D7398DD0E6A3F21299C92AD55655B00C912D9CF7F381931286A93623AAED145BC6DAEC803C16F67CC4FA4015E6CF710F2'

       
  
  bloompost = json.dumps({"type":"ALL"})
  r = requests.post(bloomurl,headers=bloomheaders,data=bloompost)
#  print(r)
  bloomcameras = r.json()
  if r.status_code == 200:
   for p in bloomcameras:
     lat = bloomcameras[p]['lat']    
     lon = bloomcameras[p]['lng']    
     if (lat < xlat and lat > nlat and lon < xlon and lon > nlon):
         tings = [bloomcameras[p]['img'], bloomcameras[p]['city'],bloomcameras[p]['name'],lat,lon,p,'10','True','240','bloomsky','https://map.bloomsky.com/weather-stations/' + str(p)]
         sql = "INSERT INTO cams (url,town,description,lat,lon,bloomsky,interval,active,keephours, method,source) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
         cursor.execute(sql,tings)
         
  
  conn.commit()



def ambient():
 url = "https://lightning.ambientweather.net/devices?%24publicBox[0][0]=-83.144531&%24publicBox[0][1]=36.72&%24publicBox[1][0]=-75.5647&%24publicBox[1][1]=40.33802332110745&%24limit=1000&rank=2"
 r = requests.get(url)
 x = r.json()
 for i in x['data']:
  if (i.get('info').get('webcam') != None and "jpg" in i.get('info').get('webcam')):
   description = i.get('info').get('coords').get('location')
   lat = i.get('info').get('coords').get('coords').get('lat')
   lon = i.get('info').get('coords').get('coords').get('lon')
   url = i.get('info').get('webcam')


   try:
    tings = [description,lat,lon,url,'normal','true','10','https://ambientweather.net/']
    sql = 'INSERT INTO cams (description, lat, lon, url, method, active, interval,source) VALUES (%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING;'
    cursor.execute(sql,tings)
   except Exception as e:
    print(e)
 conn.commit()

ambient()



def kymeso():
 ky = 'https://www.kymesonet.org/json/StationsInfo.json'
 response = json.loads(requests.get(ky).text)
 tings = []
 base = 'https://www.kymesonet.org/json/appSiteCam/'
 suff = '.jpg'
 for key,value in response.items():
  if value.get('isDiscontinued') == False:
   tings = [base + value.get('abbr')+ suff, value.get('lat'),value.get('lon'),value.get('name'),'true','normal','10','https://www.kymesonet.org/']
   sql = 'INSERT INTO cams (url, lat, lon, description, active, method, interval,source) VALUES (%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING;'
#   print(tings)
  cursor.execute(sql,tings)
  conn.commit()

kymeso()


def ky511cams():
    """
    Fetches traffic camera data, filters it, and performs an efficient
    bulk insert into the database.
    """
    # Define constants for clarity and easy maintenance
    API_URL = 'https://services2.arcgis.com/CcI36Pduqd0OR4W9/ArcGIS/rest/services/trafficCamerasCur_Prd/FeatureServer/0/query?where=1%3D1&outFields=name%2C+description%2C+snapshot%2C+status%2C+latitude%2C+longitude%2C+county&returnGeometry=false&f=pjson'
    
    # Define the SQL template with placeholders for the values
    SQL_INSERT = """
        INSERT INTO cams (description, lat, lon, url, active, method, interval, source)
        VALUES %s
        ON CONFLICT DO NOTHING;
    """

    try:
        # 1. Use .json() for efficiency and add a timeout for safety
        response = requests.get(API_URL, timeout=15)
        response.raise_for_status()  # Raise an exception for bad status codes (4xx or 5xx)
        data = response.json()
    except requests.exceptions.RequestException as e:
        print(f"Error fetching data from API: {e}")
        return
    except json.JSONDecodeError:
        print("Error: Failed to decode JSON from response.")
        return

    # 2. Use a list comprehension to process and filter data concisely
    cams_to_insert = [
        (
            cam.get('description'), 
            cam.get('latitude'), 
            cam.get('longitude'), 
            cam.get('snapshot'), 
            True,           # active
            'normalproxy',  # method
            10,             # interval
            'https://goky.ky.gov/'         # source
        )
        for feature in data.get('features', [])
        if (cam := feature.get('attributes')) and cam.get('longitude') > -83.8
    ]

    if not cams_to_insert:
        print("No new camera data meeting the criteria to insert.")
        return

    try:
        # 3. Use a single, highly efficient bulk insert operation
        execute_values(cursor, SQL_INSERT, cams_to_insert)
        conn.commit()
        print(f"✅ Successfully processed and inserted {len(cams_to_insert)} records.")
    except Exception as e:
        print(f"❌ Database error occurred: {e}")
        conn.rollback() # Roll back the transaction on error
  
  
ky511cams()  



#bloomsky()



headers = {
        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
    }





def get_elev(lon, lat):
    """Gets elevation from the local database first, then falls back to a web API."""
    try:
        # Primary method: Local PostGIS database
        sql = """
            SELECT round(ST_Value(rast, ST_SetSRID(ST_MakePoint(%s,%s),4326)) * 3.28084) AS val 
            FROM dem 
            WHERE ST_Intersects(rast, ST_SetSRID(ST_MakePoint(%s,%s),4326));
        """
        vars_tuple = (lon, lat, lon, lat)
        cursor.execute(sql, vars_tuple)
        result = cursor.fetchone()
        
        # Check if the database returned a valid, non-zero elevation
        if result and result[0] is not None and result[0] != 0:
            return int(result[0])
            
    except Exception as e:
        # Log the database error if needed
        print(f"Database error: {e}")

    # Fallback method: Web API
    # This part runs if the try block fails or returns no valid data
    elev = get_elev_backup(lon, lat)
    return int(elev) if elev is not None else None


def get_elev_backup(lon, lat):
    """Backup function to get elevation from the National Map API."""
    baseurl = 'https://epqs.nationalmap.gov/v1/json'
    params = {
        'x': lon,
        'y': lat,
        'units': 'Feet',
        'wkid': 4326,
        'includeDate': False
    }
    print(params)
    try:
        # The timeout prevents the request from hanging indefinitely
        r = requests.get(baseurl, params=params, timeout=5)
        # Raise an exception for bad status codes (4xx or 5xx)
        r.raise_for_status() 
        
        # Attempt to parse JSON and get the value
        data = r.json()
        elev = data.get('value')
        
        # The API may return a string 'no-data' for some locations
        if isinstance(elev, str) and not elev.replace('.', '', 1).isdigit():
            return None
        
        # Add a small delay to avoid hammering the API
        time.sleep(1) 
        return float(elev)

    except requests.exceptions.RequestException as e:
        # Handles connection errors, timeouts, bad status codes, etc.
        print(r.content)
        print(f"API Request failed: {e}")
        return None
    except JSONDecodeError:
        # Handles cases where the response is not valid JSON
        print(f"Failed to decode JSON from response. Response text: {r.text}")
        return None






cursor.execute("SELECT camid, lon, lat from cams WHERE elevation ISNULL or elevation = 0 and active = true")
allcams = cursor.fetchall()



for cam in allcams:
 camid, lon, lat = cam
 elev = get_elev(lon,lat)
 val = (elev,camid)
 sql = "UPDATE cams SET elevation= %s WHERE camid = %s"
 cursor.execute(sql,val)
 conn.commit()

updates = ['UPDATE public.cams SET geom = ST_SetSRID(ST_MakePoint(lon, lat), 4326)',
'UPDATE public.cams SET county = county.countyname from public.county WHERE ST_Contains(county.geom,cams.geom)',
'UPDATE public.cams SET pzone = pzone.state_zone from public.pzone WHERE ST_Contains(pzone.geom,cams.geom)',
'UPDATE public.cams SET fzone = fzone.state_zone from public.fzone WHERE ST_Contains(fzone.geom,cams.geom)',
'UPDATE public.cams SET cwa = fzone.cwa from public.fzone WHERE ST_Contains(fzone.geom,cams.geom)',
'UPDATE public.cams SET zonename = pzone.shortname from public.pzone WHERE ST_Contains(pzone.geom,cams.geom)',
'UPDATE public.cams SET keephours = 240 WHERE keephours is null',
"""UPDATE public.cams SET method = 'normal' WHERE method is null""",
'UPDATE public.cams SET state = county.state from public.county WHERE ST_Contains(county.geom,cams.geom)',
'UPDATE public.cams SET airport = EXISTS (SELECT 1 FROM public.airports WHERE ST_Intersects(cams.geom, airports.geom5)) WHERE airport IS NULL',
"UPDATE cams SET active = false FROM cwa WHERE cwa.cwa = 'RLX' AND NOT ST_DWithin(cams.geom::geography, cwa.geom::geography, 10 * 1609.344) AND cams.url ~ 'oh.us' AND cams.active = true",]

for i in updates:
 cursor.execute(i)



conn.commit()





ambient()


cursor.close()
conn.close()