(.*?) nlat and lon < xlon and lon > nlon): tings = [bloomcameras[p]['img'], bloomcameras[p]['city'],bloomcameras[p]['name'],lat,lon,p,'10','True','240','bloomsky','https://map.bloomsky.com/weather-stations/' + str(p)] sql = "INSERT INTO cams (url,town,description,lat,lon,bloomsky,interval,active,keephours, method,source) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" cursor.execute(sql,tings) conn.commit() def ambient(): url = "https://lightning.ambientweather.net/devices?%24publicBox[0][0]=-83.144531&%24publicBox[0][1]=36.72&%24publicBox[1][0]=-75.5647&%24publicBox[1][1]=40.33802332110745&%24limit=1000&rank=2" r = requests.get(url) x = r.json() for i in x['data']: if (i.get('info').get('webcam') != None and "jpg" in i.get('info').get('webcam')): description = i.get('info').get('coords').get('location') lat = i.get('info').get('coords').get('coords').get('lat') lon = i.get('info').get('coords').get('coords').get('lon') url = i.get('info').get('webcam') try: tings = [description,lat,lon,url,'normal','true','10','https://ambientweather.net/'] sql = 'INSERT INTO cams (description, lat, lon, url, method, active, interval,source) VALUES (%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING;' cursor.execute(sql,tings) except Exception as e: print(e) conn.commit() ambient() def kymeso(): ky = 'https://www.kymesonet.org/json/StationsInfo.json' response = json.loads(requests.get(ky).text) tings = [] base = 'https://www.kymesonet.org/json/appSiteCam/' suff = '.jpg' for key,value in response.items(): if value.get('isDiscontinued') == False: tings = [base + value.get('abbr')+ suff, value.get('lat'),value.get('lon'),value.get('name'),'true','normal','10','https://www.kymesonet.org/'] sql = 'INSERT INTO cams (url, lat, lon, description, active, method, interval,source) VALUES (%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING;' # print(tings) cursor.execute(sql,tings) conn.commit() kymeso() def ky511cams(): """ Fetches traffic camera data, filters it, and performs an efficient bulk insert into the database. """ # Define constants for clarity and easy maintenance API_URL = 'https://services2.arcgis.com/CcI36Pduqd0OR4W9/ArcGIS/rest/services/trafficCamerasCur_Prd/FeatureServer/0/query?where=1%3D1&outFields=name%2C+description%2C+snapshot%2C+status%2C+latitude%2C+longitude%2C+county&returnGeometry=false&f=pjson' # Define the SQL template with placeholders for the values SQL_INSERT = """ INSERT INTO cams (description, lat, lon, url, active, method, interval, source) VALUES %s ON CONFLICT DO NOTHING; """ try: # 1. Use .json() for efficiency and add a timeout for safety response = requests.get(API_URL, timeout=15) response.raise_for_status() # Raise an exception for bad status codes (4xx or 5xx) data = response.json() except requests.exceptions.RequestException as e: print(f"Error fetching data from API: {e}") return except json.JSONDecodeError: print("Error: Failed to decode JSON from response.") return # 2. Use a list comprehension to process and filter data concisely cams_to_insert = [ ( cam.get('description'), cam.get('latitude'), cam.get('longitude'), cam.get('snapshot'), True, # active 'normalproxy', # method 10, # interval 'https://goky.ky.gov/' # source ) for feature in data.get('features', []) if (cam := feature.get('attributes')) and cam.get('longitude') > -83.8 ] if not cams_to_insert: print("No new camera data meeting the criteria to insert.") return try: # 3. Use a single, highly efficient bulk insert operation execute_values(cursor, SQL_INSERT, cams_to_insert) conn.commit() print(f"✅ Successfully processed and inserted {len(cams_to_insert)} records.") except Exception as e: print(f"❌ Database error occurred: {e}") conn.rollback() # Roll back the transaction on error ky511cams() #bloomsky() headers = { 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36', } def get_elev(lon, lat): """Gets elevation from the local database first, then falls back to a web API.""" try: # Primary method: Local PostGIS database sql = """ SELECT round(ST_Value(rast, ST_SetSRID(ST_MakePoint(%s,%s),4326)) * 3.28084) AS val FROM dem WHERE ST_Intersects(rast, ST_SetSRID(ST_MakePoint(%s,%s),4326)); """ vars_tuple = (lon, lat, lon, lat) cursor.execute(sql, vars_tuple) result = cursor.fetchone() # Check if the database returned a valid, non-zero elevation if result and result[0] is not None and result[0] != 0: return int(result[0]) except Exception as e: # Log the database error if needed print(f"Database error: {e}") # Fallback method: Web API # This part runs if the try block fails or returns no valid data elev = get_elev_backup(lon, lat) return int(elev) if elev is not None else None def get_elev_backup(lon, lat): """Backup function to get elevation from the National Map API.""" baseurl = 'https://epqs.nationalmap.gov/v1/json' params = { 'x': lon, 'y': lat, 'units': 'Feet', 'wkid': 4326, 'includeDate': False } print(params) try: # The timeout prevents the request from hanging indefinitely r = requests.get(baseurl, params=params, timeout=5) # Raise an exception for bad status codes (4xx or 5xx) r.raise_for_status() # Attempt to parse JSON and get the value data = r.json() elev = data.get('value') # The API may return a string 'no-data' for some locations if isinstance(elev, str) and not elev.replace('.', '', 1).isdigit(): return None # Add a small delay to avoid hammering the API time.sleep(1) return float(elev) except requests.exceptions.RequestException as e: # Handles connection errors, timeouts, bad status codes, etc. print(r.content) print(f"API Request failed: {e}") return None except JSONDecodeError: # Handles cases where the response is not valid JSON print(f"Failed to decode JSON from response. Response text: {r.text}") return None cursor.execute("SELECT camid, lon, lat from cams WHERE elevation ISNULL or elevation = 0 and active = true") allcams = cursor.fetchall() for cam in allcams: camid, lon, lat = cam elev = get_elev(lon,lat) val = (elev,camid) sql = "UPDATE cams SET elevation= %s WHERE camid = %s" cursor.execute(sql,val) conn.commit() updates = ['UPDATE public.cams SET geom = ST_SetSRID(ST_MakePoint(lon, lat), 4326)', 'UPDATE public.cams SET county = county.countyname from public.county WHERE ST_Contains(county.geom,cams.geom)', 'UPDATE public.cams SET pzone = pzone.state_zone from public.pzone WHERE ST_Contains(pzone.geom,cams.geom)', 'UPDATE public.cams SET fzone = fzone.state_zone from public.fzone WHERE ST_Contains(fzone.geom,cams.geom)', 'UPDATE public.cams SET cwa = fzone.cwa from public.fzone WHERE ST_Contains(fzone.geom,cams.geom)', 'UPDATE public.cams SET zonename = pzone.shortname from public.pzone WHERE ST_Contains(pzone.geom,cams.geom)', 'UPDATE public.cams SET keephours = 240 WHERE keephours is null', """UPDATE public.cams SET method = 'normal' WHERE method is null""", 'UPDATE public.cams SET state = county.state from public.county WHERE ST_Contains(county.geom,cams.geom)', 'UPDATE public.cams SET airport = EXISTS (SELECT 1 FROM public.airports WHERE ST_Intersects(cams.geom, airports.geom5)) WHERE airport IS NULL', "UPDATE cams SET active = false FROM cwa WHERE cwa.cwa = 'RLX' AND NOT ST_DWithin(cams.geom::geography, cwa.geom::geography, 10 * 1609.344) AND cams.url ~ 'oh.us' AND cams.active = true",] for i in updates: cursor.execute(i) conn.commit() ambient() cursor.close() conn.close()