126 lines
5.8 KiB
Python
126 lines
5.8 KiB
Python
import requests
|
|
import json
|
|
import psycopg2
|
|
from datetime import datetime
|
|
from requests.packages.urllib3.exceptions import InsecureRequestWarning
|
|
|
|
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
|
|
|
|
# Database connection details (keep these secure and configurable)
|
|
DB_HOST = 'localhost'
|
|
DB_DATABASE = 'nws'
|
|
DB_USER = 'nws'
|
|
DB_PASSWORD = 'nws'
|
|
|
|
# API URL
|
|
LIVING_ATLAS_URL = "https://services9.arcgis.com/RHVPKKiFTONKtxq3/ArcGIS/rest/services/USA_Wildfires_v1/FeatureServer/0/query?where=1%3D1&geometry=%7B%22spatialReference%22:%7B%22latestWkid%22:3857,%22wkid%22:102100%7D,%22xmin%22:-9222738.841522107,%22ymin%22:4457648.21239309,%22xmax%22:-9009938.154776277,%22ymax%22:4723649.070825376%7D&geometryType=esriGeometryEnvelope&spatialRelationship=intersects&inSR=3857&returnGeometry=true&returnQueryGeometry=true&outFields=IrwinID,IncidentName,POOState,ModifiedOnDateTime,FireDiscoveryDateTime,FireDiscoveryAge,IncidentTypeCategory,CalculatedAcres,DailyAcres,DiscoveryAcres,PercentContained,TotalIncidentPersonnel&f=json"
|
|
|
|
|
|
S = requests.Session()
|
|
S.verify = False # Be cautious about disabling SSL verification in production
|
|
|
|
|
|
def livingatlas(url):
|
|
try:
|
|
result = S.get(url, timeout=10) # Added timeout to prevent indefinite hanging
|
|
result.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
|
|
data = result.json() # Use .json() directly, it handles json.loads() and error handling
|
|
|
|
conn = None # Initialize conn outside the loop for broader scope
|
|
cursor = None
|
|
try:
|
|
conn = psycopg2.connect(host=DB_HOST, database=DB_DATABASE, user=DB_USER, password=DB_PASSWORD)
|
|
cursor = conn.cursor()
|
|
|
|
for feature in data.get('features', []): # Safely access features
|
|
attributes = feature.get('attributes', {})
|
|
geometry = feature.get('geometry', {})
|
|
|
|
incid = attributes.get('IrwinID')
|
|
incname = attributes.get('IncidentName')
|
|
state = attributes.get('POOState')
|
|
modified_timestamp = attributes.get('ModifiedOnDateTime')
|
|
discoverytime_timestamp = attributes.get('FireDiscoveryDateTime')
|
|
discoveryage = attributes.get('FireDiscoveryAge')
|
|
inctype = attributes.get('IncidentTypeCategory')
|
|
calcacres = attributes.get('CalculatedAcres')
|
|
dailyacres = attributes.get('DailyAcres')
|
|
discoveryacres = attributes.get('DiscoveryAcres')
|
|
contained = attributes.get('PercentContained')
|
|
personnel = attributes.get('TotalIncidentPersonnel')
|
|
lat = geometry.get('y')
|
|
lon = geometry.get('x')
|
|
|
|
discoverytime = datetime.fromtimestamp(discoverytime_timestamp/1000) if discoverytime_timestamp else None
|
|
modified = datetime.fromtimestamp(modified_timestamp/1000) if modified_timestamp else None
|
|
|
|
print(incid, incname, state, modified, discoverytime, discoveryage, inctype, calcacres, dailyacres, discoveryacres, contained, personnel, lat, lon)
|
|
|
|
|
|
sql_insert = """
|
|
INSERT INTO fire (
|
|
incid, incname, state, modified, discovery, age, type,
|
|
calcacres, dailyacres, discoveryacres, contained, personnel, lat, lon
|
|
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
ON CONFLICT (incid) DO UPDATE SET
|
|
incname = %s,
|
|
state = %s,
|
|
modified = %s,
|
|
discovery = %s,
|
|
age = %s,
|
|
type = %s,
|
|
calcacres = %s,
|
|
dailyacres = %s,
|
|
discoveryacres = %s,
|
|
contained = %s,
|
|
personnel = %s,
|
|
lat = %s,
|
|
lon = %s
|
|
"""
|
|
vals = (
|
|
incid, incname, state, modified, discoverytime, discoveryage, inctype,
|
|
calcacres, dailyacres, discoveryacres, contained, personnel, lat, lon,
|
|
incname, state, modified, discoverytime, discoveryage, inctype,
|
|
calcacres, dailyacres, discoveryacres, contained, personnel, lat, lon
|
|
)
|
|
|
|
cursor.execute(sql_insert, vals)
|
|
conn.commit() # Commit after each successful insert/update
|
|
|
|
except psycopg2.Error as db_error:
|
|
if conn:
|
|
conn.rollback() # Rollback transaction on error
|
|
print(f"Database error: {db_error}")
|
|
finally:
|
|
if cursor:
|
|
cursor.close()
|
|
if conn:
|
|
conn.close()
|
|
|
|
except requests.exceptions.RequestException as req_error: # Catch broader request exceptions
|
|
print(f"API Request error: {req_error}")
|
|
except json.JSONDecodeError as json_error:
|
|
print(f"JSON Decode error: {json_error}. Response text was: {result.text if 'result' in locals() else 'No response received'}") # More informative JSON error
|
|
|
|
|
|
livingatlas(LIVING_ATLAS_URL)
|
|
|
|
|
|
conn = None # Re-initialize conn for the geometry update outside the livingatlas function
|
|
cursor = None
|
|
try:
|
|
conn = psycopg2.connect(host=DB_HOST, database=DB_DATABASE, user=DB_USER, password=DB_PASSWORD)
|
|
cursor = conn.cursor()
|
|
cursor.execute("UPDATE public.fire SET geom = ST_SetSRID(ST_MakePoint(lon, lat), 4326) WHERE (lat IS NOT NULL AND lon IS NOT NULL AND geom IS NULL)")
|
|
conn.commit()
|
|
except psycopg2.Error as db_error:
|
|
if conn:
|
|
conn.rollback()
|
|
print(f"Database error during geometry update: {db_error}")
|
|
finally:
|
|
if cursor:
|
|
cursor.close()
|
|
if conn:
|
|
conn.close()
|
|
|
|
print("Script execution completed.") |