import time import requests import json import geojson import psycopg2 import psycopg2.extensions from psycopg2.extras import Json conn = psycopg2.connect(host='localhost', database='nws', user='nws', password='nws') cursor = conn.cursor() #one rain sensor database with last obs and metadata #one rain sensor database with all obs #metsensor = [30,53,11,10,50,44,40,41] metsensor = [(3,'tempf'),(2,'raintotal'),(8,'humidity'),(7,'winddir'),(4,'windspd'),(5,'windgust')] rainsensor = [(2,'raintotal')] features = [] alldata = [] S = requests.Session() cursor.execute("SELECT siteid, sitetype from onerainsites") #WHERE (Active = True) and ((EXTRACT(EPOCH FROM (current_timestamp - lastob ))/60) > 8 or (lastob ISNULL))" allonerainsites = cursor.fetchall() def getData(siteid,sensorid): apiget = 'https://wvdhsem.onerain.com/export/flot/?method=sensor&site_id=' + str(siteid) + '&device_id=' + str(sensorid) #print(apiget) dataresponse = json.loads(S.get(apiget).text) #print(dataresponse) return dataresponse # sql = 'INSERT INTO wuobs (stationid, dateutc, winddir, windspd, windgust, tempf, dewpt, humidity, pressure, preciptotal) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (stationid,dateutc) DO NOTHING' # sql2 = 'UPDATE wusites SET lastob = %s, winddir = %s, windspd = %s, windgust= %s, tempf = %s, dewpt = %s, humidity = %s, pressure = %s, preciptotal = %s WHERE stationid = %s' # cursor.execute(sql,ob) # cursor.execute(sql2,ob2) # conn.commit() # setDerived(stationid) def setDerived(stationid): data1 = (stationid,) data = (stationid,stationid) ago24 = "UPDATE wusites set ago24 = subquery.preciptotal, ago24time = subquery.dateutc from (select preciptotal,stationid,dateutc from wuobs WHERE (dateutc < timezone('utc',current_timestamp) - interval '23 hours') and (dateutc > timezone('utc',current_timestamp) - interval '24 hours') and stationid = %s order by dateutc desc limit 1) as subquery where wusites.stationid = %s" ago3 = "UPDATE wusites set ago3 = subquery.preciptotal, ago3time = subquery.dateutc from (select preciptotal,stationid,dateutc from wuobs WHERE (dateutc < timezone('utc',current_timestamp) - interval '2.5 hours') and (dateutc > timezone('utc',current_timestamp) - interval '3 hours') and stationid = %s order by dateutc desc limit 1) as subquery where wusites.stationid = %s" ago6 = "UPDATE wusites set ago6 = subquery.preciptotal, ago6time = subquery.dateutc from (select preciptotal,stationid,dateutc from wuobs WHERE (dateutc < timezone('utc',current_timestamp) - interval '5.2 hours') and (dateutc > timezone('utc',current_timestamp) - interval '6 hours') and stationid = %s order by dateutc desc limit 1) as subquery where wusites.stationid = %s" midnight = "UPDATE wusites set lastmidnight = subquery.preciptotal from (select preciptotal,stationid from wuobs WHERE (dateutc < timezone('UTC', current_date::timestamp at time zone 'US/Eastern')) and (dateutc > timezone('UTC', current_date::timestamp at time zone 'US/Eastern' - interval '40 minutes')) and stationid = %s order by dateutc desc limit 1) as subquery where wusites.stationid = %s" windmax = "UPDATE wusites set windmax = subquery.windmax from (SELECT MAX(windgust) as windmax FROM wuobs where (dateutc > timezone('utc',current_timestamp) - interval '24 hours') and stationid = %s) as subquery where wusites.stationid = %s" maxt = "UPDATE wusites set maxt = subquery.maxt from (SELECT MAX(tempf) as maxt FROM wuobs where (dateutc > timezone('utc',current_timestamp) - interval '24 hours') and stationid = %s) as subquery where wusites.stationid = %s" mint = "UPDATE wusites set mint = subquery.mint from (SELECT MIN(tempf) as mint FROM wuobs where (dateutc > timezone('utc',current_timestamp) - interval '24 hours') and stationid = %s) as subquery where wusites.stationid = %s" cursor.execute(ago24,data) cursor.execute(ago6,data) cursor.execute(ago3,data) cursor.execute(windmax,data) cursor.execute(midnight,data) cursor.execute(maxt,data) cursor.execute(mint,data) conn.commit() rain3 = "update wusites set rain3 = (case when (wusites.ago3time < timezone('UTC', current_date::timestamp at time zone 'US/Eastern') and wusites.stationid = %s) then (wusites.preciptotal + wusites.lastmidnight - wusites.ago3) when (wusites.ago3time > timezone('UTC', current_date::timestamp at time zone 'US/Eastern') and wusites.stationid = %s) then (wusites.preciptotal - wusites.ago3) end) where wusites.stationid = %s" rain6 = "update wusites set rain6 = (case when (wusites.ago6time < timezone('UTC', current_date::timestamp at time zone 'US/Eastern') and wusites.stationid = %s) then (wusites.preciptotal + wusites.lastmidnight - wusites.ago6) when (wusites.ago6time > timezone('UTC', current_date::timestamp at time zone 'US/Eastern') and wusites.stationid = %s) then (wusites.preciptotal - wusites.ago6) end) where wusites.stationid = %s" rain24 = "update wusites set rain24 = (wusites.preciptotal + wusites.lastmidnight - wusites.ago24) where wusites.stationid = %s" data2 = (stationid,stationid,stationid) cursor.execute(rain3,data2) cursor.execute(rain6,data2) cursor.execute(rain24,data1) conn.commit() for i in allonerainsites: if i[1] == 'MET': for j in metsensor: tempdata = getData(i[0],j[0]) for p in tempdata['data']: datafield = p[1] datefield = int(p[0])/1000 ob = (i[0],datefield,int(j[0]),datafield) sql = 'INSERT INTO onerainobs (siteid, dateutc, sensorid, data1) VALUES (%s,to_timestamp(%s),%s,%s) ON CONFLICT (siteid,sensorid,dateutc,data1) DO NOTHING' cursor.execute(sql,ob) conn.commit() if i[1] == 'Rain': for j in rainsensor: tempdata = getData(i[0],j[0]) for p in tempdata['data']: datafield = p[1] datefield = int(p[0])/1000 ob = (i[0],datefield,int(j[0]),datafield) sql = 'INSERT INTO onerainobs (siteid, dateutc, sensorid, data1) VALUES (%s,to_timestamp(%s),%s,%s) ON CONFLICT (siteid,sensorid,dateutc,data1) DO NOTHING' cursor.execute(sql,ob) conn.commit() for i in allonerainsites: if i[1] == 'MET': for j in metsensor: sql = 'SELECT data1,dateutc from onerainobs where siteid = %s and sensorid = %s order by dateutc desc limit 1' getob = (i[0],j[0]) cursor.execute(sql,getob) #WHERE (Active = True) and ((EXTRACT(EPOCH FROM (current_timestamp - lastob ))/60) > 8 or (lastob ISNULL))" obdata = cursor.fetchall() if len(obdata) == 1: for l in obdata: sensor = str(j[1]) sensortime = str(j[1])+'time' sql = 'UPDATE onerainsites set ' + sensor + ' = %s, ' + sensortime + ' = %s where siteid = %s' updateob = (l[0],l[1],i[0]) cursor.execute(sql,updateob) conn.commit() if i[1] == 'Rain': for j in rainsensor: sql = 'SELECT data1,dateutc from onerainobs where siteid = %s and sensorid = %s order by dateutc desc limit 1' getob = (i[0],j[0]) cursor.execute(sql,getob) #WHERE (Active = True) and ((EXTRACT(EPOCH FROM (current_timestamp - lastob ))/60) > 8 or (lastob ISNULL))" obdata = cursor.fetchall() if len(obdata) == 1: for l in obdata: sensor = str(j[1]) sensortime = str(j[1])+'time' sql = 'UPDATE onerainsites set ' + sensor + ' = %s, ' + sensortime + ' = %s where siteid = %s' updateob = (l[0],l[1],i[0]) cursor.execute(sql,updateob) conn.commit()