229 lines
9.2 KiB
Python
229 lines
9.2 KiB
Python
import time
|
|
import json
|
|
import psycopg2
|
|
import psycopg2.extensions
|
|
from psycopg2.extras import Json
|
|
import re
|
|
import pandas as pd
|
|
import requests
|
|
import xmltodict
|
|
import datetime
|
|
from html import escape
|
|
from tabulate import tabulate
|
|
|
|
def clean_text(text):
|
|
"""Clean text by removing problematic characters that cause encoding issues."""
|
|
if text is None:
|
|
return ''
|
|
|
|
# Convert to string if not already
|
|
text = str(text)
|
|
|
|
# Remove the problematic 'Â' character that appears with degree symbols
|
|
text = text.replace('\u00a0', ' ') # Non-breaking space to regular space
|
|
text = text.replace('\xc2', '') # Remove standalone  character
|
|
text = text.replace('°', '°') # Fix degree symbol encoding issue
|
|
text = text.replace('°F', '°F') # Fix degree F encoding issue
|
|
text = text.replace('°C', '°C') # Fix degree C encoding issue
|
|
text = text.replace('\u00a0', ' ') # Remove non-breaking spaces
|
|
text = text.encode('utf-8', errors='ignore').decode('utf-8') # Handle other encoding issues
|
|
|
|
return text
|
|
|
|
allobs = []
|
|
|
|
states = ['wv', 'oh', 'va', 'ky']
|
|
ohcounties = ['-LW-', '-GL-', '-JC-', '-MS-', '-AT-', '-PY-', '-WS-', '-MG-', '-VN-']
|
|
vacounties = ['-DC-', '-BC-']
|
|
kycounties = ['-LR-', '-CT-', '-GP-', '-BD-']
|
|
datewanted = datetime.date.today().strftime("%m/%d/%Y")
|
|
|
|
try:
|
|
for state in states:
|
|
url = f'https://data.cocorahs.org/export/exportreports.aspx?state={state}&Format=XML&Date={datewanted}&responsefields=all'
|
|
response = requests.get(url)
|
|
response.raise_for_status() # Check for HTTP errors
|
|
data = xmltodict.parse(response.content.decode('utf-8')) # Explicitly decode as UTF-8
|
|
|
|
try:
|
|
daily_reports = data.get('Cocorahs', {}).get('DailyPrecipReports')
|
|
if daily_reports is None:
|
|
print(f"No reports found for state {state}")
|
|
continue
|
|
reports = daily_reports.get('DailyPrecipReport')
|
|
if reports is None:
|
|
print(f"No reports data found for state {state}")
|
|
continue
|
|
# Handle case where reports might be a single dict or a list
|
|
if isinstance(reports, dict):
|
|
reports = [reports]
|
|
|
|
for report in reports:
|
|
if state == 'wv':
|
|
allobs.append(report)
|
|
else:
|
|
for county in eval(state + 'counties'):
|
|
station_number = report.get('StationNumber', '')
|
|
if county in station_number:
|
|
allobs.append(report)
|
|
except (KeyError, TypeError) as e:
|
|
print(f"Error processing data for state {state}: {e}")
|
|
continue
|
|
|
|
# Process observations
|
|
finalobs = []
|
|
for obs in allobs:
|
|
tempob = [
|
|
clean_text(obs.get('DateTimeStamp', '')),
|
|
clean_text(obs.get('StationNumber', '')),
|
|
clean_text(obs.get('StationName', '')),
|
|
clean_text(obs.get('TotalPrecipAmt', '')),
|
|
clean_text(obs.get('NewSnowDepth', '')),
|
|
clean_text(obs.get('TotalSnowDepth', '')),
|
|
clean_text(obs.get('Notes', ''))
|
|
]
|
|
finalobs.append(tempob)
|
|
|
|
# Write to file with UTF-8 encoding
|
|
with open('/var/www/html/work/today.txt', 'w', encoding='utf-8') as f:
|
|
f.write(tabulate(
|
|
finalobs,
|
|
headers=["Date/Time of Ob (Z)", "Station Number", "Station Name",
|
|
"New Precip", "New Snow", "Snow Depth", "Comments"],
|
|
tablefmt='plain' # Changed to 'plain' for simpler text output
|
|
))
|
|
|
|
# Write HTML table to today.html
|
|
html_content = """<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<title>Cocorahs Weather Data - """ + datewanted + """</title>
|
|
<style>
|
|
body { font-family: Arial, sans-serif; margin: 20px; }
|
|
table { border-collapse: collapse; width: 100%; }
|
|
th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }
|
|
th { background-color: #f2f2f2; font-weight: bold; cursor: pointer; }
|
|
th:hover { background-color: #e0e0e0; }
|
|
tr:nth-child(even) { background-color: #f9f9f9; }
|
|
.sort-arrow { margin-left: 5px; font-size: 0.8em; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>Cocorahs Weather Data - """ + datewanted + """</h1>
|
|
<table id="dataTable">
|
|
<thead>
|
|
<tr>
|
|
<th onclick="sortTable(0)" title="Click to sort by Date/Time">Date/Time of Ob (Z)<span class="sort-arrow"></span></th>
|
|
<th onclick="sortTable(1)" title="Click to sort by Station Number">Station Number<span class="sort-arrow"></span></th>
|
|
<th onclick="sortTable(2)" title="Click to sort by Station Name">Station Name<span class="sort-arrow"></span></th>
|
|
<th onclick="sortTable(3)" title="Click to sort by New Precip">New Precip<span class="sort-arrow"></span></th>
|
|
<th onclick="sortTable(4)" title="Click to sort by New Snow">New Snow<span class="sort-arrow"></span></th>
|
|
<th onclick="sortTable(5)" title="Click to sort by Snow Depth">Snow Depth<span class="sort-arrow"></span></th>
|
|
<th onclick="sortTable(6)" title="Click to sort by Comments">Comments<span class="sort-arrow"></span></th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
"""
|
|
|
|
for row in finalobs:
|
|
html_content += " <tr>\n"
|
|
for cell in row:
|
|
html_content += f" <td>{escape(str(cell))}</td>\n"
|
|
html_content += " </tr>\n"
|
|
|
|
html_content += """ </tbody>
|
|
</table>
|
|
|
|
<script>
|
|
let sortDirection = Array(7).fill(true); // true for ascending, false for descending for 7 columns
|
|
|
|
function sortTable(columnIndex) {
|
|
const table = document.getElementById('dataTable');
|
|
const tbody = table.querySelector('tbody');
|
|
const rows = Array.from(tbody.rows);
|
|
|
|
// Determine the data type for this column
|
|
let isNumeric = [3, 4, 5].includes(columnIndex); // New Precip, New Snow, Snow Depth are numeric
|
|
let isDateTime = columnIndex === 0; // Date/Time is special
|
|
|
|
// Toggle sort direction
|
|
sortDirection[columnIndex] = !sortDirection[columnIndex];
|
|
const direction = sortDirection[columnIndex];
|
|
|
|
rows.sort((a, b) => {
|
|
const aCell = a.cells[columnIndex].textContent.trim();
|
|
const bCell = b.cells[columnIndex].textContent.trim();
|
|
|
|
let comparison = 0;
|
|
|
|
if (isDateTime) {
|
|
// Parse date for comparison (assuming format like: MM/DD/YYYY HH:MM:SS)
|
|
const dateA = parseDateTime(aCell);
|
|
const dateB = parseDateTime(bCell);
|
|
comparison = dateA - dateB;
|
|
} else if (isNumeric) {
|
|
// Convert to numeric values for comparison
|
|
const numA = parseFloat(aCell) || 0;
|
|
const numB = parseFloat(bCell) || 0;
|
|
comparison = numA - numB;
|
|
} else {
|
|
// Text comparison for A-Z/Z-A
|
|
comparison = aCell.localeCompare(bCell, undefined, {numeric: true, sensitivity: 'base'});
|
|
}
|
|
|
|
return direction ? comparison : -comparison;
|
|
});
|
|
|
|
// Reorder the rows in the table
|
|
rows.forEach(row => tbody.appendChild(row));
|
|
|
|
// Update sort arrows
|
|
updateSortArrows(columnIndex, direction);
|
|
}
|
|
|
|
function parseDateTime(dateString) {
|
|
// Handle various date formats, default to string if not parseable
|
|
if (!dateString || dateString.trim() === '') {
|
|
return 0;
|
|
}
|
|
|
|
// Try to parse date in common formats
|
|
let date = new Date(dateString);
|
|
if (isNaN(date)) {
|
|
// Try alternative parsing if the default doesn't work
|
|
const parts = dateString.replace(',', '').split(/[\s\/:]+/);
|
|
if (parts.length >= 3) {
|
|
// Assuming MM/DD/YYYY format
|
|
date = new Date(parts[2], parts[0] - 1, parts[1]);
|
|
}
|
|
}
|
|
return date.getTime ? date.getTime() : 0;
|
|
}
|
|
|
|
function updateSortArrows(columnIndex, direction) {
|
|
// Clear all arrows first
|
|
const allArrows = document.querySelectorAll('.sort-arrow');
|
|
allArrows.forEach(arrow => arrow.textContent = '');
|
|
|
|
// Set arrow for the current column
|
|
const currentArrow = document.querySelectorAll('.sort-arrow')[columnIndex];
|
|
currentArrow.textContent = direction ? ' ▲' : ' ▼';
|
|
}
|
|
|
|
// Initialize arrows on load
|
|
document.addEventListener('DOMContentLoaded', function() {
|
|
const arrows = document.querySelectorAll('.sort-arrow');
|
|
arrows.forEach(arrow => arrow.textContent = '');
|
|
});
|
|
</script>
|
|
</body>
|
|
</html>"""
|
|
|
|
with open('/var/www/html/work/today.html', 'w', encoding='utf-8') as f:
|
|
f.write(html_content)
|
|
|
|
except requests.RequestException as e:
|
|
print(f"Error fetching data: {e}")
|
|
except Exception as e:
|
|
print(f"Unexpected error: {e}") |