#!/usr/bin/env python3
import json
import pandas as pd
from datetime import datetime
import re
import sys
import warnings
import psycopg2
import traceback
import os
import time
from google.oauth2 import service_account
from googleapiclient.discovery import build
from googleapiclient.http import MediaIoBaseDownload
import io

# Configuration
SCOPES = ['https://www.googleapis.com/auth/drive']  # Full access for sync and delete
SERVICE_ACCOUNT_FILE = '/var/www/html/work/noaa_staff.json'  # Path to your service account JSON key
DRIVE_FOLDER_ID = '1xCPU7Lhy-2cTg2Ul6tSQt6iRZeBGH3AW'  # Replace with your Google Drive folder ID
LOCAL_DIR = os.path.expanduser('/var/www/html/work/NOAA')
USER_EMAIL = 'stoat@stoat.org'


conn = psycopg2.connect(host='localhost', database='nws', user='nws', password='nws')
cursor = conn.cursor()

def get_drive_service():
    credentials = service_account.Credentials.from_service_account_file(
        SERVICE_ACCOUNT_FILE, scopes=SCOPES)
    credentials =  credentials.with_subject(USER_EMAIL)  # Impersonate your account  
    return build('drive', 'v3', credentials=credentials)

def get_folder_files(service, folder_id):
    query = f"'{folder_id}' in parents and trashed=false"
    results = service.files().list(q=query, fields="files(id, name, mimeType, modifiedTime, size)").execute()
    return results.get('files', [])

def download_file(service, file_id, file_name, local_path, modified_time):
    request = service.files().get_media(fileId=file_id)
    fh = io.FileIO(local_path, 'wb')
    downloader = MediaIoBaseDownload(fh, request)
    done = False
    while not done:
        status, done = downloader.next_chunk()
    fh.close()
    mod_time = time.mktime(time.strptime(modified_time, "%Y-%m-%dT%H:%M:%S.%fZ"))
    os.utime(local_path, times=(mod_time, mod_time))

def sync_folder():
    if not os.path.exists(LOCAL_DIR):
        os.makedirs(LOCAL_DIR)

    service = get_drive_service()
    drive_files = get_folder_files(service, DRIVE_FOLDER_ID)

    local_files = {f: os.path.getmtime(os.path.join(LOCAL_DIR, f)) for f in os.listdir(LOCAL_DIR) if os.path.isfile(os.path.join(LOCAL_DIR, f))}

    for file in drive_files:
        if file['mimeType'] == 'application/vnd.google-apps.folder':
            continue
        file_name = file['name']
        file_id = file['id']
        modified_time = file['modifiedTime']
        local_path = os.path.join(LOCAL_DIR, file_name)

        drive_mod_time = time.mktime(time.strptime(modified_time, "%Y-%m-%dT%H:%M:%S.%fZ"))

        if file_name not in local_files or abs(local_files[file_name] - drive_mod_time) > 1:
            print(f"Syncing {file_name}...")
            download_file(service, file_id, file_name, local_path, modified_time)
        else:
            print(f"{file_name} is up-to-date.")

    for local_file in local_files:
        if local_file not in [f['name'] for f in drive_files]:
            print(f"Removing {local_file} from local directory...")
            os.remove(os.path.join(LOCAL_DIR, local_file))

def remove_files(service, filenames):
    """
    Remove specified files from both local sync folder and the Google Drive folder.
    With Editor permissions, files are moved to Trash and unlinked from the folder.
    Args:
        service: Google Drive API service instance.
        filenames (list): List of filenames to remove.
    """
    drive_files = get_folder_files(service, DRIVE_FOLDER_ID)
    drive_file_map = {f['name']: f['id'] for f in drive_files}

    for filename in filenames:
        # Remove from local folder
        local_path = os.path.join(LOCAL_DIR, filename)
        if os.path.exists(local_path):
            try:
                os.remove(local_path)
                print(f"Removed {filename} from local directory.")
            except Exception as e:
                print(f"Error removing {filename} locally: {e}")
        else:
            print(f"{filename} not found in local directory.")

        # Remove from Google Drive folder (move to Trash and unlink)
        if filename in drive_file_map:
            file_id = drive_file_map[filename]
            try:
                # Move to Trash and remove from the folder
                service.files().update(
                    fileId=file_id,
                    body={'trashed': True},  # Move to Trash
                    removeParents=DRIVE_FOLDER_ID  # Unlink from the original folder
                ).execute()
                print(f"Moved {filename} to Trash and removed from folder in Google Drive.")
            except Exception as e:
                print(f"Error processing {filename} in Google Drive: {e}")
        else:
            print(f"{filename} not found in Google Drive folder.")


def excel_to_dict(file_path, sheet_name=0):
    # Read the Excel file
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore", category=UserWarning, module=re.escape('openpyxl.styles.stylesheet'))
        df = pd.read_excel(file_path, sheet_name=sheet_name)
    
    # Convert DataFrame to dictionary where headers are keys
    # 'records' orientation makes each row a separate dict
    result = df.to_dict(orient='index')
    return result



def filter_dict_by_wfo(data, active="active"):

    return {key: inner_dict for key, inner_dict in data.items() 
            #and 'WFO' in inner_dict['NOAA_ORG_TITLE']
            if 'NOAA_ORG_TITLE' in inner_dict and "NOAA" in inner_dict['EMPL_CODE']}


def collect_and_organize_by_org(data, fields_to_collect, position_title_lookup):
    """
    Collect specific fields, normalize NOAA_POSITION_TITLE, and organize by NOAA_ORG_TITLE with counts.
    
    :param data: Dictionary with nested personnel data
    :param fields_to_collect: List of fields to extract
    :param position_title_lookup: Dict mapping NOAA_POSITION_TITLE variations to standardized titles
    :return: Tuple of collected data, org-specific counts, and overall position counts
    """
    collected_data = {}
    org_title_counts = {}  # NOAA_ORG_TITLE -> NOAA_POSITION_TITLE -> count
    overall_position_counts = {}  # Overall NOAA_POSITION_TITLE -> count
    
    # Loop through the data
    for outer_key, inner_dict in data.items():
        entry = {}
        
        # Collect specified fields
        for field in fields_to_collect:
            if field in inner_dict:
                if field == 'NOAA_POSITION_TITLE':
                    raw_title = inner_dict[field].strip()
                    normalized_title = position_title_lookup.get(raw_title, raw_title)
                    entry['ORIG_TITLE'] = raw_title
                    entry[field] = normalized_title
                else:
                    entry[field] = inner_dict[field]
                    entry['ORIG_TITLE'] = inner_dict['NOAA_POSITION_TITLE']
            else:
                entry[field] = ''
        
        # Store the entry
        collected_data[outer_key] = entry

    return collected_data





def collect_data(file):
    #data = csv_dict(import_file=r"C:\Users\john.peck\Downloads\NSD.xlsx")
    data = excel_to_dict(file, sheet_name=0)
    data = filter_dict_by_wfo(data)

    fields_to_collect = [
        'NOAA_POSITION_TITLE', 'ACCT_STATUS', 'OFFICE', 'NOAA_ORG_TITLE', 'PERSON_ID', 'EMPL_CODE',
        'LAST_NAME', 'FIRST_NAME', 'MIDDLE_NAME', 'MGR_NAME', 'LAST_UPDATED'
    ]



    
    # Lookup table for NOAA_POSITION_TITLE normalization
    position_title_lookup = {
        'Electronic Technician': 'Electronics Technician',
        'Electronics Tech': 'Electronics Technician',
        'Electronics Technician': 'Electronics Technician',
        'El Tech': 'Electronics Technician',
        'ET': 'Electronics Technician',
        'El Tech': 'Electronics Technician',
        'EKA - Electronics Technician': 'Electronics Technician',
        'Electronics Tecnician': 'Electronics Technician',
        'FGZ - Electronics Technician': 'Electronics Technician',
        'EL TECH': 'Electronics Technician',
        'PQR - Electronics Technician': 'Electronics Technician',
        'MSO - El Tech': 'Electronics Technician',
        'TFX - Electronics Technician': 'Electronics Technician',
        'Eltech': 'Electronics Technician',
        'GGW - Electronics Technician': 'Electronics Technician',
        'SEW - El Tech': 'Electronics Technician',
        'Electrical Technician': 'Electronics Technician',
        'Electronic Techncian': 'Electronics Technician',
        'Meteorologist': 'Meteorologist (Could Include Leads)',
        'Forecaster': 'Meteorologist (Could Include Leads)',
        'General Forecaster': 'Meteorologist (Could Include Leads)',
        'Meteorologist Intern': 'Meteorologist (Could Include Leads)',
        'General Meteorologist': 'Meteorologist (Could Include Leads)',
        'NOAA Federal Employee': 'Meteorologist (Could Include Leads)',
        'Met Intern': 'Meteorologist (Could Include Leads)',
        'Journey Forecaster': 'Meteorologist (Could Include Leads)',
        'Meteorologist - IMET': 'Meteorologist (Could Include Leads)',
        'METEOROLOGIST': 'Meteorologist (Could Include Leads)',
        'Meteorlogist': 'Meteorologist (Could Include Leads)',
        'PDT - Meteorologist': 'Meteorologist (Could Include Leads)',
        'MTR - General Forecaster': 'Meteorologist (Could Include Leads)',
        'LKN - Forecaster': 'Meteorologist (Could Include Leads)',
        'Meteorolgist': 'Meteorologist (Could Include Leads)',
        'PIH - Meteorologist': 'Meteorologist (Could Include Leads)',
        'Meterologist': 'Meteorologist (Could Include Leads)',
        'Journeyman Forecaster': 'Meteorologist (Could Include Leads)',
        'Meteorological Intern': 'Meteorologist (Could Include Leads)',
        'OTX - Forecaster': 'Meteorologist (Could Include Leads)',
        'NWS Intern': 'Meteorologist (Could Include Leads)',
        'Meteorologist - General Forecaster': 'Meteorologist (Could Include Leads)',
        'MET Intern': 'Meteorologist (Could Include Leads)',
        'MIT': 'Meteorologist (Could Include Leads)',
        'Forecaster/Incident Meteorologist': 'Meteorologist (Could Include Leads)',
        'Entry Level Meteorologist': 'Meteorologist (Could Include Leads)',
        'Meteorologist and IMET': 'Meteorologist (Could Include Leads)',
        'Fire Weather Program Manager': 'Meteorologist (Could Include Leads)',
        'Meteorologist Intern WSFO JAN': 'Meteorologist (Could Include Leads)',
        'Meteorologist ASA': 'Meteorologist (Could Include Leads)',
        'Lead Meteorologist and IMET': 'Meteorologist (Could Include Leads)',
        'meteorologist': 'Meteorologist (Could Include Leads)',
        'PIH - General Forecaster': 'Meteorologist (Could Include Leads)',
        'TFX - Meteorologist': 'Meteorologist (Could Include Leads)',
        'SEW Forecaster': 'Meteorologist (Could Include Leads)',
        'Metorologist': 'Meteorologist (Could Include Leads)',
        'MET': 'Meteorologist (Could Include Leads)',
        'Meteorologist General': 'Meteorologist (Could Include Leads)',
        'Meteorogist': 'Meteorologist (Could Include Leads)',
        'LKN - General Forecaster': 'Meteorologist (Could Include Leads)',
        'EKA - Forecaster': 'Meteorologist (Could Include Leads)',
        'Meteorologist  - Journey': 'Meteorologist (Could Include Leads)',
        'REV General Forecaster': 'Meteorologist (Could Include Leads)',
        'VEF - General Forecaster': 'Meteorologist (Could Include Leads)',
        'MTR - Meteorologist': 'Meteorologist (Could Include Leads)',
        'Metorologist - National NWSChat Admin': 'Meteorologist (Could Include Leads)',
        'MSO-Meteorologist': 'Meteorologist (Could Include Leads)',
        'VEF - Meteorologist': 'Meteorologist (Could Include Leads)',
        'GGW - Meteorologist': 'Meteorologist (Could Include Leads)',
        'Meteorologist - Journey': 'Meteorologist (Could Include Leads)',
        'EKA - Meteorologist': 'Meteorologist (Could Include Leads)',
        'Meteorologist Senior Forecaster': 'Lead Meteorologist',
        'Senior Forecaster - LIX': 'Lead Meteorologist',
        'TWC - Lead Forecaster': 'Lead Meteorologist',
        'Meteorologist - Lead': 'Lead Meteorologist',
        'Senior Forecaster-Fire Weather Program Manager': 'Lead Meteorologist',
        'Lead Forecasters': 'Lead Meteorologist',
        'Meteorologist - Senior': 'Lead Meteorologist',
        'lead Meteorologist': 'Lead Meteorologist',
        'Senior Forecaster Lead Meteorologist': 'Lead Meteorologist',
        'Lead Meteorologist': 'Lead Meteorologist',
        'Senior Meteorologist': 'Lead Meteorologist',
        'Senior Forecaster': 'Lead Meteorologist',
        'Lead Forecaster': 'Lead Meteorologist',
        'Meteorologist - Lead Forecaster': 'Lead Meteorologist',
        'Meteorologist - Senior Forecaster': 'Lead Meteorologist',
        'Meteorologist   Lead Forecaster': 'Lead Meteorologist',
        'Information Technology Officer': 'ITO (May Include non ITO IT at WFOs)',
        'IT Officer': 'ITO (May Include non ITO IT at WFOs)',
        'ITO': 'ITO (May Include non ITO IT at WFOs)',
        'Information Technology Specialist': 'ITO (May Include non ITO IT at WFOs)',
        'IT Specialist': 'ITO (May Include non ITO IT at WFOs)',
        'FGZ ITO': 'ITO (May Include non ITO IT at WFOs)',
        'Information Technology Specialist ITO': 'ITO (May Include non ITO IT at WFOs)',
        'Information Technology Officer(ITO)/Meteorologist': 'ITO (May Include non ITO IT at WFOs)',
        'VEF - Information Technology Officer': 'ITO (May Include non ITO IT at WFOs)',
        'Information Technolgy Officer': 'ITO (May Include non ITO IT at WFOs)',      
        'Information Technology Officer -ITO': 'ITO (May Include non ITO IT at WFOs)',
        'Supervisory IT Specialist': 'ITO (May Include non ITO IT at WFOs)',
        'IT Specialist - Systems Administrator': 'ITO (May Include non ITO IT at WFOs)',
        'Information Technology Specialist ITO (May Include non ITO IT at WFOs)': 'ITO (May Include non ITO IT at WFOs)',
        'Electronics Systems Analyst': 'ESA',
        'Electronic System Analyst': 'ESA',
        'Electronic Systems Analyst': 'ESA',
        'Electronics System Analyst': 'ESA',
        'Electronic Systems Analyst - ESA': 'ESA',
        'AESA': 'ESA',
        'IT Specialist - Electronics System Analyst': 'ESA',
        'OTX ESA': 'ESA',
        'HNX - Electronic Systems Analyst': 'ESA',
        'Supervisory Information Technology Specialist - ESA': 'ESA',
        'Electronic Systems Analyst - ESA IT Specialist': 'ESA',
        'IT Specialist A-ESA': 'ESA',
        'Electronics Systems Analyst ESA': 'ESA',
        'STO ESA': 'ESA',
        'Electronics Systems Analyst -ESA': 'ESA',
        'Assistant ESA': 'ESA',
        'PQR - Assistant ESA': 'ESA',
        'Electronic Systems Analyst -ESA': 'ESA',
        'Meteorologist - Science Operations Officer': 'SOO',
        'SOO': 'SOO',
        'Science and Operations Officer': 'SOO',
        'Science Operations Officer': 'SOO',
        'Meteorologist - SOO': 'SOO',
        'Meteorologist - Science and Operations Officer': 'SOO',
        'Science and Operations Officer - AMIC': 'SOO',
        'Meteorologist -SOO': 'SOO',
        'Science Operations Officer SOO': 'SOO',
        'Science and Operations Officer - SOO': 'SOO',
        'Science amp; Operations Officer': 'SOO',
        'Meteorologist SOO': 'SOO',
        'Science and Operations Officer DOC NOAA NWS Taunton MA': 'SOO',
        'Science and Operations Officer - NWS New York - NY': 'SOO',
        'Warning Coordination Meteorologist': 'WCM',
        'WCM': 'WCM',
        'Meteorologist - Warning Coordination Meteorologist': 'WCM',
        'Warning Coordination Meteorololgist': 'WCM',
        'Warning Coordination Meteorologist - WCM': 'WCM',
        'Meteorologist WCM': 'WCM',
        'WCM - Meteorologist': 'WCM',
        'Warning and Coordination Meteorologist': 'WCM',
        'HNX WCM': 'WCM',
        'Warning Coordination Meeorologist': 'WCM',
        'Warning Coordination Meteorlogist': 'WCM',
        'Meteorologist In Charge - MIC': 'MIC',
        'MIC': 'MIC',
        'Meteorologist-In-Charge': 'MIC',
        'Meteorologist In Charge': 'MIC',
        'Meteorologist in Charge': 'MIC',
        'Meteorologist-in-Charge': 'MIC',
        'Meterorologist in Charge MIC': 'MIC',
        'Meteorologist-in-Charge MIC': 'MIC',
        'Meteorologist In Charge MIC': 'MIC',
        'HNX MIC': 'MIC',
        'Observations Program Leader': 'OPL',
        'OPL': 'OPL',
        'Observation Program Leader': 'OPL',
        'Observing Program Leader': 'OPL',
        'Observation Program Leader -OPL': 'OPL',
        'Observation Program Leader - OPL': 'OPL',
        'Observing Progam Leader': 'OPL',
        'Observer Program Leader': 'OPL',
        'Observer Program Leader -OPL': 'OPL',
        'Observing Program Leader - OPL': 'OPL',
        'PIH - Observation Program Lead': 'OPL',
        'Observing Program Lead': 'OPL',
        'Observations Program Leader - OPL': 'OPL',
        'Observation Programs Lead': 'OPL',
        'Meteorological Technician - OPL': 'OPL',
        'Meteorologist - Observing Progam Leader': 'OPL',
        'Lead Meteorological Technician': 'OPL',
        'Observation Program Manager': 'OPL',
        'Cooperative Program Manager': 'OPL',
        'Data Acquisition Program Manager': 'OPL',
        'Senior Hydrologist': 'Service Hydrologist',
        'Senior Service Hydrologist': 'Service Hydrologist',
        'Hydrologist': 'Service Hydrologist',
        'Service Hydrologist': 'Service Hydrologist',
        'Hydrologic Forecaster': 'Service Hydrologist',
        'Service Hydrologist Meteorologist': 'Service Hydrologist',
        'Lead Hydrologist': 'Service Hydrologist',
        'Sr. Service Hydrologist': 'Service Hydrologist',
        'Senior Service Hydrologist/Meteorologist': 'Service Hydrologist',
        'EKA Hydrologist': 'Service Hydrologist',
        'Meteorological Technician': 'HMT',
        'HMT': 'HMT',
        'Port Meteorological Officer': 'HMT',
        'Hydro-Meteorological Technician': 'HMT',
        'Hydrometeorological Technician': 'HMT',
        'PMO': 'HMT',
        'AROS Site Operator': 'HMT',
        'Upper  Air Weather Observer': 'HMT',
        'Meteorologist Technician': 'HMT',
        'Ice-SST Specialist': 'HMT',
        'Great Lakes PMO': 'HMT',
        'Ice SST Specialist': 'HMT',
        'Upper Air Weather Observer': 'HMT',
        'ASA': 'ASA',
        'Administrative Support Asst.': 'ASA',
        'Administrative Support Assistant': 'ASA',
        'Administrative Support Assistant ASA': 'ASA',
        'Administative Support Assistant': 'ASA',
        'ASa': 'ASA',
        'FGZ - Administrative Support': 'ASA',
        'Administrative Support Assitant': 'ASA',
        'Administrative Support Assistant - ASA - COTR': 'ASA',
        'Administrative Assistant': 'ASA',
        'Admin Suppt Asst': 'ASA',
        'Supervisory Meteorologist': 'Unspecified',
        'Operations Manager': 'Unspecified',
        'Director of Operations': 'Unspecified',
        'Assistant Meteorologist Ice-SST': 'Unspecified',
        'Skillbridge Electronics Technician': 'Unspecified',
        'Regional Equipment Specialist  ER NWR Focal Point': 'Unspecified',
        'Virtual Volunteer': 'Unspecified',
        'WRH Service Evolution Program Leader': 'Unspecified',
        'Applications Integration Meteorologist': 'Unspecified',
        'Skillbridge Volunteer': 'Unspecified',
        'Contrator': 'Contractor',
        'Contracto': 'Contractor',
        'Contractor': 'Contractor',
        'FET': 'Facilities Engineering Technician',
        'FET': 'FET',
        'VEF - Engineering Technician': 'FET',
        'Facilities Technician': 'FET',
        'Engineering Technician': 'FET',
        'Facilities Engineering Technician': 'FET',
        'Field Engineering Tech': 'FET',
        'Facilities Engineering Tech': 'FET',
        'Field Engineering Technician': 'FET',
        'Regional Maintenance Specialist': 'RMS',
        'RMS': 'RMS',
        'ASOS RMS': 'RMS',
        'Pathways Student': 'Pathways',
        'Pathway Student Trainee': 'Pathways',
        'Pathways Intern': 'Pathways',
        'Pathways': 'Pathways',
        'Pathway': 'Pathways',
        'MTR - Student Intern': 'Pathways',
        'Student Fellow': 'Pathways',
        'Hollings Scholar': 'Pathways',
        'Student Trainee Meteorology': 'Pathways',
        'Pathway Intern': 'Pathways',
        'Emergency Response Specialist': 'ERS',
        'ERS': 'ERS',
        
        
        
    }


    collected_data = collect_and_organize_by_org(data, fields_to_collect, position_title_lookup)
    return collected_data



def loop_through_xls():
    """
    Loops through Excel files in a directory and returns a dictionary containing
    file information sorted by modification time.
    
    Returns:
        dict: Dictionary with file names, paths, and modification times
    """
    directory = "/var/www/html/work/NOAA"
    result = {}
    
    # Get list of .xlsx files
    xlsx_files = [f for f in os.listdir(directory) if f.endswith('.xlsx')]
    
    # Sort files by modification time
    xlsx_files.sort(key=lambda f: os.path.getmtime(os.path.join(directory, f)))
    
    # Populate result dictionary
    
    for file in xlsx_files:
        full_path = os.path.join(directory, file)
        # Get modification time and convert to datetime
        mod_time = datetime.fromtimestamp(os.path.getmtime(full_path))
        
        # Add file info to result dictionary using filename as key
        result[file] = {
            'path': full_path,
            'last_updated': mod_time,
            'file_name': file
        }
    
    return result




def get_inner_dict(big_dict, j):
    return next((inner for inner in big_dict.values() if inner['PERSON_ID'] == j), None)


def compare_personids(personids, person_dict):
    try:
        # Extract person IDs from the dictionary into a set, only if 'PERSON_ID' exists
        dict_person_ids = {inner_dict['PERSON_ID'] for inner_dict in person_dict.values() if 'PERSON_ID' in inner_dict}
        
        # Convert the list of person IDs to a set
        list_person_ids = set(personids)
        
        # Compute the three sets
        in_both = list_person_ids & dict_person_ids         # Intersection: IDs in both
        only_in_list = list_person_ids - dict_person_ids    # Difference: IDs only in list
        only_in_dict = dict_person_ids - list_person_ids    # Difference: IDs only in dict
        
        # Return results in a dictionary
        return {
        'in_both': in_both,
        'only_in_list': only_in_list,
        'only_in_dict': only_in_dict
        }
    except Exception as e:
        # Output the error and traceback
        print("Content-Type: text/plain\n")
        print("An error occurred:\n")
        traceback.print_exc(file=sys.stdout)



def insert_data(data):
    try:
        #replace this timestamp with the latest value in the data
        
        #now = datetime.now()
        #formatted_time = now.strftime("%m/%d/%Y %I:%M:%S %p")
        sql = "SELECT DISTINCT personid FROM nws ORDER BY personid"
        cursor.execute(sql)
        personids_tuple = cursor.fetchall()
        personids = [row[0] for row in personids_tuple]
        for i in data:
            post_data = data[i]    
            formatted_time = i
            result = compare_personids(personids, post_data)
            both = result['in_both']
            nowgone = result['only_in_list'] 
            onlynew = result['only_in_dict']
        
            # Process 'both'
            for j in both:
                record = get_inner_dict(post_data, j)
                sql = """
                    INSERT INTO nws (personid, first, middle, last, title, otitle, status, lastupdate, office, mgrname, orgtitle, recordtime) 
                    VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) 
                    ON CONFLICT DO NOTHING
                """
                parms = (
                    record['PERSON_ID'], record['FIRST_NAME'], record['MIDDLE_NAME'], record['LAST_NAME'],
                    record['NOAA_POSITION_TITLE'], record['ORIG_TITLE'], record['ACCT_STATUS'],
                    formatted_time, record['OFFICE'], record['MGR_NAME'], record['NOAA_ORG_TITLE'], record['LAST_UPDATED']
                )
                cursor.execute(sql, parms)
        
            # Process 'nowgone'
            for j in nowgone:
                cursor.execute("SELECT * FROM nws WHERE personid = %s ORDER BY lastupdate DESC LIMIT 1", (j,))
                row = cursor.fetchone()
                if row:
                    column_names = [desc[0] for desc in cursor.description]
                    result = dict(zip(column_names, row))
                    if result['status'] != "gone":
                        sql = """
                            INSERT INTO nws (personid, first, middle, last, title, otitle, status, lastupdate, office, mgrname, orgtitle, recordtime) 
                            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) 
                            ON CONFLICT DO NOTHING
                        """
                        parms = (
                            j, result['first'], result['middle'], result['last'], result['title'],
                            result['otitle'], 'inactive', formatted_time, result['office'],
                            result['mgrname'], result['orgtitle'], result['lastupdate']
                        )
                        cursor.execute(sql, parms)
        
            # Process 'onlynew'
            for j in onlynew:
                record = get_inner_dict(post_data, j)
                sql = """
                    INSERT INTO nws (personid, first, middle, last, title, otitle, status, lastupdate, office, mgrname, orgtitle, recordtime) 
                    VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) 
                    ON CONFLICT DO NOTHING
                """
                parms = (
                    record['PERSON_ID'], record['FIRST_NAME'], record['MIDDLE_NAME'], record['LAST_NAME'],
                    record['NOAA_POSITION_TITLE'], record['ORIG_TITLE'], record['ACCT_STATUS'],
                    formatted_time, record['OFFICE'], record['MGR_NAME'], record['NOAA_ORG_TITLE'], record['LAST_UPDATED']
                )
                cursor.execute(sql, parms)
        
            conn.commit()  # Single commit at the end    
        cursor.execute("update nws set status = 'gone' where status = '' or status = 'NaN'")
        conn.commit()
    except Exception as e:
        print(e)















  


if __name__ == '__main__':
        alldata = {}
        sync_folder()
        deletable = []
        xlsx_files = loop_through_xls()
        print(xlsx_files)
        for p in xlsx_files:
            full_path = xlsx_files[p]['path']
            update = xlsx_files[p]['last_updated']
            file = xlsx_files[p]['file_name']
            # Get the formatted update time
            formatted_time = update.strftime('%m/%d/%Y %I:%M:%S %p')
            # Collect data from the file
            datedata = collect_data(full_path)
            deletable.append(file)
            # Add additional file info if desired (optional)            
            #datedata['path'] = xlsx_files[p]['path']
            
            # Use the formatted time as the key, with datedata as the value
            alldata[formatted_time] = datedata            
            #print(post_json_to_cgi(alldata))
            #call database insert here
            insert_data(alldata)
            #print(alldata)
        #newalldata = remove_duplicate_records(alldata)
        #with open("nws.json", "w") as file:
        #    json.dump(newalldata, file, indent=4)
        #print(post_json_to_cgi(newalldata))
        service = get_drive_service()
        

        
        # Example: Remove specific files from both local and Drive
        files_to_remove = deletable  # Replace with your filenames
        remove_files(service, files_to_remove)
        conn.close()
