generate_demographics_cache.py

import pandas as pd
import numpy as np
import plotly.express as px
import plotly.graph_objects as go
import json
from datetime import datetime
import glob
import os
from data_preprocessing import preprocess_dataframe

def read_all_csvs_in_folder(folder_name):
    """Reads and combines all CSV files from a folder"""
    file_paths = glob.glob(os.path.join(folder_name, "*.csv"))
    if not file_paths:
        return pd.DataFrame()
    
    df_list = []
    for file in file_paths:
        try:
            temp_df = pd.read_csv(file, low_memory=False)
            df_list.append(temp_df)
        except Exception as e:
            print(f"Skipping bad file: {file}")
    
    if df_list:
        return pd.concat(df_list, ignore_index=True)
    else:
        return pd.DataFrame()

def generate_demographics_cache():
    """Pre-generate all Demographics & Policy visualizations and save to cache"""
    
    print("Loading data...")
    df_enrol = read_all_csvs_in_folder("api_data_aadhar_enrolment")
    df_bio = read_all_csvs_in_folder("api_data_aadhar_biometric")
    df_demo = read_all_csvs_in_folder("api_data_aadhar_demographic")
    
    # Preprocess
    if not df_enrol.empty:
        df_enrol = preprocess_dataframe(df_enrol, "Enrollment")
    if not df_bio.empty:
        df_bio = preprocess_dataframe(df_bio, "Biometric")
    if not df_demo.empty:
        df_demo = preprocess_dataframe(df_demo, "Demographic")
    
    demographics_cache = {}
    
    # Get all districts for pre-computation
    districts = sorted(df_enrol['district'].dropna().unique())[:200]
    states = sorted(df_enrol['state'].dropna().unique())
    
    print(f"Processing {len(districts)} districts and {len(states)} states...")
    
    # ===== CHART 1: Age Distribution (Per District) =====
    print("Generating age distribution data...")
    age_distribution = {}
    
    for district in districts:
        dist_data = df_enrol[df_enrol['district'] == district]
        age_distribution[district] = {
            '0-5 years': float(dist_data['age_0_5'].sum()),
            '5-18 years': float(dist_data['age_5_17'].sum()),
            '18+ years': float(dist_data['age_18_greater'].sum())
        }
    
    demographics_cache['age_distribution'] = age_distribution
    
    # ===== CHART 2: Health Score (Per District) =====
    print("Calculating health scores...")
    health_scores = {}
    
    for district in districts:
        enrol_total = df_enrol[df_enrol['district'] == district][['age_0_5', 'age_5_17', 'age_18_greater']].sum().sum()
        
        demo_total = (df_demo[df_demo['district'] == district][['demo_age_5_17', 'demo_age_17_']].sum().sum() if not df_demo.empty else 0)
        bio_total = (df_bio[df_bio['district'] == district][['bio_age_5_17', 'bio_age_17_']].sum().sum() if not df_bio.empty else 0)
        
        updates_total = demo_total + bio_total
        health_score = min((updates_total / enrol_total * 100) if enrol_total > 0 else 0, 100)
        
        health_scores[district] = {
            'health_score': float(health_score),
            'total_enrollment': float(enrol_total),
            'total_updates': float(updates_total)
        }
    
    demographics_cache['health_scores'] = health_scores
    
    # ===== CHART 3: Update Lag Distribution (Global + Per Age Group) =====
    print("Analyzing update lag distributions...")
    
    # Simulate lag data (replace with actual calculation from your backend_logic if available)
    # If you have real lag calculation logic, use that instead
    lag_distributions = {
        'All Ages': {
            'values': list(np.random.exponential(30, 1000)),
            'mean': 30.0,
            'median': 25.0,
            'std': 15.0
        },
        '5-18 years': {
            'values': list(np.random.exponential(28, 800)),
            'mean': 28.0,
            'median': 23.0,
            'std': 14.0
        },
        '18+ years': {
            'values': list(np.random.exponential(35, 1200)),
            'mean': 35.0,
            'median': 30.0,
            'std': 18.0
        }
    }
    
    demographics_cache['lag_distributions'] = lag_distributions
    
    # ===== CHART 4: Age Growth Over Time (Per District) =====
    print("Computing age group growth trajectories...")
    age_growth = {}
    
    for district in districts[:50]:  # Limit to top 50 for performance
        growth_data = df_enrol[df_enrol['district'] == district].groupby('date')[
            ['age_0_5', 'age_5_17', 'age_18_greater']
        ].sum().reset_index()
        
        if not growth_data.empty:
            age_growth[district] = {
                'dates': growth_data['date'].dt.strftime('%Y-%m-%d').tolist(),
                'age_0_5': growth_data['age_0_5'].tolist(),
                'age_5_17': growth_data['age_5_17'].tolist(),
                'age_18_greater': growth_data['age_18_greater'].tolist()
            }
    
    demographics_cache['age_growth'] = age_growth
    
    # ===== CHART 5: Behavioral Segmentation (State Comparisons) =====
    print("Computing behavioral segmentation...")
    state_comparisons = {}
    
    for state in states:
        demo_total = (df_demo[df_demo['state'] == state][['demo_age_5_17', 'demo_age_17_']].sum().sum() if not df_demo.empty else 0)
        bio_total = (df_bio[df_bio['state'] == state][['bio_age_5_17', 'bio_age_17_']].sum().sum() if not df_bio.empty else 0)
        
        state_comparisons[state] = {
            'demo_updates': float(demo_total),
            'bio_updates': float(bio_total),
            'total_updates': float(demo_total + bio_total)
        }
    
    demographics_cache['state_comparisons'] = state_comparisons
    
    # ===== Metadata =====
    demographics_cache['metadata'] = {
        'generated_at': datetime.now().isoformat(),
        'total_districts': len(districts),
        'total_states': len(states),
        'total_enrollments': int(df_enrol[['age_0_5', 'age_5_17', 'age_18_greater']].sum().sum()),
        'date_range': {
            'start': df_enrol['date'].min().strftime('%Y-%m-%d') if not df_enrol.empty else None,
            'end': df_enrol['date'].max().strftime('%Y-%m-%d') if not df_enrol.empty else None
        }
    }
    
    # Save to JSON
    print("Saving to demographics_cache.json...")
    with open('demographics_cache.json', 'w') as f:
        json.dump(demographics_cache, f, indent=2)
    
    print("✅ Demographics cache generated successfully!")
    print(f"   - {len(districts)} districts processed")
    print(f"   - {len(states)} states analyzed")
    print(f"   - Generated at: {demographics_cache['metadata']['generated_at']}")

if __name__ == "__main__":
    generate_demographics_cache()