# batch_MergeAndCharts_ATT.py
#
# Batch runner for MergeAndChartsv19_ATT.py
# Iterates over every path in the ATT Rosetta Stone and runs the full
# merge + charts pipeline for each one.
#
# Usage:
#   python d:\att\batch_MergeAndCharts_ATT.py
#
# Optional arguments:
#   --start 1050          Start at path_num 1050 (skip earlier paths)
#   --end   1200          Stop after path_num 1200
#   --paths 1050 1075 1100  Run only the specified path numbers
#
# Output files go to d:\att\complete\<Subfolder>\ (directories must already exist)
# A log file is written to d:\att\batch_MergeAndCharts_KinderMorgan_log_<YYYYMMDD_HHMMSS>.txt

import os
import sys
import argparse
import numpy as np
import pandas as pd
import matplotlib
matplotlib.use('Agg')   # non-interactive backend - no chart windows, saves only
import matplotlib.pyplot as plt
from datetime import datetime

# ---------------------------------------------------------------------------
# Configuration
# ---------------------------------------------------------------------------
ROSETTA_PATH = r'd:\KinderMorgan\KinderMorgan_Rosetta_populated.csv'
OUTPUT_BASE  = r'd:\KinderMorgan\complete'

# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------
parser = argparse.ArgumentParser(description='Batch MergeAndCharts for ATT project')
parser.add_argument('--start',  type=int, default=None, help='First path_num to process')
parser.add_argument('--end',    type=int, default=None, help='Last path_num to process (inclusive)')
parser.add_argument('--paths',  type=int, nargs='+',    help='Run only these specific path numbers')
args = parser.parse_args()

# ---------------------------------------------------------------------------
# Logging setup
# ---------------------------------------------------------------------------
now        = datetime.now()
log_name   = now.strftime('batch_MergeAndCharts_KinderMorgan_log_%Y%m%d_%H%M%S.txt')
log_path   = os.path.join(r'd:\KinderMorgan', log_name)

class Tee:
    """Write to both console and log file simultaneously."""
    def __init__(self, filepath):
        self.terminal = sys.stdout
        self.log      = open(filepath, 'w', encoding='utf-8')
    def write(self, message):
        self.terminal.write(message)
        self.log.write(message)
    def flush(self):
        self.terminal.flush()
        self.log.flush()

import warnings

tee = Tee(log_path)
sys.stdout = tee
sys.stderr = tee  # also capture warnings and errors to log

# Route Python warnings through the logger
warnings.showwarning = lambda msg, cat, fname, lineno, file=None, line=None: \
    print(f"WARNING ({cat.__name__}): {msg}  [{fname}:{lineno}]")

# ---------------------------------------------------------------------------
# Load Rosetta Stone
# ---------------------------------------------------------------------------
print("=" * 70)
print(" Kinder Morgan MergeAndCharts BATCH RUNNER")
print(f" Started: {now.strftime('%Y-%m-%d %H:%M:%S')}")
print(f" Log:     {log_path}")
print("=" * 70)
print()

filenaming = pd.read_csv(ROSETTA_PATH)

# Apply path filters
if args.paths:
    paths_to_run = filenaming[filenaming['path_num'].isin([str(p) for p in args.paths])]
elif args.start is not None or args.end is not None:
    mask = pd.Series([True] * len(filenaming), index=filenaming.index)
    if args.start is not None:
        mask &= filenaming['path_num'] >= str(args.start)
    if args.end is not None:
        mask &= filenaming['path_num'] <= str(args.end)
    paths_to_run = filenaming[mask]
else:
    paths_to_run = filenaming

total      = len(paths_to_run)
succeeded  = 0
failed     = 0
skipped    = 0
failures   = []   # list of (path_num, subfolder, error message)

print(f"Paths to process: {total}")
print()

# ---------------------------------------------------------------------------
# Core processing function
# ---------------------------------------------------------------------------
def process_path(pathrow):
    """Run the full merge + charts pipeline for one path row."""

    path_num       = pathrow['path_num']
    AReceiver      = str(pathrow['A_Rcvr_Call_Sign'])
    BReceiver      = str(pathrow['B_Rcvr_Call_Sign'])
    amergefilename = str(pathrow['A_Rcvr_Merged_File'])
    bmergefilename = str(pathrow['B_Rcvr_Merged_File'])
    subfolder      = str(pathrow['Subfolder'])
    base = r'd:\KinderMorgan\complete' + '\\' + subfolder + '\\' + subfolder
    AIntInputfile  = base + '_A_Interference_Obstruction.csv'
    BIntInputfile  = base + '_B_Interference_Obstruction.csv'
    ATerrInputfile = base + '_A_Terrain_Obstruction.csv'
    BTerrInputfile = base + '_B_Terrain_Obstruction.csv'
    outputpath     = os.path.join(OUTPUT_BASE, subfolder)

    # --- verify output directory exists ---
    if not os.path.isdir(outputpath):
        raise FileNotFoundError(f"Output directory does not exist: {outputpath}")

    # --- verify all four input files exist ---
    for f in [AIntInputfile, BIntInputfile, ATerrInputfile, BTerrInputfile]:
        if not os.path.isfile(f):
            raise FileNotFoundError(f"Input file not found: {f}")

    # --- load data (try UTF-8 first, fall back to latin-1 for legacy files) ---
    def read_csv_safe(filepath, **kwargs):
        try:
            return pd.read_csv(filepath, encoding='utf-8', **kwargs)
        except UnicodeDecodeError:
            return pd.read_csv(filepath, encoding='latin-1', **kwargs)

    Ainterference_data = read_csv_safe(AIntInputfile)
    Aobstruction_data  = read_csv_safe(ATerrInputfile, low_memory=False)
    Binterference_data = read_csv_safe(BIntInputfile)
    Bobstruction_data  = read_csv_safe(BTerrInputfile, low_memory=False)

    # --- merge ---
    Amerged_data = pd.merge(Ainterference_data, Aobstruction_data,
                            left_on='Building', right_on='Int Building', how='inner')
    Bmerged_data = pd.merge(Binterference_data, Bobstruction_data,
                            left_on='Building', right_on='Int Building', how='inner')

    # --- site check ---
    Amerged_data['Site_Check'] = Amerged_data['Building'] - Amerged_data['Int Building']
    Bmerged_data['Site_Check'] = Bmerged_data['Building'] - Bmerged_data['Int Building']
    discA = Amerged_data[Amerged_data['Site_Check'] != 0]
    if not discA.empty:
        print(f"    WARNING: A direction - {len(discA)} alignment discrepancies")
    discB = Bmerged_data[Bmerged_data['Site_Check'] != 0]
    if not discB.empty:
        print(f"    WARNING: B direction - {len(discB)} alignment discrepancies")

    # --- composite interference ---
    Amerged_data['Composite_Interference'] = (Amerged_data['Raw Interference (dBm)']
                                              - Amerged_data['Obstruction Loss (dB)'])
    Bmerged_data['Composite_Interference'] = (Bmerged_data['Raw Interference (dBm)']
                                              - Bmerged_data['Obstruction Loss (dB)'])

    # --- RFM calculations ---
    for df in [Amerged_data, Bmerged_data]:
        ci = df['Composite_Interference']
        df['RFM 10 dB BEL'] = np.round(10 * np.log10(
            np.power(10, (-10 + ci) / 10) + np.power(10, -94 / 10)) + 94, decimals=1)
        df['RFM 20 dB BEL'] = np.round(10 * np.log10(
            np.power(10, (-20 + ci) / 10) + np.power(10, -94 / 10)) + 94, decimals=1)
        df['RFM 30 dB BEL'] = np.round(10 * np.log10(
            np.power(10, (-30 + ci) / 10) + np.power(10, -94 / 10)) + 94, decimals=1)

    # --- QAM calculations ---
    for df in [Amerged_data, Bmerged_data]:
        rfm = df['RFM 20 dB BEL']
        df['1024 QAM'] = np.round(100 * ((31557600 - ((315.576 * (10 ** (rfm / 10))) / (10 ** ((3 * 0) / 10)))) / 31557600), 5)
        df['512 QAM']  = np.round(100 * ((31557600 - ((315.576 * (10 ** (rfm / 10))) / (10 ** ((3 * 1) / 10)))) / 31557600), 5)
        df['256 QAM']  = np.round(100 * ((31557600 - ((315.576 * (10 ** (rfm / 10))) / (10 ** ((3 * 2) / 10)))) / 31557600), 5)
        df['128 QAM']  = np.round(100 * ((31557600 - ((315.576 * (10 ** (rfm / 10))) / (10 ** ((3 * 3) / 10)))) / 31557600), 5)
        df['64 QAM']   = np.round(100 * ((31557600 - ((315.576 * (10 ** (rfm / 10))) / (10 ** ((3 * 4) / 10)))) / 31557600), 5)
        df['32 QAM']   = np.round(100 * ((31557600 - ((315.576 * (10 ** (rfm / 10))) / (10 ** ((3 * 5) / 10)))) / 31557600), 5)
        df['16 QAM']   = np.round(100 * ((31557600 - ((315.576 * (10 ** (rfm / 10))) / (10 ** ((3 * 6) / 10)))) / 31557600), 5)
        df['4 QAM']    = np.round(100 * ((31557600 - ((315.576 * (10 ** (rfm / 10))) / (10 ** ((3 * 8) / 10)))) / 31557600), 5)

    # --- sort ---
    Amerged_data = Amerged_data.sort_values(by='Building')
    Bmerged_data = Bmerged_data.sort_values(by='Building')

    # --- drop unwanted columns ---
    drop_cols = ['Raw Interference (dBm)', 'Path', 'Int Building',
                 'Near Rx Site', 'Far Tx Site', 'Path Clearance Description',
                 'Obstruction Loss (dB)', 'Site_Check']
    Amerged_data.drop(columns=drop_cols, inplace=True)
    Bmerged_data.drop(columns=drop_cols, inplace=True)

    # --- insert site name column ---
    # Site Name and Direction comes from the input CSV - no insert needed

    # --- save CSVs ---
    os.chdir(outputpath)
    filenameoutA = amergefilename + '.csv'
    filenameoutB = bmergefilename + '.csv'
    Amerged_data.to_csv(filenameoutA, index=False)
    Bmerged_data.to_csv(filenameoutB, index=False)

    # --- charts ---
    def make_chart(csvfile, colnumber, titlename, filename, color):
        data = pd.read_csv(csvfile, sep=',', header=0,
                           usecols=[colnumber], index_col=None, dtype=float)
        data = round(data, 0)
        plt.figure(figsize=(14, 8))
        values, bins, bars = plt.hist(data.iloc[:, 0], bins=30, range=(0, 30),
                                      rwidth=0.75, color=color, edgecolor='black')
        plt.yscale('log')
        plt.ylabel('Number of Interference Paths')
        plt.xlabel('Fade Margin Reduction (db)')
        plt.title(titlename)
        plt.bar_label(bars, fontsize=10, color='black')
        plt.savefig(filename)
        plt.close()

    for col, bel in [(4, '10'), (5, '20'), (6, '30')]:
        make_chart(filenameoutA, col,
                   f"{amergefilename}   {bel}dB Building Entry Loss   {subfolder}   Path Number {path_num}",
                   f"{subfolder}_A_{bel}dBBEL", 'steelblue')
        make_chart(filenameoutB, col,
                   f"{bmergefilename}   {bel}dB Building Entry Loss   {subfolder}   Path Number {path_num}",
                   f"{subfolder}_B_{bel}dBBEL", 'forestgreen')

    # --- 3D combined BEL charts ---
    def make_3d_bel_chart(mergefilename, titlename, filename):
        import matplotlib.patches as mpatches

        df = pd.read_csv(mergefilename)
        counts_10 = [0] * 30
        counts_20 = [0] * 30
        counts_30 = [0] * 30

        for col, counts in [('RFM 10 dB BEL', counts_10),
                            ('RFM 20 dB BEL', counts_20),
                            ('RFM 30 dB BEL', counts_30)]:
            vals = df[col].dropna().round(0).astype(int)
            for v in vals:
                if 0 <= v < 30:
                    counts[v] += 1

        DATA = {
            '10 dB BEL': counts_10,
            '20 dB BEL': counts_20,
            '30 dB BEL': counts_30,
        }
        SERIES_COLORS = [
            {'front': '#1d4ed8', 'side': '#3b82f6', 'top': '#93c5fd'},
            {'front': '#15803d', 'side': '#22c55e', 'top': '#86efac'},
            {'front': '#b91c1c', 'side': '#ef4444', 'top': '#fca5a5'},
        ]
        BAR_WIDTH  = 0.22
        BAR_GAP    = 0.03
        GROUP_PAD  = 0.12
        DX         = 0.20
        DZ         = 0.10
        DEPTH_STEP = 0.22
        BG_COLOR   = '#f8fafc'

        series_names = list(DATA.keys())
        series_data  = list(DATA.values())
        n_series     = len(series_data)
        n_bins       = 30
        log_max      = np.log10(10000)
        group_step   = BAR_WIDTH * n_series + BAR_GAP * (n_series - 1) + GROUP_PAD

        def log_height(val):
            if val <= 0:
                return 0.0
            return np.log10(max(val, 1))

        def draw_bar(ax, x0, y0, bh, colors, zorder):
            kw = dict(linewidth=0.3, edgecolor='#00000033')
            ax.fill([x0,             x0 + BAR_WIDTH,      x0 + BAR_WIDTH,      x0],
                    [y0,             y0,                  y0 + bh,             y0 + bh],
                    color=colors['front'], zorder=zorder, **kw)
            ax.fill([x0 + BAR_WIDTH, x0 + BAR_WIDTH + DX, x0 + BAR_WIDTH + DX, x0 + BAR_WIDTH],
                    [y0,             y0 + DZ,              y0 + bh + DZ,        y0 + bh],
                    color=colors['side'], zorder=zorder + 1, **kw)
            ax.fill([x0,             x0 + BAR_WIDTH,      x0 + BAR_WIDTH + DX, x0 + DX],
                    [y0 + bh,        y0 + bh,             y0 + bh + DZ,        y0 + bh + DZ],
                    color=colors['top'], zorder=zorder + 2, **kw)

        fig, ax = plt.subplots(figsize=(16, 8))
        ax.set_facecolor(BG_COLOR)
        fig.patch.set_facecolor(BG_COLOR)

        for ix in range(n_bins - 1, -1, -1):
            for s in range(n_series - 1, -1, -1):
                val = series_data[s][ix]
                if val <= 0:
                    continue
                bh           = log_height(val)
                depth_offset = (n_series - 1 - s) * DEPTH_STEP
                x0           = ix * group_step + s * (BAR_WIDTH + BAR_GAP) + depth_offset
                y0           = (n_series - 1 - s) * DZ * 0.4
                zorder       = ix * 10 + s * 3 + 1
                draw_bar(ax, x0, y0, bh, SERIES_COLORS[s], zorder)

        grid_labels = ['1', '10', '100', '1k', '10k']
        for exp in range(int(log_max) + 1):
            ax.axhline(exp, color='#bbbbbb', linewidth=0.5, zorder=0, linestyle='--')
            ax.text(-0.6, exp, grid_labels[exp], va='center', ha='right', fontsize=9, color='#555')

        mid_series = n_series // 2
        for ix in range(n_bins):
            depth_offset = (n_series - 1 - mid_series) * DEPTH_STEP
            x_center = (ix * group_step + mid_series * (BAR_WIDTH + BAR_GAP)
                        + BAR_WIDTH / 2 + depth_offset)
            ax.text(x_center, -0.22, str(ix), ha='center', va='top', fontsize=8, color='#555')

        ax.set_xlim(-1.2, n_bins * group_step + 1.0)
        ax.set_ylim(-0.45, log_max + 0.7)
        ax.axis('off')
        ax.set_title(titlename, fontsize=12, color='#333', pad=16)
        ax.text(-1.1, log_max / 2, 'Number of Interference Paths (log scale)',
                rotation=90, va='center', ha='right', fontsize=10, color='#555')
        ax.text(n_bins * group_step / 2, -0.48, 'Fade Margin Reduction (dB)',
                ha='center', va='top', fontsize=10, color='#555')

        legend_patches = [
            mpatches.Patch(facecolor=SERIES_COLORS[i]['side'], edgecolor='#33333355',
                           label=series_names[i])
            for i in range(n_series)
        ]
        ax.legend(handles=legend_patches, loc='upper right',
                  fontsize=10, framealpha=0.85, edgecolor='#ccc', fancybox=False)

        plt.tight_layout()
        plt.savefig(filename, dpi=150, bbox_inches='tight', facecolor=BG_COLOR)
        plt.close(fig)

    make_3d_bel_chart(filenameoutA,
                      f"{amergefilename}  ·  Interference Paths by Fade Margin Reduction  ·  Path {path_num}",
                      f"{subfolder}_A_3D_BEL_Chart")
    make_3d_bel_chart(filenameoutB,
                      f"{bmergefilename}  ·  Interference Paths by Fade Margin Reduction  ·  Path {path_num}",
                      f"{subfolder}_B_3D_BEL_Chart")

# ---------------------------------------------------------------------------
# Main loop
# ---------------------------------------------------------------------------
for i, (_, pathrow) in enumerate(paths_to_run.iterrows(), start=1):
    path_num  = pathrow['path_num']
    subfolder = str(pathrow['Subfolder'])
    t_start   = datetime.now()

    print(f"[{i:4d}/{total}]  path {path_num}  {subfolder}  ...", end='  ', flush=True)

    try:
        process_path(pathrow)
        elapsed = (datetime.now() - t_start).total_seconds()
        print(f"OK  ({elapsed:.1f}s)")
        succeeded += 1

    except FileNotFoundError as e:
        print(f"SKIPPED - {e}")
        skipped += 1
        failures.append((path_num, subfolder, f"SKIPPED: {e}"))

    except Exception as e:
        print(f"FAILED - {e}")
        failed += 1
        failures.append((path_num, subfolder, str(e)))

# ---------------------------------------------------------------------------
# Summary
# ---------------------------------------------------------------------------
total_elapsed = datetime.now() - now
print()
print("=" * 70)
print(" BATCH COMPLETE")
print(f"   Total paths : {total}")
print(f"   Succeeded   : {succeeded}")
print(f"   Skipped     : {skipped}  (missing input files or output directory)")
print(f"   Failed      : {failed}   (unexpected errors)")
print(f"   Elapsed     : {str(total_elapsed).split('.')[0]}")
print("=" * 70)

if failures:
    print()
    print("Paths that did not complete:")
    for path_num, subfolder, msg in failures:
        print(f"  {path_num}  {subfolder}  ->  {msg}")

print()
print(f"Full log saved to: {log_path}")