/* WSPR Data Reduce
 *
 * This program reads in a CSV file in the formate delivered by WSPRNet.org (archives
 * after they have been unzipped) and culls the entries that don't match the callsign.
 *
 * The output is shrunken a little bit (eliminated the callsign and grid square
 * and a couple other fields that aren't really adding to data analysis).
 *
 * The timestamp is converted from UTC to local time and the distance is expressed
 * in miles rather than Km.
 *
 * I also generated a summary data file in the hopes that it might be useful to
 * somebody. The code groups receive notices together for individual times and lists
 * the number of notices and the average distance. In other words, say three stations
 * reported hearing your WSPR wspr station. The summary data file would show one
 * entry for the time slot, followed by 3 (three stations), followed by the average
 * mileage to all three stations. I have no idea if this will be useful to anybody.
 * The summary data file is named the same as the output file with '.summary' tacked
 * on before '.csv'.
 *
 * Written by Bruce Raymond/ND8I   23 Oct 2016
 *
 * version  0.2
 *
 * WSPR_Data_Reduce is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * WSPR_Data_Reduce is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See the GNU General Public License for more details:
 *    www.gnu.org/licenses
 */

using System;
using System.Drawing;
using System.IO;
using System.Windows.Forms;

namespace WSPR_Data_Reduce
{
   public partial class WSPR_Data_Reduce_Form : Form
   {
      private const int COLS = 9;
      private const float KmtoMiles = 0.621371192F;
      private const int NO_MATCHES = 2;   // just 2 header lines in the file
      private const int ROWS = 25000;     // transmit every 2 minutes for a month = 22320 (max we should ever see)
      private const int START = 2;        // jump past first two rows in file
      const int timeColumn = 0;
      const int snrColumn = 3;
      const int mileageColumn = 7;
      const int azimuthColumn = 8;
      private string callsign = "";
      private string inFile = "";
      private string outFile = "";
      private string summary = "";

      public WSPR_Data_Reduce_Form()
      {
         InitializeComponent();
         this.Text = "WSPR Data Filter          version " + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version.ToString();
         callsign = Properties.Settings.Default.callsign.ToUpper();
         inFile = Properties.Settings.Default.inFileName;
         outFile = Properties.Settings.Default.outFileName;
         textBoxCallsign.Text = callsign;
         labelStatus.Text = "Idle";
         labelSource.Text = inFile;
         labelTarget.Text = outFile;
      }

      private void buttonExit_Click( object sender, EventArgs e )
      {
         try
         {
            // store callsign and filenames for future use
            Properties.Settings.Default.callsign = callsign.ToUpper();
            Properties.Settings.Default.inFileName = inFile;
            Properties.Settings.Default.outFileName = outFile;
            Properties.Settings.Default.Save();
            Application.Exit();
         }
         catch( Exception ex )
         {
            labelStatus.Text = ex.Message;
         }
      }

      private void buttonFilter_Click( object sender, EventArgs e )
      {
         try
         {
            long fileLength;
            long numberOfCharacters = 0;
            fileLength = FilterSetup();
            Application.DoEvents();
            DateTime epoch = new DateTime( 1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc );

            numberOfCharacters = GenerateRawData( fileLength, numberOfCharacters, epoch );
            labelStatus.Text = "Idle";
            WSPRprogressBar.Value = 100;
            GenerateSummaryData();
         }
         catch( Exception ex )
         {
            labelStatus.Text = ex.Message;
         }
         labelStatus.ForeColor = DefaultForeColor;
      }

      private long FilterSetup()
      {
         // stuff '.summary' before '.csv' in display
         summary = outFile.Remove( outFile.Length - 3 ) + "summary.csv";
         labelStatus.Text = "Processing ...";
         labelStatus.ForeColor = Color.Red;
         labelStatus.Invalidate();
         WSPRprogressBar.Value = 0;
         FileInfo fileinfo = new FileInfo( inFile );
         return fileinfo.Length;
      }

      private void GenerateSummaryData()
      {
         using( StreamWriter writer = new StreamWriter( summary ) )
         {
            writer.WriteLine( callsign );
            writer.WriteLine( "Timestamp (local), Number of Stations, Average SNR, Average Mileage, Composite Azimuth" );

            // read in the raw data file to boil it down - yeah, I know there was probably a
            // more efficient way to do this rather than write out the data and then read it back again.
            using( StreamReader reader = new StreamReader( outFile ) )
            {
               string[] lines = new string[ ROWS ];
               lines = File.ReadAllLines( outFile );
               string[ , ] data = new string[ ROWS, COLS ];
               string[] oneRow = new string[ COLS ];
               int rows = lines.Length;

               // couldn't figure out how to do this with LINQ
               for( int row = START; row < rows; row++ )
               {
                  oneRow = lines[ row ].Split( ',' );
                  for( int c = 0; c < oneRow.Length; c++ )
                     data[ row, c ] = oneRow[ c ];
               }

               WriteSummaryData( writer, data, rows );
            }
         }
      }

      private void WriteSummaryData( StreamWriter writer, string[ , ] data, int rows )
      {
         int length = 1;
         double sumSnr = Convert.ToDouble( data[ START, snrColumn ] );
         double temp = Convert.ToDouble( data[ START, mileageColumn ] );
         double sumDistance = temp;
         double sumDistance2 = temp * temp;
         double sumAzimuth = temp * temp * ( Convert.ToDouble( data[ START, azimuthColumn ] ) + 360 );

         string oldString = data[ START + 1, timeColumn ];

         if( rows > NO_MATCHES )    // our callsign showed up at least once
         {
            // walk through all of the time points and group the data
            for( int row = START + 1; row < rows; row++ )
            {
               if( oldString.Equals( data[ row, timeColumn ] ) )
               {
                  length++;                        // increment number of elements
                  if( !string.IsNullOrEmpty( data[ row, snrColumn ] ) )
                     sumSnr += Convert.ToDouble( data[ row, snrColumn ] );
                  if( !string.IsNullOrEmpty( data[ row, azimuthColumn ] ) )
                  {
                     double distance = Convert.ToDouble( data[ row, mileageColumn ] );
                     double alpha = Convert.ToDouble( data[ row, azimuthColumn ] ) + 360;
                     sumAzimuth += distance * distance * alpha;
                     sumDistance += distance;
                     sumDistance2 += distance * distance;
                  }
               }
               else
               {
                  double azimuth = sumAzimuth / sumDistance2;
                  while( azimuth > 360 ) azimuth -= 360;
                  double w = sumDistance / (double)length;
                  writer.WriteLine( "{0}, {1}, {2:0.0}, {3:0.0}, {4:0.0}", oldString, length,
                     sumSnr / (double)length, sumDistance / (double)length, azimuth );
                  oldString = data[ row, timeColumn ];
                  length = 1;
                  sumSnr = Convert.ToDouble( data[ row, snrColumn ] );
                  temp = Convert.ToDouble( data[ row, mileageColumn ] );
                  sumDistance = temp;
                  sumDistance2 = temp * temp;
                  sumAzimuth = temp * temp * ( Convert.ToDouble( data[ row, azimuthColumn ] ) + 360 );
               }
            }
         }
         else
            labelStatus.Text = "Your Callsign didn't show up in this file.";
      }

      private long GenerateRawData( long fileLength, long numberOfCharacters, DateTime epoch )
      {
         using( StreamWriter writer = new StreamWriter( outFile ) )
         {
            writer.WriteLine( callsign );
            writer.WriteLine( "Timestamp (local), Reporter, Reporter's Grid, SNR (dB), Frequency (MHz), Power (dBm), Drift (Hz), Distance (Mi), Azimuth" );
            foreach( string line in File.ReadLines( inFile ) )
            {
               // This is the format of the data coming from WSPRnet site.
               //
               //                   0          1         2           3           4       5          6        7
               // read data   => Spot ID, Timestamp, Reporter, Reporter's Grid, SNR, Frequency, Call Sign, Grid,
               //      8       9      10       11       12     13     14
               //    Power, Drift, Distance, Azimuth, Band, Version, Code
               numberOfCharacters = WriteDataBlock( fileLength, numberOfCharacters, epoch, writer, line );
            }
         }
         return numberOfCharacters;
      }

      private long WriteDataBlock( long fileLength, long numberOfCharacters, DateTime epoch, StreamWriter writer, string line )
      {
         try
         {
            numberOfCharacters += line.Length;
            WSPRprogressBar.Value = (int)( ( 100 * numberOfCharacters ) / fileLength );
            string[] fields = line.Split( ',' );

            // incoming data (stored in array locations)
            //                    1          2           3           4       5        8      9       10        11     12
            // write data  => Timestamp, Reporter, Reporter's Grid, SNR, Frequency, Power, Drift, Distance, Azimuth, Band
            //
            //
            // This is the format of the data being written out to the (raw) data file (.csv).
            //                    0          1           2           3       4        5      6        7        8  
            // write data  => Timestamp, Reporter, Reporter's Grid, SNR, Frequency, Power, Drift, Distance, Azimuth
            //

            string call = fields[ 6 ].ToUpper();
            if( call.Contains( callsign ) )
            {
               var date = epoch.AddSeconds( Convert.ToInt64( fields[ 1 ] ) );
               DateTime timeStamp = ( Convert.ToDateTime( date ) );
               DateTime localDateTime = timeStamp.ToLocalTime();

               writer.Write( localDateTime.ToString() + "," );                // timestamp
               writer.Write( fields[ 2 ] + "," );                             // reporter
               writer.Write( fields[ 3 ] + "," );                             // reporter's grid
               writer.Write( fields[ 4 ] + "," );                             // snr
               writer.Write( fields[ 5 ] + "," );                             // frequency
               writer.Write( fields[ 8 ] + "," );                             // power
               writer.Write( fields[ 9 ] + "," );                             // drift
               double miles = KmtoMiles * Convert.ToSingle( fields[ 10 ] );
               writer.Write( String.Format( "{0:0.}", miles ) + "," );        // distance
               writer.WriteLine( fields[ 11 ] );                              // azimuth
            }
         }
         catch
         {
         }
         return numberOfCharacters;
      }

      private void buttonOpen_Click( object sender, EventArgs e )
      {
         labelStatus.Text = "";
         try
         {
            openFileDialog1.InitialDirectory = Path.GetDirectoryName( inFile );
            openFileDialog1.FileName = inFile;
            openFileDialog1.Filter = "csv files (*.csv)|*.csv|All files (*.*)|*.*";
            openFileDialog1.CheckFileExists = true;
            openFileDialog1.CheckPathExists = true;
            openFileDialog1.FilterIndex = 1;
            openFileDialog1.RestoreDirectory = true;
            inFile = openFileDialog1.ShowDialog() == DialogResult.OK ? openFileDialog1.FileName : "";
         }
         catch( Exception ex )
         {
            labelStatus.Text = ex.Message;
            inFile = @"c:\";
         }
         labelSource.Text = inFile;
      }

      private void buttonSave_Click( object sender, EventArgs e )
      {
         labelStatus.Text = "";
         try
         {
            saveFileDialog1.InitialDirectory = Path.GetDirectoryName( outFile );
            saveFileDialog1.FileName = outFile;
            saveFileDialog1.Filter = "csv files (*.csv)|*.csv|All files (*.*)|*.*";
            saveFileDialog1.FilterIndex = 1;
            saveFileDialog1.RestoreDirectory = true;
            outFile = saveFileDialog1.ShowDialog() == DialogResult.OK ? saveFileDialog1.FileName : "";
         }
         catch( Exception ex )
         {
            labelStatus.Text = ex.Message;
            outFile = @"c:\";
         }
         labelTarget.Text = outFile;
      }

      private void buttonUnZip_Click( object sender, EventArgs e )
      {
         // I had hopes of actually starting from the .zip file, but it got ugly.
         //
         //   var options = new ReadOptions { StatusMessageWriter = Console.Out };
         //   try
         //   {
         //      using( ZipFile zip = ZipFile.Read( inFile, options ) )
         //      {
         //         zip.ExtractAll( Path.GetDirectoryName( outFile ) );
         //      }
         //   }
         //   catch( Exception ex )
         //   {
         //      labelStatus.Text = ex.Message;
         //   }
      }

      private void textBoxCallsign_TextChanged( object sender, EventArgs e )
      {
         callsign = textBoxCallsign.Text.ToUpper();
      }
   }
}