1 #!/usr/bin/perl -w
   2 #
   3 # File: AnalyzeTextFilesData.pl
   4 # Author: Manish Sud <msud@san.rr.com>
   5 #
   6 # Copyright (C) 2024 Manish Sud. All rights reserved.
   7 #
   8 # This file is part of MayaChemTools.
   9 #
  10 # MayaChemTools is free software; you can redistribute it and/or modify it under
  11 # the terms of the GNU Lesser General Public License as published by the Free
  12 # Software Foundation; either version 3 of the License, or (at your option) any
  13 # later version.
  14 #
  15 # MayaChemTools is distributed in the hope that it will be useful, but without
  16 # any warranty; without even the implied warranty of merchantability of fitness
  17 # for a particular purpose.  See the GNU Lesser General Public License for more
  18 # details.
  19 #
  20 # You should have received a copy of the GNU Lesser General Public License
  21 # along with MayaChemTools; if not, see <http://www.gnu.org/licenses/> or
  22 # write to the Free Software Foundation Inc., 59 Temple Place, Suite 330,
  23 # Boston, MA, 02111-1307, USA.
  24 #
  25 
  26 use strict;
  27 use FindBin; use lib "$FindBin::Bin/../lib";
  28 use Getopt::Long;
  29 use File::Basename;
  30 use Text::ParseWords;
  31 use Benchmark;
  32 use FileUtil;
  33 use TextUtil;
  34 use StatisticsUtil;
  35 
  36 my($ScriptName, %Options, $StartTime, $EndTime, $TotalTime);
  37 
  38 # Autoflush STDOUT
  39 $| = 1;
  40 
  41 # Starting message...
  42 $ScriptName = basename($0);
  43 print "\n$ScriptName: Starting...\n\n";
  44 $StartTime = new Benchmark;
  45 
  46 # Get the options and setup script...
  47 SetupScriptUsage();
  48 if ($Options{help} || @ARGV < 1) {
  49   die GetUsageFromPod("$FindBin::Bin/$ScriptName");
  50 }
  51 
  52 my(@TextFilesList);
  53 @TextFilesList = ExpandFileNames(\@ARGV, "csv tsv");
  54 
  55 print "Processing options...\n";
  56 my(%OptionsInfo);
  57 ProcessOptions();
  58 
  59 # Collect column information for all the text files...
  60 print "Checking input text file(s)...\n";
  61 my(%TextFilesInfo);
  62 RetrieveTextFilesInfo();
  63 ProcessColumnsInfo();
  64 
  65 # Generate output files...
  66 my($FileIndex);
  67 if (@TextFilesList > 1) {
  68   print "\nProcessing text files...\n";
  69 }
  70 for $FileIndex (0 .. $#TextFilesList) {
  71   if ($TextFilesInfo{FileOkay}[$FileIndex]) {
  72     print "\nProcessing file $TextFilesList[$FileIndex]...\n";
  73     AnalyzeTextFile($FileIndex);
  74   }
  75 }
  76 print "\n$ScriptName:Done...\n\n";
  77 
  78 $EndTime = new Benchmark;
  79 $TotalTime = timediff ($EndTime, $StartTime);
  80 print "Total time: ", timestr($TotalTime), "\n";
  81 
  82 ###############################################################################
  83 
  84 # Analyze data...
  85 sub AnalyzeTextFile {
  86   my($Index) = @_;
  87   my($TextFile, $Line, $InDelim, $ColNum, $Value, @LineWords, @ColNumsToAnalyze, %ColValuesToAnalyzeMap);
  88 
  89   $TextFile = $TextFilesList[$Index];
  90   $InDelim = $TextFilesInfo{InDelim}[$Index];
  91   @ColNumsToAnalyze = @{$TextFilesInfo{UniqueColNumsToAnalyze}[$Index]};
  92   %ColValuesToAnalyzeMap = ();
  93   for $ColNum (@ColNumsToAnalyze) {
  94     @{$ColValuesToAnalyzeMap{$ColNum}} = ();
  95   }
  96 
  97   my($LineCount, $InvalidLineCount, @InvalidColLabels);
  98 
  99   open TEXTFILE, "$TextFile" or die "Error: Can't open $TextFile: $! \n";
 100   # Skip over column labels line in text file and collect appropriate column data
 101   # for analysis...
 102   $Line = GetTextLine(\*TEXTFILE);
 103   $LineCount = 1;
 104   $InvalidLineCount = 0;
 105   while ($Line = GetTextLine(\*TEXTFILE)) {
 106     $LineCount++;
 107     @LineWords = quotewords($InDelim, 0, $Line);
 108     @InvalidColLabels = ();
 109     COLNUM: for $ColNum (@ColNumsToAnalyze) {
 110       $Value = $LineWords[$ColNum];
 111       if ($OptionsInfo{CheckData}) {
 112         if (!IsNumerical($Value)) {
 113           push @InvalidColLabels, $TextFilesInfo{ColLabels}[$Index][$ColNum];
 114           next COLNUM;
 115         }
 116       }
 117       push @{$ColValuesToAnalyzeMap{$ColNum}}, $Value;
 118     }
 119     if (@InvalidColLabels) {
 120       $InvalidLineCount++;
 121       if ($OptionsInfo{DetailLevel} >=4 ) {
 122         print "Line number $LineCount contains ", scalar(@InvalidColLabels)," non-numerical or empty value(s) for column(s) - ", JoinWords(\@InvalidColLabels, ", ", 0)," - to be analyzed: $Line \n";
 123       }
 124       elsif ($OptionsInfo{DetailLevel} >= 3) {
 125         print "Line number $LineCount contains ", scalar(@InvalidColLabels)," non-numerical or empty value(s) for column(s) - ", JoinWords(\@InvalidColLabels, ", ", 0)," - to be analyzed...\n";
 126       }
 127       elsif ($OptionsInfo{DetailLevel} >= 2) {
 128         print "Line number $LineCount contains ", scalar(@InvalidColLabels)," non-numerical or empty value(s) for columns to be analyzed...\n";
 129       }
 130     }
 131   }
 132   if ($InvalidLineCount && ($OptionsInfo{DetailLevel} >= 1)) {
 133     print "Non-numerical or empty data present in $InvalidLineCount line(s)...\n";
 134   }
 135   close TEXTFILE;
 136 
 137   # Perform the analysis...
 138   my(@SpecifiedFunctionNames, $SpecifiedFunction);
 139   @SpecifiedFunctionNames = ();
 140 
 141   for $SpecifiedFunction (@{$OptionsInfo{SpecifiedStatisticalFunctions}}) {
 142     if ($SpecifiedFunction !~ /^(Covariance|Correlation|Frequency|Rsquare|StandardScores|StandardScoresN)$/i) {
 143       push @SpecifiedFunctionNames, $OptionsInfo{SpecifiedStatisticalFunctionsMap}{lc($SpecifiedFunction)};
 144     }
 145   }
 146   if (@SpecifiedFunctionNames) {
 147     PerformAnalysis($Index, \@SpecifiedFunctionNames, \%ColValuesToAnalyzeMap)
 148   }
 149   if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance}) || exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation}) || exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare})) {
 150     if ($OptionsInfo{AllColumnPairs}) {
 151       PerformMatrixAnalysis($Index, \%ColValuesToAnalyzeMap);
 152     }
 153     else {
 154       # Perform pairwise analysis for specified columns and write out calculated values - correlation
 155       # rsquare, or covariance - in the same file.
 156       PerformColumnPairAnalysis($Index, \%ColValuesToAnalyzeMap);
 157     }
 158   }
 159   if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{standardscores}) || exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{standardscoresn}) ) {
 160     PerformStandardScoresAnalysis($Index, \%ColValuesToAnalyzeMap);
 161   }
 162   if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{frequency})) {
 163     PerformFrequencyAnalysis($Index, \%ColValuesToAnalyzeMap);
 164   }
 165 }
 166 
 167 # Calculate values for various statistical functions...
 168 sub PerformAnalysis {
 169   my($Index, $SpecifiedFunctionNamesRef, $ColValuesToAnalyzeMapRef) = @_;
 170   my($NewTextFile, $Line, $SpecifiedFunction, $Label, @ColLabels, @ColNumsToAnalyze);
 171 
 172   $NewTextFile = $TextFilesInfo{OutFileRoot}[$Index] . $OptionsInfo{FileNameMode} . "." . $TextFilesInfo{OutFileExt}[$Index];
 173 
 174   print "Generating new text file $NewTextFile...\n";
 175   open NEWTEXTFILE, ">$NewTextFile" or die "Error: Can't open $NewTextFile: $! \n";
 176 
 177   # Write out column labels...
 178   @ColLabels = ();
 179   push @ColLabels, "ColumnID";
 180   for $SpecifiedFunction (@{$SpecifiedFunctionNamesRef}) {
 181     $Label = $SpecifiedFunction;
 182     if ($SpecifiedFunction =~ /^(KLargest|KSmallest)$/i) {
 183       my($KthValue);
 184       $KthValue = ($SpecifiedFunction =~ /^KLargest$/i) ? $OptionsInfo{KLargest} : $OptionsInfo{KSmallest};
 185       $Label = AddNumberSuffix($KthValue) . "$SpecifiedFunction";
 186       $Label =~ s/K//g;
 187     }
 188     elsif ($SpecifiedFunction =~ /^TrimMean$/i) {
 189       $Label = "${SpecifiedFunction}($OptionsInfo{TrimFraction})";
 190     }
 191     push @ColLabels, $Label;
 192   }
 193   $Line = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
 194   print NEWTEXTFILE "$Line\n";
 195 
 196   # Go over each column to be analyzed...
 197   @ColNumsToAnalyze = @{$TextFilesInfo{ColNumsToAnalyze}[$Index]};
 198 
 199   # Turn off "strict"; otherwise, invoking statistical functions using function name string
 200   # is problematic.
 201   no strict;
 202 
 203   my($ColValuesRef, $ColNum, $Value, @RowValues, %CalculatedValues);
 204   %CalculatedValues = ();
 205   for $ColNum (@ColNumsToAnalyze) {
 206     @RowValues = ();
 207     # Setup column id...
 208     push @RowValues, $TextFilesInfo{ColLabels}[$Index][$ColNum];
 209     $ColValuesRef =  \@{$ColValuesToAnalyzeMapRef->{$ColNum}};
 210     FUNCTIONNAME: for $SpecifiedFunction (@{$SpecifiedFunctionNamesRef}) {
 211       $Value = "";
 212       if (!@{$ColValuesToAnalyzeMapRef->{$ColNum}}) {
 213         # Invalid column values...
 214         push @RowValues, $Value;
 215         next FUNCTIONNAME;
 216       }
 217       if ($SpecifiedFunction =~ /^Count$/i) {
 218         $Value = @{$ColValuesToAnalyzeMapRef->{$ColNum}};
 219       }
 220       elsif ($SpecifiedFunction =~ /^KLargest$/i) {
 221         $Value = &$SpecifiedFunction($ColValuesRef, $OptionsInfo{KLargest});
 222       }
 223       elsif ($SpecifiedFunction =~ /^KSmallest$/i) {
 224         $Value = &$SpecifiedFunction($ColValuesRef, $OptionsInfo{KSmallest});
 225       }
 226       elsif ($SpecifiedFunction =~ /^StandardDeviation$/i) {
 227         if (exists($CalculatedValues{$ColNum}{StandardDeviation})) {
 228           $Value = $CalculatedValues{$ColNum}{StandardDeviation};
 229         }
 230         else {
 231           $Value = &$SpecifiedFunction($ColValuesRef);
 232           $CalculatedValues{$ColNum}{StandardDeviation} = $Value;
 233         }
 234       }
 235       elsif ($SpecifiedFunction =~ /^StandardError$/i) {
 236         if (!exists($CalculatedValues{$ColNum}{StandardDeviation})) {
 237           $Value = StandardDeviation($ColValuesRef);
 238           $CalculatedValues{$ColNum}{StandardDeviation} = $Value;
 239         }
 240         if (defined $CalculatedValues{$ColNum}{StandardDeviation}) {
 241           $Value = &$SpecifiedFunction($CalculatedValues{$ColNum}{StandardDeviation}, @{$ColValuesToAnalyzeMapRef->{$ColNum}});
 242         }
 243       }
 244       elsif ($SpecifiedFunction =~ /^TrimMean$/i) {
 245         $Value = &$SpecifiedFunction($ColValuesRef, $OptionsInfo{TrimFraction});
 246       }
 247       else {
 248         $Value = &$SpecifiedFunction($ColValuesRef);
 249       }
 250       # Format the output value. And add zero to get rid of tariling zeros...
 251       $Value = (defined($Value) && length($Value)) ? (sprintf("%.$OptionsInfo{Precision}f", $Value) + 0) : "";
 252       push @RowValues, $Value;
 253     }
 254     $Line = JoinWords(\@RowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
 255     print NEWTEXTFILE "$Line\n";
 256   }
 257   close NEWTEXTFILE;
 258 }
 259 
 260 # Calculate covariance, correlation, rsquare for specified column pairs....
 261 sub PerformColumnPairAnalysis {
 262   my($Index, $ColValuesToAnalyzeMapRef) = @_;
 263   my($NewTextFile, @ColLabels, $Line, $CalculateCorrelation, $CalculateRSquare, $CalculateCovariance);
 264   $CalculateCorrelation = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation}) ? 1 : 0;
 265   $CalculateRSquare = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare}) ? 1 : 0;
 266   $CalculateCovariance = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance}) ? 1 : 0;
 267 
 268   $NewTextFile = $TextFilesInfo{OutFileRoot}[$Index] . "ColumnPairsAnalysis." .  $TextFilesInfo{OutFileExt}[$Index];
 269   print "Generating new text file $NewTextFile...\n";
 270   open NEWTEXTFILE, ">$NewTextFile" or die "Error: Can't open $NewTextFile: $! \n";
 271 
 272   # Write out the column labels...
 273   @ColLabels = ();
 274   push @ColLabels, ("ColumnID1", "ColumnID2");
 275   if ($CalculateCorrelation || $CalculateRSquare) {
 276     push @ColLabels, "Correlation";
 277     if ($CalculateRSquare) {
 278       push @ColLabels, "RSquare";
 279     }
 280   }
 281   if ($CalculateCovariance) {
 282     push @ColLabels, "Covariance";
 283   }
 284   $Line = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
 285   print NEWTEXTFILE "$Line\n";
 286 
 287   # Go over each column pair...
 288   my($CorrelationValue, $RSquareValue, $CovarianceValue,  $ColIndex, $ColNum1, $ColNum2, $ColValuesRef1, $ColValuesRef2, @ColPairs1ToAnalyze, @ColPairs2ToAnalyze, @RowValues, $Value);
 289 
 290   @ColPairs1ToAnalyze = @{$TextFilesInfo{ColPairs1ToAnalyze}[$Index]};
 291   @ColPairs2ToAnalyze = @{$TextFilesInfo{ColPairs2ToAnalyze}[$Index]};
 292   for $ColIndex (0 .. $#ColPairs1ToAnalyze) {
 293     @RowValues = ();
 294     $ColNum1 = $ColPairs1ToAnalyze[$ColIndex];
 295     $ColNum2 = $ColPairs2ToAnalyze[$ColIndex];
 296     $ColValuesRef1 =  \@{$ColValuesToAnalyzeMapRef->{$ColNum1}};
 297     $ColValuesRef2 =  \@{$ColValuesToAnalyzeMapRef->{$ColNum2}};
 298 
 299     # Setup column ids...
 300     push @RowValues, $TextFilesInfo{ColLabels}[$Index][$ColNum1];
 301     push @RowValues, $TextFilesInfo{ColLabels}[$Index][$ColNum2];
 302 
 303     if (@$ColValuesRef1 != @$ColValuesRef2) {
 304       # Print a warning...
 305       warn "Warning: Skipping analysis for column pair $TextFilesInfo{ColLabels}[$Index][$ColNum1], $TextFilesInfo{ColLabels}[$Index][$ColNum2]: Number of valid data values must be same.\n";
 306       if ($CalculateCorrelation || $CalculateRSquare) {
 307         push @RowValues, "";
 308         if ($CalculateRSquare) {
 309           push @RowValues, "";
 310         }
 311       }
 312       if ($CalculateCovariance) {
 313         push @RowValues, "";
 314       }
 315     }
 316     else {
 317       # Calculate appropriate value...
 318       if ($CalculateCorrelation || $CalculateRSquare) {
 319         $CorrelationValue = Correlation($ColValuesRef1, $ColValuesRef2);
 320         $Value = (defined($CorrelationValue) && length($CorrelationValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $CorrelationValue) + 0) : "";
 321         push @RowValues, $Value;
 322         if ($CalculateRSquare) {
 323           $RSquareValue = (defined($CorrelationValue) && length($CorrelationValue)) ? ($CorrelationValue ** 2) : "";
 324           $Value = (length($RSquareValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $RSquareValue) + 0) : "";
 325           push @RowValues, $Value;
 326         }
 327       }
 328       if ($CalculateCovariance) {
 329         $CovarianceValue = Covariance($ColValuesRef1, $ColValuesRef2);
 330         $Value = (defined($CovarianceValue) && length($CovarianceValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $CovarianceValue) + 0) : "";
 331         push @RowValues, $Value;
 332       }
 333     }
 334     $Line = JoinWords(\@RowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
 335     print NEWTEXTFILE "$Line\n";
 336   }
 337   close NEWTEXTFILE;
 338 }
 339 
 340 # Generate histogram numbers...
 341 sub PerformFrequencyAnalysis {
 342   my($Index, $ColValuesToAnalyzeMapRef) = @_;
 343   my($NewTextFile, $ColLabel, @ColLabels, @RowValues, $Line, $ColNum, @ColNumsToAnalyze, $ColValuesRef, $BinValue, $FrequencyValue, $Value, %FrequencyMap);
 344 
 345   @ColNumsToAnalyze = @{$TextFilesInfo{ColNumsToAnalyze}[$Index]};
 346   for $ColNum (@ColNumsToAnalyze) {
 347     $NewTextFile = $TextFilesInfo{OutFileRoot}[$Index] . $TextFilesInfo{ColLabels}[$Index][$ColNum] . "FrequencyAnalysis." .  $TextFilesInfo{OutFileExt}[$Index];
 348     print "Generating new text file $NewTextFile...\n";
 349     open NEWTEXTFILE, ">$NewTextFile" or die "Error: Can't open $NewTextFile: $! \n";
 350 
 351     # Write out the column labels...
 352     @ColLabels = ();
 353     push @ColLabels , ("Bins", "Frequency");
 354     $Line = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
 355     print NEWTEXTFILE "$Line\n";
 356 
 357     #Calculate and write out frequency values...
 358     %FrequencyMap = ();
 359     $ColValuesRef =  \@{$ColValuesToAnalyzeMapRef->{$ColNum}};
 360     if (@$ColValuesRef) {
 361       if (@{$OptionsInfo{BinRange}}) {
 362         %FrequencyMap = Frequency($ColValuesRef, \@{$OptionsInfo{BinRange}});
 363       }
 364       else {
 365         %FrequencyMap = Frequency($ColValuesRef, $OptionsInfo{NumOfBins});
 366       }
 367     }
 368     for $BinValue (sort { $a <=> $b }  keys %FrequencyMap) {
 369       $FrequencyValue = $FrequencyMap{$BinValue};
 370 
 371       @RowValues = ();
 372       $Value = (length($BinValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $BinValue) + 0) : "";
 373       push @RowValues, $Value;
 374       $Value = (length($FrequencyValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $FrequencyValue) + 0) : "";
 375       push @RowValues, $Value;
 376 
 377       $Line = JoinWords(\@RowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
 378       print NEWTEXTFILE "$Line\n";
 379     }
 380     close NEWTEXTFILE;
 381   }
 382 }
 383 
 384 # Calculate covariance, correlation/rsquare matrices....
 385 sub PerformMatrixAnalysis {
 386   my($Index, $ColValuesToAnalyzeMapRef) = @_;
 387   my($CorrelationTextFile, $CovarianceTextFile, $RSquareTextFile, $CalculateCorrelation, $CalculateRSquare, $CalculateCovariance);
 388 
 389   $CalculateCorrelation = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation}) ? 1 : 0;
 390   $CalculateRSquare = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare}) ? 1 : 0;
 391   $CalculateCovariance = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance}) ? 1 : 0;
 392 
 393   $CorrelationTextFile = $TextFilesInfo{OutFileRoot}[$Index] . "CorrelationMatrix." .  $TextFilesInfo{OutFileExt}[$Index];
 394   $RSquareTextFile = $TextFilesInfo{OutFileRoot}[$Index] . "RSquareMatrix." .  $TextFilesInfo{OutFileExt}[$Index];
 395   $CovarianceTextFile = $TextFilesInfo{OutFileRoot}[$Index] . "CovarianceMatrix." .  $TextFilesInfo{OutFileExt}[$Index];
 396 
 397   my($TextFilesList, $Delimiter);
 398   $TextFilesList =  "";
 399   if ($CalculateCorrelation || $CalculateRSquare) {
 400     $TextFilesList = $CorrelationTextFile;
 401     if ($CalculateRSquare) {
 402       $TextFilesList .= ", $CorrelationTextFile";
 403     }
 404   }
 405   $Delimiter = length($TextFilesList) ? "," : "";
 406   if ($CalculateCovariance) {
 407     $TextFilesList .= "${Delimiter} ${CorrelationTextFile}";
 408   }
 409   if ($TextFilesList =~ /\,/) {
 410     print "Generating new text files $TextFilesList...\n"
 411   }
 412   else {
 413     print "Generating new text file $TextFilesList...\n"
 414   }
 415   if ($CalculateCorrelation || $CalculateRSquare) {
 416     open CORRELATIONTEXTFILE, ">$CorrelationTextFile" or die "Error: Can't open $CorrelationTextFile: $! \n";
 417     if ($CalculateRSquare) {
 418       open RSQUARETEXTFILE, ">$RSquareTextFile" or die "Error: Can't open $RSquareTextFile: $! \n";
 419     }
 420   }
 421   if ($CalculateCovariance) {
 422     open COVARIANCETEXTFILE, ">$CovarianceTextFile" or die "Error: Can't open $CovarianceTextFile: $! \n";
 423   }
 424 
 425   my($Line, $Value, $CorrelationValue, $RSquareValue, $CovarianceValue, $ColNum, $ColNum1, $ColNum2, $ColValuesRef1, $ColValuesRef2, @ColLabels, @CovarianceRowValues, @CorrelationRowValues, @RSquareRowValues);
 426 
 427   # Write out the column labels...
 428   @ColLabels = ();
 429   push @ColLabels, "";
 430   for $ColNum (0 .. ($TextFilesInfo{ColCount}[$Index] - 1)) {
 431     push @ColLabels, $TextFilesInfo{ColLabels}[$Index][$ColNum];
 432   }
 433   $Line = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
 434   if ($CalculateCorrelation || $CalculateRSquare) {
 435     print CORRELATIONTEXTFILE "$Line\n";
 436     if ($CalculateRSquare) {
 437       print RSQUARETEXTFILE "$Line\n";
 438     }
 439   }
 440   if ($CalculateCovariance) {
 441     print COVARIANCETEXTFILE "$Line\n";
 442   }
 443 
 444   # Due to symmetric nature of these matrices, only one half needs to be
 445   # calculated. So, just calculate the lower half and copy it to upper half...
 446   my(%CorrelationMatrixMap, %RSquareMatrixMap, %CovarianceMatrixMap);
 447 
 448   %CorrelationMatrixMap = (); %RSquareMatrixMap = (); %CovarianceMatrixMap = ();
 449   for $ColNum1 (0 .. ($TextFilesInfo{ColCount}[$Index] - 1)) {
 450     for $ColNum2 (0 .. $ColNum1) {
 451       $ColValuesRef1 =  \@{$ColValuesToAnalyzeMapRef->{$ColNum1}};
 452       $ColValuesRef2 =  \@{$ColValuesToAnalyzeMapRef->{$ColNum2}};
 453       if ($CalculateCorrelation || $CalculateRSquare) {
 454         $CorrelationValue = Correlation($ColValuesRef1, $ColValuesRef2);
 455         $CorrelationValue = (defined($CorrelationValue) && length($CorrelationValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $CorrelationValue) + 0) : "";
 456         $CorrelationMatrixMap{$ColNum1}{$ColNum2} = $CorrelationValue;
 457         if ($ColNum1 != $ColNum2) {
 458           $CorrelationMatrixMap{$ColNum2}{$ColNum1} = $CorrelationValue;
 459         }
 460         if ($CalculateRSquare) {
 461           $RSquareValue = (defined($CorrelationValue) && length($CorrelationValue)) ? ($CorrelationValue ** 2) : "";
 462           $RSquareValue = (length($RSquareValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $RSquareValue) + 0) : "";
 463           $RSquareMatrixMap{$ColNum1}{$ColNum2} = $RSquareValue;
 464           if ($ColNum1 != $ColNum2) {
 465             $RSquareMatrixMap{$ColNum2}{$ColNum1} = $RSquareValue;
 466           }
 467         }
 468       }
 469       if ($CalculateCovariance) {
 470         $CovarianceValue = Covariance($ColValuesRef1, $ColValuesRef2);
 471         $CovarianceValue = (defined($CovarianceValue) && length($CovarianceValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $CovarianceValue) + 0) : "";
 472         $CovarianceMatrixMap{$ColNum1}{$ColNum2} = $CovarianceValue;
 473         if ($ColNum1 != $ColNum2) {
 474           $CovarianceMatrixMap{$ColNum2}{$ColNum1} = $CovarianceValue;
 475         }
 476       }
 477     }
 478   }
 479 
 480   # Write out the matrices...
 481   for $ColNum1 (0 .. ($TextFilesInfo{ColCount}[$Index] - 1)) {
 482     @CorrelationRowValues = ();
 483     @RSquareRowValues = ();
 484     @CovarianceRowValues = ();
 485     if ($CalculateCorrelation || $CalculateRSquare) {
 486       push @CorrelationRowValues, $TextFilesInfo{ColLabels}[$Index][$ColNum1];
 487       if ($CalculateRSquare) {
 488         push @RSquareRowValues, $TextFilesInfo{ColLabels}[$Index][$ColNum1];
 489       }
 490     }
 491     if ($CalculateCovariance) {
 492       push @CovarianceRowValues, $TextFilesInfo{ColLabels}[$Index][$ColNum1];
 493     }
 494     for $ColNum2 (0 .. ($TextFilesInfo{ColCount}[$Index] - 1)) {
 495       if ($CalculateCorrelation || $CalculateRSquare) {
 496         push @CorrelationRowValues, $CorrelationMatrixMap{$ColNum1}{$ColNum2};
 497         if ($CalculateRSquare) {
 498           push @RSquareRowValues, $RSquareMatrixMap{$ColNum1}{$ColNum2};
 499         }
 500       }
 501       if ($CalculateCovariance) {
 502         push @CovarianceRowValues, $CovarianceMatrixMap{$ColNum1}{$ColNum2};
 503       }
 504     }
 505     if ($CalculateCorrelation || $CalculateRSquare) {
 506       $Line = JoinWords(\@CorrelationRowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
 507       print CORRELATIONTEXTFILE "$Line\n";
 508       if ($CalculateRSquare) {
 509         $Line = JoinWords(\@RSquareRowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
 510         print RSQUARETEXTFILE "$Line\n";
 511       }
 512     }
 513     if ($CalculateCovariance) {
 514       $Line = JoinWords(\@CovarianceRowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
 515       print COVARIANCETEXTFILE "$Line\n";
 516     }
 517   }
 518   if ($CalculateCorrelation || $CalculateRSquare) {
 519     close CORRELATIONTEXTFILE;
 520     if ($CalculateRSquare) {
 521       close RSQUARETEXTFILE;
 522     }
 523   }
 524   if ($CalculateCovariance) {
 525     close COVARIANCETEXTFILE;
 526   }
 527 }
 528 
 529 # Calculate standard scores...
 530 sub PerformStandardScoresAnalysis {
 531   my($Index, $ColValuesToAnalyzeMapRef) = @_;
 532   my($StandardScores, $StandardScoresN, $NewTextFile, @ColLabels, $Label, $NewLine);
 533 
 534   $StandardScores = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{standardscores}) ? 1 : 0;
 535   $StandardScoresN = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{standardscoresn}) ? 1 : 0;
 536 
 537   $NewTextFile = $TextFilesInfo{OutFileRoot}[$Index] . "StandardScores." .  $TextFilesInfo{OutFileExt}[$Index];
 538   print "Generating new text file $NewTextFile...\n";
 539   open NEWTEXTFILE, ">$NewTextFile" or die "Error: Can't open $NewTextFile: $! \n";
 540 
 541   my($ColValuesRef, $ColNum, @ColNumsToAnalyze);
 542   # Write out column labels...
 543   @ColLabels = ();
 544   @ColNumsToAnalyze = @{$TextFilesInfo{ColNumsToAnalyze}[$Index]};
 545   for $ColNum (@ColNumsToAnalyze) {
 546     $Label = $TextFilesInfo{ColLabels}[$Index][$ColNum];
 547     if ($StandardScores) {
 548       push @ColLabels, "${Label}\(StandardScores)";
 549     }
 550     if ($StandardScoresN) {
 551       push @ColLabels, "${Label}\(StandardScoresN)";
 552     }
 553   }
 554   $NewLine = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
 555   print NEWTEXTFILE "$NewLine\n";
 556 
 557   # Go over each column to be analyzed and calculate standard deviation
 558   # and mean values...
 559   my(%StandardDeviationMap, %StandardDeviationNMap, %MeanMap);
 560   %StandardDeviationMap = ();
 561   %StandardDeviationNMap = ();
 562   %MeanMap = ();
 563   for $ColNum (@ColNumsToAnalyze) {
 564     $ColValuesRef =  \@{$ColValuesToAnalyzeMapRef->{$ColNum}};
 565     if (!exists($MeanMap{$ColNum})) {
 566       $MeanMap{$ColNum} = Mean($ColValuesRef);
 567     }
 568     if ($StandardScores) {
 569       if (!exists($StandardDeviationMap{$ColNum})) {
 570         $StandardDeviationMap{$ColNum} = StandardDeviation($ColValuesRef);
 571       }
 572     }
 573     if ($StandardScoresN) {
 574       if (!exists($StandardDeviationNMap{$ColNum})) {
 575         $StandardDeviationNMap{$ColNum} = StandardDeviationN($ColValuesRef);
 576       }
 577     }
 578   }
 579   #
 580   # Go over each row and calculate standard scores for each column
 581   # using (x[i] - mean) / (n - 1) for StandardScores and (x[i] - mean) / n
 582   # for StandardScoresN; write out the calculated values as well...
 583 
 584   my($TextFile, $InDelim, $Line, $Value, $ValueOkay, $ScoreValue, @RowValues, @LineWords);
 585   $TextFile = $TextFilesList[$Index];
 586   $InDelim = $TextFilesInfo{InDelim}[$Index];
 587 
 588   open TEXTFILE, "$TextFile" or die "Error: Can't open $TextFile: $! \n";
 589   $Line = GetTextLine(\*TEXTFILE);
 590   while ($Line = GetTextLine(\*TEXTFILE)) {
 591     @LineWords = quotewords($InDelim, 0, $Line);
 592     @RowValues = ();
 593     COLNUM: for $ColNum (@ColNumsToAnalyze) {
 594       $Value = $LineWords[$ColNum];
 595       $ValueOkay = ($OptionsInfo{CheckData} && !IsNumerical($Value)) ? 0 : 1;
 596       if ($StandardScores) {
 597         $ScoreValue = $ValueOkay ? (($Value - $MeanMap{$ColNum})/$StandardDeviationMap{$ColNum}) : "";
 598         $ScoreValue = (defined($ScoreValue) && length($ScoreValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $ScoreValue) + 0) : "";
 599         push @RowValues, $ScoreValue;
 600       }
 601       if ($StandardScoresN) {
 602         $ScoreValue = $ValueOkay ? (($Value - $MeanMap{$ColNum})/$StandardDeviationNMap{$ColNum}) : "";
 603         $ScoreValue = (defined($ScoreValue) && length($ScoreValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $ScoreValue) + 0) : "";
 604         push @RowValues, $ScoreValue;
 605       }
 606     }
 607     $NewLine = JoinWords(\@RowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
 608     print NEWTEXTFILE "$NewLine\n";
 609   }
 610   close TEXTFILE;
 611   close NEWTEXTFILE;
 612 }
 613 
 614 # Make sure the specified columns exists in text files...
 615 sub ProcessColumnsInfo {
 616   my($Index, $TextFile, $ColNum, $NewColNum, $ColIndex, @ColNumsToAnalyze, %UniqueColNumsToAnalyzeMap);
 617 
 618   @{$TextFilesInfo{ColNumsToAnalyze}} = ();
 619   @{$TextFilesInfo{ColPairs1ToAnalyze}} = ();
 620   @{$TextFilesInfo{ColPairs2ToAnalyze}} = ();
 621   @{$TextFilesInfo{UniqueColNumsToAnalyze}} = ();
 622 
 623   FILELIST: for $Index (0 .. $#TextFilesList) {
 624     $TextFile = $TextFilesList[$Index];
 625 
 626     @{$TextFilesInfo{ColNumsToAnalyze}[$Index]} = ();
 627     @{$TextFilesInfo{ColPairs1ToAnalyze}[$Index]} = ();
 628     @{$TextFilesInfo{ColPairs2ToAnalyze}[$Index]} = ();
 629     @{$TextFilesInfo{UniqueColNumsToAnalyze}[$Index]} = ();
 630 
 631     %UniqueColNumsToAnalyzeMap = ();
 632 
 633     if ($TextFilesInfo{FileOkay}[$Index]) {
 634       @ColNumsToAnalyze = ();
 635       if (@{$OptionsInfo{SpecifiedColumns}}) {
 636         if ($OptionsInfo{ColMode} =~ /^colnum$/i) {
 637           for $ColNum (@{$OptionsInfo{SpecifiedColumns}}) {
 638             if ($ColNum >=1 && $ColNum <= $TextFilesInfo{ColCount}[$Index]) {
 639               $NewColNum = $ColNum -1;
 640               push @ColNumsToAnalyze, $NewColNum;
 641             }
 642           }
 643         }
 644         else {
 645           my($ColLabel);
 646           for $ColLabel (@{$OptionsInfo{SpecifiedColumns}}) {
 647             if (exists($TextFilesInfo{ColLabelToNumMap}[$Index]{$ColLabel})) {
 648               push @ColNumsToAnalyze, $TextFilesInfo{ColLabelToNumMap}[$Index]{$ColLabel};
 649             }
 650           }
 651         }
 652       }
 653       elsif (defined  $OptionsInfo{Columns} && $OptionsInfo{Columns} =~ /^All$/i) {
 654         for $ColNum (0 .. ($TextFilesInfo{ColCount}[$Index] - 1)) {
 655           push @ColNumsToAnalyze, $ColNum;
 656         }
 657       }
 658       else {
 659         push @ColNumsToAnalyze, 0;
 660       }
 661       if (@ColNumsToAnalyze) {
 662         push @{$TextFilesInfo{ColNumsToAnalyze}[$Index]}, @ColNumsToAnalyze;
 663         # Set up unique columns map as well...
 664         for $ColNum (@ColNumsToAnalyze) {
 665           if (!exists $UniqueColNumsToAnalyzeMap{$ColNum}) {
 666             $UniqueColNumsToAnalyzeMap{$ColNum} = $ColNum;
 667           }
 668         }
 669       }
 670       else {
 671         warn "Warning: Ignoring file $TextFile: None of the columns specified, @{$OptionsInfo{SpecifiedColumns}}, using \"--columns\" option exist.\n";
 672         $TextFilesInfo{FileOkay}[$Index] = 0;
 673         next FILELIST;
 674       }
 675       if (!$OptionsInfo{Overwrite} && exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{frequency})) {
 676         # Make sure specific frequency files don't exist...
 677         my($FrequencyFile);
 678         for $ColNum (@ColNumsToAnalyze) {
 679           $FrequencyFile = $TextFilesInfo{OutFileRoot}[$Index] . $TextFilesInfo{ColLabels}[$Index][$ColNum] . "FrequencyAnalysis." .  $TextFilesInfo{OutFileExt}[$Index];
 680           if (-e $FrequencyFile) {
 681             warn "Warning: Ignoring file $TextFile: The file $FrequencyFile already exists.\n";
 682             $TextFilesInfo{FileOkay}[$Index] = 0;
 683             next FILELIST;
 684           }
 685         }
 686       }
 687       # Setup specified column pairs...
 688       if (exists $OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation} || exists $OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance} || exists $OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare}) {
 689         my(@ColPairsToAnalyze, $ColNum1, $ColNum2);
 690         if (@{$OptionsInfo{SpecifiedColumnPairs}}) {
 691           # Make sure both columns exist...
 692           if ($OptionsInfo{ColMode} =~ /^colnum$/i) {
 693             for ($ColIndex = 0; (($ColIndex + 1) < @{$OptionsInfo{SpecifiedColumnPairs}}); $ColIndex += 2 ) {
 694               $ColNum1 = $OptionsInfo{SpecifiedColumnPairs}[$ColIndex];
 695               $ColNum2 = $OptionsInfo{SpecifiedColumnPairs}[$ColIndex + 1];
 696               if ($ColNum1 >=1 && $ColNum1 <= $TextFilesInfo{ColCount}[$Index] && $ColNum2 >=1 && $ColNum2 <= $TextFilesInfo{ColCount}[$Index]) {
 697                 $ColNum1 -= 1;
 698                 $ColNum2 -= 1;
 699                 push @ColPairsToAnalyze, ($ColNum1, $ColNum2);
 700               }
 701             }
 702           }
 703           else {
 704             my($ColLabel1, $ColLabel2);
 705             for ($ColIndex = 0; (($ColIndex + 1) < @{$OptionsInfo{SpecifiedColumnPairs}}); $ColIndex += 2 ) {
 706               $ColLabel1 = $OptionsInfo{SpecifiedColumnPairs}[$ColIndex];
 707               $ColLabel2 = $OptionsInfo{SpecifiedColumnPairs}[$ColIndex + 1];
 708               if (exists($TextFilesInfo{ColLabelToNumMap}[$Index]{$ColLabel1}) && exists($TextFilesInfo{ColLabelToNumMap}[$Index]{$ColLabel2})) {
 709                 $ColNum1 = $TextFilesInfo{ColLabelToNumMap}[$Index]{$ColLabel1};
 710                 $ColNum2 = $TextFilesInfo{ColLabelToNumMap}[$Index]{$ColLabel2};
 711                 push @ColPairsToAnalyze, ($ColNum1, $ColNum2);
 712               }
 713             }
 714           }
 715         }
 716         elsif ($OptionsInfo{AllColumnPairs}) {
 717           for $ColNum1 (0 .. ($TextFilesInfo{ColCount}[$Index] - 1)) {
 718             for $ColNum2 (0 .. ($TextFilesInfo{ColCount}[$Index] - 1)) {
 719               push @ColPairsToAnalyze, ($ColNum1, $ColNum2);
 720             }
 721           }
 722         }
 723         else {
 724           if ($TextFilesInfo{ColCount}[$Index] >= 2) {
 725             push @ColPairsToAnalyze, (0,1);
 726           }
 727         }
 728         if (@ColPairsToAnalyze) {
 729           if (@ColPairsToAnalyze % 2) {
 730             warn "Warning: Ignoring file $TextFile: Invalid number of values specified using \"--columnpairs\" option: It must contain even number of valid values.\n";
 731             $TextFilesInfo{FileOkay}[$Index] = 0;
 732             next FILELIST;
 733           }
 734           else {
 735             for ($ColIndex = 0; $ColIndex < @ColPairsToAnalyze; $ColIndex += 2) {
 736               push @{$TextFilesInfo{ColPairs1ToAnalyze}[$Index]}, $ColPairsToAnalyze[$ColIndex];
 737               push @{$TextFilesInfo{ColPairs2ToAnalyze}[$Index]}, $ColPairsToAnalyze[$ColIndex + 1];
 738             }
 739             # Set up unique columns map as well...
 740             for $ColNum (@ColPairsToAnalyze) {
 741               if (!exists $UniqueColNumsToAnalyzeMap{$ColNum}) {
 742                 $UniqueColNumsToAnalyzeMap{$ColNum} = $ColNum;
 743               }
 744             }
 745           }
 746         }
 747       }
 748       # Setup uniques columns array...
 749       push @{$TextFilesInfo{UniqueColNumsToAnalyze}[$Index]}, (sort keys %UniqueColNumsToAnalyzeMap);
 750     }
 751   }
 752 }
 753 
 754 # Retrieve information about input text files...
 755 sub RetrieveTextFilesInfo {
 756   my($Index, $TextFile, $FileDir, $FileName, $FileExt, $InDelim, $Line, @ColLabels, $OutFileRoot,  $OutFile, $OutFileExt, $ColNum, $ColLabel);
 757 
 758   %TextFilesInfo = ();
 759 
 760   @{$TextFilesInfo{FileOkay}} = ();
 761   @{$TextFilesInfo{ColCount}} = ();
 762   @{$TextFilesInfo{ColLabels}} = ();
 763   @{$TextFilesInfo{ColLabelToNumMap}} = ();
 764   @{$TextFilesInfo{InDelim}} = ();
 765   @{$TextFilesInfo{OutFileRoot}} = ();
 766   @{$TextFilesInfo{OutFileExt}} = ();
 767 
 768   FILELIST: for $Index (0 .. $#TextFilesList) {
 769     $TextFile = $TextFilesList[$Index];
 770 
 771     $TextFilesInfo{FileOkay}[$Index] = 0;
 772     $TextFilesInfo{ColCount}[$Index] = 0;
 773     $TextFilesInfo{InDelim}[$Index] = "";
 774     $TextFilesInfo{OutFileRoot}[$Index] = "";
 775     $TextFilesInfo{OutFileExt}[$Index] = "";
 776 
 777     @{$TextFilesInfo{ColLabels}[$Index]} = ();
 778     %{$TextFilesInfo{ColLabelToNumMap}[$Index]} = ();
 779 
 780     if (!(-e $TextFile)) {
 781       warn "Warning: Ignoring file $TextFile: It doesn't exist\n";
 782       next FILELIST;
 783     }
 784     if (!CheckFileType($TextFile, "csv tsv")) {
 785       warn "Warning: Ignoring file $TextFile: It's not a csv or tsv file\n";
 786       next FILELIST;
 787     }
 788     ($FileDir, $FileName, $FileExt) = ParseFileName($TextFile);
 789     if ($FileExt =~ /^tsv$/i) {
 790       $InDelim = "\t";
 791     }
 792     else {
 793       $InDelim = "\,";
 794       if ($Options{indelim} !~ /^(comma|semicolon)$/i) {
 795         warn "Warning: Ignoring file $TextFile: The value specified, $Options{indelim}, for option \"--indelim\" is not valid for csv files\n";
 796         next FILELIST;
 797       }
 798       if ($Options{indelim} =~ /^semicolon$/i) {
 799         $InDelim = "\;";
 800       }
 801     }
 802 
 803     if (!open TEXTFILE, "$TextFile") {
 804       warn "Warning: Ignoring file $TextFile: Couldn't open it: $! \n";
 805       next FILELIST;
 806     }
 807 
 808     $Line = GetTextLine(\*TEXTFILE);
 809     @ColLabels = quotewords($InDelim, 0, $Line);
 810     close TEXTFILE;
 811 
 812     $FileDir = ""; $FileName = ""; $FileExt = "";
 813     ($FileDir, $FileName, $FileExt) = ParseFileName($TextFile);
 814     $FileExt = "csv";
 815     if ($Options{outdelim} =~ /^tab$/i) {
 816       $FileExt = "tsv";
 817     }
 818     $OutFileExt = $FileExt;
 819     if ($Options{root} && (@TextFilesList == 1)) {
 820       my ($RootFileDir, $RootFileName, $RootFileExt) = ParseFileName($Options{root});
 821       if ($RootFileName && $RootFileExt) {
 822         $FileName = $RootFileName;
 823       }
 824       else {
 825         $FileName = $Options{root};
 826       }
 827       $OutFileRoot = $FileName;
 828     }
 829     else {
 830       $OutFileRoot = $FileName;
 831     }
 832     $OutFile = $OutFileRoot . $OptionsInfo{FileNameMode} . ".$OutFileExt";
 833 
 834     if (lc($OutFile) eq lc($TextFile)) {
 835       warn "Warning: Ignoring file $TextFile:Output file name, $OutFile, is same as input text file name, $TextFile\n";
 836       next FILELIST;
 837     }
 838     if (!$Options{overwrite}) {
 839       if (-e $OutFile) {
 840         warn "Warning: Ignoring file $TextFile: The file $OutFile already exists\n";
 841         next FILELIST;
 842       }
 843       if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance}) || exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation}) || exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare})) {
 844         if ($OptionsInfo{AllColumnPairs}) {
 845           if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance}) && (-e "${OutFileRoot}CovarianceMatrix.${FileExt}")) {
 846             warn "Warning: Ignoring file $TextFile: The file ${OutFileRoot}Covariance.${FileExt} already exists.\n";
 847             next FILELIST;
 848           }
 849           if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation}) && (-e "${OutFileRoot}CorrelationMatrix.${FileExt}")) {
 850             warn "Warning: Ignoring file $TextFile: The file ${OutFileRoot}CorrelationMatrix.${FileExt} already exists.\n";
 851             next FILELIST;
 852           }
 853           if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare}) && (-e "${OutFileRoot}RSquareMatrix.${FileExt}")) {
 854             warn "Warning: Ignoring file $TextFile: The file ${OutFileRoot}RSquareMatrix.${FileExt} already exists.\n";
 855             next FILELIST;
 856           }
 857         }
 858         else {
 859           if (-e "${OutFileRoot}ColumnPairsAnalysis.${FileExt}") {
 860             warn "Warning: Ignoring file $TextFile: The file ${OutFileRoot}ColumnPairsAnalysis.${FileExt} already exists.\n";
 861             next FILELIST;
 862           }
 863         }
 864       }
 865       if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{standardscores}) && (-e "${OutFileRoot}StandardScores.${FileExt}")) {
 866         warn "Warning: Ignoring file $TextFile: The file ${OutFileRoot}StandardScores.${FileExt} already exists.\n";
 867         next FILELIST;
 868       }
 869     }
 870 
 871     $TextFilesInfo{FileOkay}[$Index] = 1;
 872     $TextFilesInfo{InDelim}[$Index] = $InDelim;
 873     $TextFilesInfo{OutFileRoot}[$Index] = "$OutFileRoot";
 874     $TextFilesInfo{OutFileExt}[$Index] = "$OutFileExt";
 875 
 876     $TextFilesInfo{ColCount}[$Index] = @ColLabels;
 877     push @{$TextFilesInfo{ColLabels}[$Index]}, @ColLabels;
 878     for $ColNum (0 .. $#ColLabels) {
 879       $ColLabel = $ColLabels[$ColNum];
 880       $TextFilesInfo{ColLabelToNumMap}[$Index]{$ColLabel} = $ColNum;
 881     }
 882   }
 883 }
 884 
 885 # Process option values...
 886 sub ProcessOptions {
 887   %OptionsInfo = ();
 888 
 889   $OptionsInfo{Mode} = $Options{mode};
 890 
 891   $OptionsInfo{DetailLevel} = $Options{detail};
 892 
 893   # Setup supported statistical functions...
 894   my($SupportedFunction, @SupportedStatisticaFunctions, %SupportedStatisticaFunctionsMap);
 895   %SupportedStatisticaFunctionsMap = ();
 896   @SupportedStatisticaFunctions = qw(Average AverageDeviation Correlation Count Covariance GeometricMean Frequency HarmonicMean KLargest KSmallest Kurtosis Maximum Minimum Mean Median Mode RSquare Skewness Sum SumOfSquares StandardDeviation StandardDeviationN StandardError StandardScores StandardScoresN TrimMean Variance VarianceN);
 897 
 898   for $SupportedFunction (@SupportedStatisticaFunctions) {
 899     $SupportedStatisticaFunctionsMap{lc($SupportedFunction)} = $SupportedFunction;
 900   }
 901 
 902   # Setup a list of functions to use for analysis...
 903   my($SpecifiedFunction);
 904   %{$OptionsInfo{SpecifiedStatisticalFunctionsMap}} = ();
 905   @{$OptionsInfo{SpecifiedStatisticalFunctions}} = ();
 906   # Check mode values...
 907   if ($Options{mode} =~ /^DescriptiveStatisticsBasic$/i ) {
 908     $OptionsInfo{FileNameMode} = "DescriptiveStatisticsBasic";
 909     @{$OptionsInfo{SpecifiedStatisticalFunctions}} = qw(Count Maximum Minimum Mean Median StandardDeviation StandardError Variance Sum);
 910   }
 911   elsif ($Options{mode} =~ /^DescriptiveStatisticsAll$/i ) {
 912     $OptionsInfo{FileNameMode} = "DescriptiveStatisticsAll";
 913     @{$OptionsInfo{SpecifiedStatisticalFunctions}} = qw(Count Maximum Minimum Mean GeometricMean HarmonicMean TrimMean Median Mode StandardDeviation Kurtosis Skewness StandardError Variance  RSquare Frequency  KLargest KSmallest Sum);
 914   }
 915   elsif ($Options{mode} =~ /^All$/i ) {
 916     $OptionsInfo{FileNameMode} = "AllStatistics";
 917     @{$OptionsInfo{SpecifiedStatisticalFunctions}} = @SupportedStatisticaFunctions;
 918   }
 919   else {
 920     $OptionsInfo{FileNameMode} = "SpecifiedStatistics";
 921     # Comma delimited list of functions...
 922     my($Mode, @SpecifiedFunctions, @UnsupportedSpecifiedFunctions);
 923     $Mode = $Options{mode};
 924     $Mode =~ s/ //g;
 925     @SpecifiedFunctions = split ",", $Mode;
 926     @UnsupportedSpecifiedFunctions = ();
 927     for $SpecifiedFunction (@SpecifiedFunctions) {
 928       if (exists($SupportedStatisticaFunctionsMap{lc($SpecifiedFunction)})) {
 929         push @{$OptionsInfo{SpecifiedStatisticalFunctions}}, $SpecifiedFunction;
 930       }
 931       else {
 932         push @UnsupportedSpecifiedFunctions, $SpecifiedFunction;
 933       }
 934     }
 935     if (@UnsupportedSpecifiedFunctions) {
 936       if (@UnsupportedSpecifiedFunctions > 1) {
 937         warn "Error: The values specified - ", JoinWords(\@UnsupportedSpecifiedFunctions, ", ", 0)," - for option \"-m --mode\" are not valid.\n";
 938       }
 939       else {
 940         warn "Error: The value specified, @UnsupportedSpecifiedFunctions , for option \"-m --mode\" is not valid.\n";
 941       }
 942       die "Allowed values:", JoinWords(\@SupportedStatisticaFunctions, ", ", 0), "\n";
 943     }
 944   }
 945   FUNCTION: for $SpecifiedFunction (@{$OptionsInfo{SpecifiedStatisticalFunctions}}) {
 946     if (exists $OptionsInfo{SpecifiedStatisticalFunctionsMap}{lc($SpecifiedFunction)} ) {
 947       next FUNCTION;
 948     }
 949     $OptionsInfo{SpecifiedStatisticalFunctionsMap}{lc($SpecifiedFunction)} = $SupportedStatisticaFunctionsMap{lc($SpecifiedFunction)};
 950   }
 951 
 952   $OptionsInfo{OutDelim} = ($Options{outdelim} =~ /tab/i ) ? "\t" : (($Options{outdelim} =~ /semicolon/i) ? "\;" : "\,");
 953   $OptionsInfo{OutQuote} = ($Options{quote} =~ /yes/i ) ? 1 : 0;
 954 
 955   $OptionsInfo{Overwrite} = defined $Options{overwrite} ? $Options{overwrite} : undef;
 956   $OptionsInfo{Root} = defined $Options{root} ? $Options{root} : undef;
 957 
 958   $OptionsInfo{CheckData} = $Options{fast} ? 0 : 1;
 959   $OptionsInfo{Precision} = $Options{precision};
 960 
 961   $OptionsInfo{KLargest} = $Options{klargest};
 962   $OptionsInfo{KSmallest} = $Options{ksmallest};
 963 
 964   $OptionsInfo{TrimFraction} = $Options{trimfraction};
 965 
 966   # Setup frequency bin values...
 967   $OptionsInfo{NumOfBins} = 10;
 968   @{$OptionsInfo{BinRange}} = ();
 969   if ($Options{frequencybins} =~ /\,/) {
 970     my($BinValue, @SpecifiedBinRange);
 971     @SpecifiedBinRange = split /\,/,  $Options{frequencybins};
 972     if (@SpecifiedBinRange < 2) {
 973       die "Error: The value specified, $Options{frequencybins}, for option \"--frequencybins\" is not valid: Must contain at least two values. \n";
 974     }
 975     for $BinValue (@SpecifiedBinRange) {
 976       if (!IsNumerical($BinValue)) {
 977         die "Error: The value specified, $Options{frequencybins}, for option \"--frequencybins\" is not valid: Contains non numeric values. \n";
 978       }
 979     }
 980     my($Index1, $Index2);
 981     for $Index1 (0 .. $#SpecifiedBinRange) {
 982       for $Index2 (($Index1 + 1) .. $#SpecifiedBinRange) {
 983         if ($SpecifiedBinRange[$Index1] >= $SpecifiedBinRange[$Index2]) {
 984           die "Error: The value specified, $Options{frequencybins}, for option \"--frequencybins\" is not valid: Must contain values in ascending order. \n";
 985         }
 986       }
 987     }
 988     push @{$OptionsInfo{BinRange}}, @SpecifiedBinRange;
 989   }
 990   else {
 991     $OptionsInfo{NumOfBins} = $Options{frequencybins};
 992     if (!IsPositiveInteger($OptionsInfo{NumOfBins})) {
 993       die "Error: The value specified, $Options{frequencybins}, for option \"--frequencybins\" is not valid. Allowed values: positive integer or \"number,number,[number]...\". \n";
 994     }
 995   }
 996 
 997   # Setup specified columns...
 998   $OptionsInfo{ColMode} = $Options{colmode};
 999   $OptionsInfo{Columns} = defined $Options{columns} ? $Options{columns} : undef;
1000 
1001   @{$OptionsInfo{SpecifiedColumns}} = ();
1002   if (defined $Options{columns} && $Options{columns} !~ /^All$/i) {
1003     my(@SpecifiedValues) = split ",", $Options{columns};
1004     if ($Options{colmode} =~ /^colnum$/i) {
1005       my($ColValue);
1006       for $ColValue (@SpecifiedValues) {
1007         if (!IsPositiveInteger($ColValue)) {
1008           die "Error: Column value, $ColValue, specified using \"--columns\" is not valid: Allowed integer values: > 0.\n";
1009         }
1010       }
1011     }
1012     push @{$OptionsInfo{SpecifiedColumns}}, @SpecifiedValues;
1013   }
1014   @{$OptionsInfo{SpecifiedColumnPairs}} = ();
1015   $OptionsInfo{AllColumnPairs} = (defined($Options{columnpairs}) && $Options{columnpairs} =~ /^AllPairs$/i) ? 1 : 0;
1016   if (defined($Options{columnpairs}) && !$OptionsInfo{AllColumnPairs}) {
1017     my(@SpecifiedValues) = split ",", $Options{columnpairs};
1018     if (@SpecifiedValues % 2) {
1019       die "Error: Invalid number of values specified using \"--columnpairs\" option: It must contain even number of values.\n";
1020     }
1021     if ($Options{colmode} =~ /^colnum$/i) {
1022       my($ColValue);
1023       for $ColValue (@SpecifiedValues) {
1024         if (!IsPositiveInteger($ColValue)) {
1025           die "Error: Column value, $ColValue, specified using \"--columnpairs\" is not valid: Allowed integer values: > 0.\n";
1026         }
1027       }
1028     }
1029     push @{$OptionsInfo{SpecifiedColumnPairs}}, @SpecifiedValues;
1030   }
1031 
1032 }
1033 
1034 # Setup script usage  and retrieve command line arguments specified using various options...
1035 sub SetupScriptUsage {
1036 
1037   # Retrieve all the options...
1038   %Options = ();
1039   $Options{colmode} = "colnum";
1040   $Options{detail} = 1;
1041   $Options{indelim} = "comma";
1042   $Options{frequencybins} = 10;
1043   $Options{klargest} = 2;
1044   $Options{ksmallest} = 2;
1045   $Options{mode} = "DescriptiveStatisticsBasic";
1046   $Options{outdelim} = "comma";
1047   $Options{precision} = 2;
1048   $Options{quote} = "yes";
1049   $Options{trimfraction} = 0.1;
1050 
1051   if (!GetOptions(\%Options, "colmode|c=s", "columns=s", "columnpairs=s", "detail|d=i", "frequencybins=s", "fast|f", "help|h", "indelim=s", "klargest=i", "ksmallest=i", "mode|m=s", "outdelim=s", "overwrite|o", "precision|p=i", "quote|q=s", "root|r=s", "trimfraction=f", "workingdir|w=s")) {
1052     die "\nTo get a list of valid options and their values, use \"$ScriptName -h\" or\n\"perl -S $ScriptName -h\" command and try again...\n";
1053   }
1054   if ($Options{workingdir}) {
1055     if (! -d $Options{workingdir}) {
1056       die "Error: The value specified, $Options{workingdir}, for option \"-w --workingdir\" is not a directory name.\n";
1057     }
1058     chdir $Options{workingdir} or die "Error: Couldn't chdir $Options{workingdir}: $! \n";
1059   }
1060   if ($Options{colmode} !~ /^(colnum|collabel)$/i) {
1061     die "Error: The value specified, $Options{colmode}, for option \"-c --colmode\" is not valid. Allowed values: colnum or collabel\n";
1062   }
1063   if (!IsPositiveInteger($Options{detail})) {
1064     die "Error: The value specified, $Options{detail}, for option \"-d --detail\" is not valid. Allowed values: > 0\n";
1065   }
1066   if ($Options{indelim} !~ /^(comma|semicolon)$/i) {
1067     die "Error: The value specified, $Options{indelim}, for option \"--indelim\" is not valid. Allowed values: comma or semicolon\n";
1068   }
1069   if ($Options{outdelim} !~ /^(comma|semicolon|tab)$/i) {
1070     die "Error: The value specified, $Options{outdelim}, for option \"--outdelim\" is not valid. Allowed values: comma, tab, or semicolon\n";
1071   }
1072   if ($Options{quote} !~ /^(yes|no)$/i) {
1073     die "Error: The value specified, $Options{quote}, for option \"-q --quote\" is not valid. Allowed values: yes or no\n";
1074   }
1075   if (!IsPositiveInteger($Options{precision})) {
1076     die "Error: The value specified, $Options{precision}, for option \"-p --precision\" is not valid. Allowed values: > 0 \n";
1077   }
1078   if (!IsPositiveInteger($Options{klargest})) {
1079     die "Error: The value specified, $Options{klargest}, for option \"--klargest\" is not valid. Allowed values: > 0 \n";
1080   }
1081   if (!IsPositiveInteger($Options{ksmallest})) {
1082     die "Error: The value specified, $Options{ksmallest}, for option \"--ksmallest\" is not valid. Allowed values: > 0 \n";
1083   }
1084   if (IsFloat($Options{trimfraction})) {
1085     if ($Options{trimfraction} <= 0 || $Options{trimfraction} >= 1.0) {
1086       die "Error: The value specified, $Options{trimfraction}, for option \"--trimfraction\" is not valid. Allowed values: > 0 and < 1.0\n";
1087     }
1088   }
1089   else {
1090     die "Error: The value specified, $Options{trimfraction}, for option \"--trimfraction\" is not valid. Allowed values: > 0 and < 1.0\n";
1091   }
1092 }
1093