1 #!/usr/bin/perl -w
   2 #
   3 # File: SplitTextFiles.pl
   4 # Author: Manish Sud <msud@san.rr.com>
   5 #
   6 # Copyright (C) 2024 Manish Sud. All rights reserved.
   7 #
   8 # This file is part of MayaChemTools.
   9 #
  10 # MayaChemTools is free software; you can redistribute it and/or modify it under
  11 # the terms of the GNU Lesser General Public License as published by the Free
  12 # Software Foundation; either version 3 of the License, or (at your option) any
  13 # later version.
  14 #
  15 # MayaChemTools is distributed in the hope that it will be useful, but without
  16 # any warranty; without even the implied warranty of merchantability of fitness
  17 # for a particular purpose.  See the GNU Lesser General Public License for more
  18 # details.
  19 #
  20 # You should have received a copy of the GNU Lesser General Public License
  21 # along with MayaChemTools; if not, see <http://www.gnu.org/licenses/> or
  22 # write to the Free Software Foundation Inc., 59 Temple Place, Suite 330,
  23 # Boston, MA, 02111-1307, USA.
  24 #
  25 
  26 use strict;
  27 use FindBin; use lib "$FindBin::Bin/../lib";
  28 use Getopt::Long;
  29 use File::Basename;
  30 use Text::ParseWords;
  31 use Benchmark;
  32 use FileUtil;
  33 use TextUtil;
  34 
  35 my($ScriptName, %Options, $StartTime, $EndTime, $TotalTime);
  36 
  37 # Autoflush STDOUT
  38 $| = 1;
  39 
  40 # Starting message...
  41 $ScriptName = basename $0;
  42 print "\n$ScriptName:Starting...\n\n";
  43 $StartTime = new Benchmark;
  44 
  45 # Get the options and setup script...
  46 SetupScriptUsage();
  47 if ($Options{help} || @ARGV < 1) {
  48   die GetUsageFromPod("$FindBin::Bin/$ScriptName");
  49 }
  50 
  51 my(@TextFilesList);
  52 @TextFilesList = ExpandFileNames(\@ARGV, "csv tsv");
  53 
  54 # Process options...
  55 print "Processing options...\n";
  56 my(%OptionsInfo);
  57 ProcessOptions();
  58 
  59 print "Checking input text file(s)...\n";
  60 my(%TextFilesInfo);
  61 RetrieveTextFilesInfo();
  62 
  63 # Generate output files...
  64 my($FileIndex);
  65 if (@TextFilesList > 1) {
  66   print "\nProcessing text files...\n";
  67 }
  68 for $FileIndex (0 .. $#TextFilesList) {
  69   if ($TextFilesInfo{FileOkay}[$FileIndex]) {
  70     print "\nProcessing file $TextFilesList[$FileIndex]...\n";
  71     SplitTextFile($FileIndex);
  72   }
  73 }
  74 
  75 print "\n$ScriptName:Done...\n\n";
  76 
  77 $EndTime = new Benchmark;
  78 $TotalTime = timediff ($EndTime, $StartTime);
  79 print "Total time: ", timestr($TotalTime), "\n";
  80 
  81 ###############################################################################
  82 
  83 # Split a Text file...
  84 #
  85 sub SplitTextFile {
  86   my($FileIndex) = @_;
  87   my($TextFile, $LineCount, $MaxLinesPerFile, $MaxNumOfFiles);
  88 
  89   $TextFile = $TextFilesList[$FileIndex];
  90 
  91   if (!open TEXTFILE, "$TextFile") {
  92     warn "Warning: Ignoring file $TextFile: Couldn't open it: $! \n";
  93     return;
  94   }
  95 
  96   $MaxNumOfFiles = $OptionsInfo{NumOfFiles};
  97 
  98   # Count number of lines to figure out maximum number of lines per file...
  99   $LineCount = 0;
 100   while (<TEXTFILE>) {
 101       $LineCount++;
 102   }
 103   close TEXTFILE;
 104 
 105   if ($LineCount < $MaxNumOfFiles) {
 106     warn "Warning: Ignoring file $TextFile: Total number of lines, $LineCount, is smaller than\nnumber of new files, $MaxNumOfFiles\n";
 107     return;
 108   }
 109 
 110   $MaxLinesPerFile = int $LineCount / $MaxNumOfFiles;
 111 
 112   GenerateTextFiles($FileIndex, $MaxNumOfFiles, $MaxLinesPerFile);
 113 }
 114 
 115 # Generate new Text files...
 116 #
 117 sub GenerateTextFiles {
 118   my($FileIndex, $NumOfFiles, $NumOfLinesPerFile) = @_;
 119   my($TextFile, $LineCount, $NewFileIndex, $NewFileName, $MaxLinesCount, $InDelim, $OutDelim, $OutQuote, $ColLabelsLine, $Line, @ColLabels, @Words, @NewTextFilesList);
 120 
 121   # Setup new file names list...
 122   @NewTextFilesList = ();
 123   for $NewFileIndex (1 .. $NumOfFiles) {
 124     $NewFileName = $TextFilesInfo{OutFileRoot}[$FileIndex] . "Part${NewFileIndex}." . $TextFilesInfo{OutFileExt}[$FileIndex];
 125     if (!$OptionsInfo{OverwriteFiles}) {
 126       if (-e $NewFileName) {
 127         warn "Warning: Ignoring file $TextFile: New Text file, $NewFileName, already exists\n";
 128         return;
 129       }
 130     }
 131     push @NewTextFilesList, $NewFileName;
 132   }
 133 
 134   $TextFile = $TextFilesList[$FileIndex];
 135 
 136   if (!open TEXTFILE, "$TextFile") {
 137     warn "Warning: Ignoring file $TextFile: Couldn't open it: $! \n";
 138     return;
 139   }
 140 
 141   $InDelim = $TextFilesInfo{InDelim}[$FileIndex];
 142 
 143   $OutDelim = $OptionsInfo{OutDelim};
 144   $OutQuote = $OptionsInfo{OutQuote};
 145 
 146   $MaxLinesCount = $NumOfLinesPerFile;
 147   $LineCount = 0;
 148   $NewFileIndex = 1;
 149 
 150   open NEWTEXTFILE, ">$NewTextFilesList[$NewFileIndex - 1]" or die "Error: Can't open $NewTextFilesList[$NewFileIndex -1]: $! \n";
 151   print "Generating $NewTextFilesList[$NewFileIndex - 1] file...\n";
 152 
 153   if ($OptionsInfo{Label}) {
 154     if ($OptionsInfo{Fast}) {
 155       $ColLabelsLine = GetTextLine(\*TEXTFILE);
 156     }
 157     else {
 158       $Line = GetTextLine(\*TEXTFILE);
 159       @ColLabels = quotewords($InDelim, 0, $Line);
 160       $ColLabelsLine = JoinWords(\@ColLabels, $OutDelim, $OutQuote);
 161     }
 162     print NEWTEXTFILE "$ColLabelsLine\n";
 163   }
 164 
 165   while ($Line = GetTextLine(\*TEXTFILE)) {
 166     $LineCount++;
 167 
 168     if (!$Options{fast}) {
 169       @Words = quotewords($InDelim, 0, $Line);
 170       $Line = JoinWords(\@Words, $OutDelim, $OutQuote);
 171     }
 172     print NEWTEXTFILE "$Line\n";
 173 
 174     if ($NewFileIndex <= $NumOfFiles) {
 175       if ($LineCount >= $MaxLinesCount) {
 176         if ($NewFileIndex < $NumOfFiles) {
 177           close NEWTEXTFILE;
 178         }
 179         $NewFileIndex++;
 180         $MaxLinesCount = $NumOfLinesPerFile * $NewFileIndex;
 181 
 182         if ($NewFileIndex <= $NumOfFiles) {
 183           open NEWTEXTFILE, ">$NewTextFilesList[$NewFileIndex - 1]" or die "Error: Can't open $NewTextFilesList[$NewFileIndex -1]: $! \n";
 184           print "Generating $NewTextFilesList[$NewFileIndex - 1] file...\n";
 185 
 186           if ($OptionsInfo{Label}) {
 187             print NEWTEXTFILE "$ColLabelsLine\n";
 188           }
 189         }
 190       }
 191     }
 192   }
 193   close NEWTEXTFILE;
 194   close TEXTFILE;
 195 }
 196 
 197 # Retrieve information about Text files...
 198 sub RetrieveTextFilesInfo {
 199   my($Index, $TextFile, $InDelim, $FileDir, $FileName, $FileExt, $OutFileRoot, $OutFileExt);
 200 
 201   %TextFilesInfo = ();
 202   @{$TextFilesInfo{FileOkay}} = ();
 203   @{$TextFilesInfo{InDelim}} = ();
 204   @{$TextFilesInfo{OutFileRoot}} = ();
 205   @{$TextFilesInfo{OutFileExt}} = ();
 206 
 207   FILELIST: for $Index (0 .. $#TextFilesList) {
 208     $TextFilesInfo{FileOkay}[$Index] = 0;
 209     $TextFilesInfo{InDelim}[$Index] = "";
 210     $TextFilesInfo{OutFileRoot}[$Index] = "";
 211     $TextFilesInfo{OutFileExt}[$Index] = "";
 212 
 213     $TextFile = $TextFilesList[$Index];
 214     if (!(-e $TextFile)) {
 215       warn "Warning: Ignoring file $TextFile: It doesn't exist\n";
 216       next FILELIST;
 217     }
 218     if (!CheckFileType($TextFile, "csv tsv")) {
 219       warn "Warning: Ignoring file $TextFile: It's not a Text file\n";
 220       next FILELIST;
 221     }
 222     if (! open TEXTFILE, "$TextFile") {
 223       warn "Warning: Ignoring file $TextFile: Couldn't open it: $! \n";
 224       next FILELIST;
 225     }
 226     close TEXTFILE;
 227 
 228     ($FileDir, $FileName, $FileExt) = ParseFileName($TextFile);
 229 
 230     # Setup input delimiter...
 231     $InDelim = '';
 232     if (!$OptionsInfo{Fast}) {
 233       if ($FileExt =~ /^tsv$/i) {
 234         $InDelim = "\t";
 235       }
 236       else {
 237         $InDelim = "\,";
 238         if ($OptionsInfo{InDelim} !~ /^(comma|semicolon)$/i) {
 239           warn "Warning: Ignoring file $TextFile: The value specified, $OptionsInfo{InDelim}, for option \"--indelim\" is not valid for csv files\n";
 240           next FILELIST;
 241         }
 242         if ($OptionsInfo{InDelim} =~ /^semicolon$/i) {
 243           $InDelim = "\;";
 244         }
 245       }
 246     }
 247 
 248     # Setup output file root...
 249     $OutFileExt = $OptionsInfo{Fast} ? $FileExt : (($Options{outdelim} =~ /^tab$/i ) ? "tsv" : "csv");
 250 
 251     if ($OptionsInfo{OutFileRoot} && (@TextFilesList == 1)) {
 252       my ($RootFileDir, $RootFileName, $RootFileExt) = ParseFileName($OptionsInfo{OutFileRoot});
 253       if ($RootFileName && $RootFileExt) {
 254         $FileName = $RootFileName;
 255       }
 256       else {
 257         $FileName = $OptionsInfo{OutFileRoot};
 258       }
 259       $OutFileRoot = $FileName;
 260     }
 261     else {
 262       $OutFileRoot = $FileName;
 263     }
 264 
 265     $TextFilesInfo{FileOkay}[$Index] = 1;
 266     $TextFilesInfo{InDelim}[$Index] = $InDelim;
 267     $TextFilesInfo{OutFileRoot}[$Index] = $OutFileRoot;
 268     $TextFilesInfo{OutFileExt}[$Index] = $OutFileExt;
 269   }
 270 }
 271 
 272 # Process option values...
 273 sub ProcessOptions {
 274 
 275   %OptionsInfo = ();
 276 
 277   $OptionsInfo{Fast} = defined $Options{fast} ? $Options{fast} : undef;
 278 
 279   $OptionsInfo{InDelim} = $Options{indelim};
 280   $OptionsInfo{Label} = ($Options{label} =~ /^yes$/i) ? 1 : 0;
 281 
 282   $OptionsInfo{NumOfFiles} = $Options{numfiles};
 283 
 284   $OptionsInfo{OutFileRoot} = $Options{root} ? $Options{root} : undef;
 285   $OptionsInfo{OverwriteFiles} = $Options{overwrite} ? $Options{overwrite} : undef;
 286 
 287   $OptionsInfo{OutDelim} = ($Options{outdelim} =~ /^tab$/i ) ? "\t" : (($Options{outdelim} =~ /^semicolon$/i) ? "\;" : "\,");
 288   $OptionsInfo{OutQuote} = ($Options{quote} =~ /^yes$/i) ? 1 : 0;
 289 }
 290 
 291 # Setup script usage  and retrieve command line arguments specified using various options...
 292 sub SetupScriptUsage {
 293 
 294   # Retrieve all the options...
 295   %Options = ();
 296   $Options{label} = "yes";
 297   $Options{numfiles} = 2;
 298   $Options{indelim} = "comma";
 299   $Options{outdelim} = "comma";
 300   $Options{quote} = "yes";
 301   if (!GetOptions(\%Options, "fast|f", "help|h", "indelim=s", "label|l=s", "numfiles|n=i", "outdelim=s", "overwrite|o", "quote|q=s", "root|r=s", "workingdir|w=s")) {
 302     die "\nTo get a list of valid options and their values, use \"$ScriptName -h\" or\n\"perl -S $ScriptName -h\" command and try again...\n";
 303   }
 304   if ($Options{workingdir}) {
 305     if (! -d $Options{workingdir}) {
 306       die "Error: The value specified, $Options{workingdir},  for option \"-w --workingdir\" is not a directory name.\n";
 307     }
 308     chdir $Options{workingdir} or die "Error: Couldn't chdir $Options{workingdir}: $! \n";
 309   }
 310   if ($Options{numfiles} < 2) {
 311     die "Error: The value specified, $Options{numfiles},  for option \"-n --numfiles\" is not valid. Allowed values: >= 2 \n";
 312   }
 313   if ($Options{indelim} !~ /^(comma|semicolon)$/i) {
 314     die "Error: The value specified, $Options{indelim}, for option \"--indelim\" is not valid. Allowed values: comma or semicolon\n";
 315   }
 316   if ($Options{outdelim} !~ /^(comma|semicolon|tab)$/i) {
 317     die "Error: The value specified, $Options{outdelim}, for option \"--outdelim\" is not valid. Allowed values: comma, tab, or semicolon\n";
 318   }
 319   if ($Options{quote} !~ /^(yes|no)$/i) {
 320     die "Error: The value specified, $Options{quote}, for option \"-q --quote\" is not valid. Allowed values: yes or no\n";
 321   }
 322   if ($Options{label} !~ /^(yes|no)$/i) {
 323     die "Error: The value specified, $Options{label}, for option \"-l --label\" is not valid. Allowed values: yes or no\n";
 324   }
 325 }
 326