/*::.
==================================================================================================================================
=================================================¦ Copyright © 2008 Allen Baker ¦=================================================
+------------------------------+
File: DelimitedFile.java
Originator: Allen Baker (2008.02.23 12:25)
LayoutRev: 5
================================================================================================================================== */
/*.
==========================================================================================
Package
------------------------------------------------------------------------------------------ */
package cosmicabyss.com.lib;
/*.
==========================================================================================
Imports
------------------------------------------------------------------------------------------ */
import java.io.*;
import java.util.*;
import java.net.*;
/*::
======================================================================================================================== *//**
Instances of this class read delimited files such as tab-separated-values files (.tsv files) and comma-separated-values
files (.csv files such as those used by MS Excel). The file is first cleansed of garbage characters and then parsed into
the tokens that are separated by the delimiter. The default delimiter is the comma and the default quote character is
the double quote. These special characters may be changed by calling the appropriate setter methods.
Salient characteristics of a DelimitedFile include:
<BLOCKQUOTE>
=> The first line of the file is expected to contain column headers. Instances of this class use the delimited
Strings on that first line to determine how many columns are in the file as well as to identify each
column.<BR>
=> The comma character is considered the delimiter by default.<BR>
=> The delimiter can be changed from the comma to anything else through a method call,<BR>
=> The double quote character is considered the quote character by default.<BR>
=> The quote character can be changed from the double quote to anything else through a method call,<BR>
=> Instances of this class can return lines of the file in already parsed token arrays.
</BLOCKQUOTE>
Subclass<P>
This class is derived from the TextFile class. The abstraction it layers on top of TextFile includes the following:
<BLOCKQUOTE>
=> It interprets each line of text in the file as a series of deliited Strings.<BR>
=> The first line of the file is expected to contain column headers.<BR>
=> All lines subsequent to the first line (column headers line) are expected to contain a series of Strings
Separated by the delimiter, which is a comma by default.<BR>
=> All lines subsequent to the first are expected to have as many delimiter-separated Strings as there are in
The First line.
</BLOCKQUOTE>
<P>
<DL>
<DT>
<B>
Example usage:
</B>
<DD>
<BLOCKQUOTE>
<PRE id="unindent">
========================================================================================
create an object and send its output to the ConsoleStream
----------------------------------------------------------------------------------------
DelimitedFile obj = new DelimitedFile(pArgs[0]);
========================================================================================
the file we will read uses the double quote to enclose tokens and
uses the tab as a delimiter.
----------------------------------------------------------------------------------------
obj.setQuote('\"');
obj.setDelimiter('\t');
========================================================================================
test code
----------------------------------------------------------------------------------------
obj.test();
========================================================================================
parse each line of the delimited file into its separate tokens and
print each line of tokens out on a line with each tken separated by
a comma.
----------------------------------------------------------------------------------------
while (obj.hasNext())
{
ArrayList<XString> tokens = obj.parsedLine();
for (XString tkn : tokens)
{
cOut.print(tkn.concat(","));
}
cOut.println();
}
}
</PRE>
</BLOCKQUOTE>
</DD>
</DT>
<DT>
<B>
View Source:
</B>
<DD>
<A href="DelimitedFile.java.html">
DelimitedFile.java
</A>
</DD>
</DT>
<DT>
<B>
Author:
</B>
<DD>
<A href="mailto:sourcecode.v01@cosmicabyss.com">
Allen Baker
</A>
</DD>
</DT>
</DL>
*//*
======================================================================================================================== */
public class DelimitedFile extends TextFile
{
/*: :METHOD:000:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method creates an instance of the DelimitedFile class.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#000">View source</A>
*//*
---------------------------------------------------------------------------------------------------- */
public DelimitedFile(XFile xfile) throws Exception
{
super(xfile.getCanonicalPath());
initialize();
}
/*: :METHOD:001:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method creates an instance of the DelimitedFile class.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#001">View source</A>
*//*
---------------------------------------------------------------------------------------------------- */
public DelimitedFile(TextFile tfile) throws Exception
{
super(tfile.getCanonicalPath());
initialize();
}
/*: :METHOD:002:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method creates an instance of the DelimitedFile class.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#002">View source</A>
*//*
---------------------------------------------------------------------------------------------------- */
public DelimitedFile(DelimitedFile tfile) throws Exception
{
super(tfile.getCanonicalPath());
initialize();
}
/*: :METHOD:003:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method creates an instance of the DelimitedFile class.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#003">View source</A>
*//*
---------------------------------------------------------------------------------------------------- */
public DelimitedFile(URI uri) throws Exception
{
super(uri);
initialize();
}
/*: :METHOD:004:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method creates an instance of the DelimitedFile class.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#004">View source</A>
*//*
---------------------------------------------------------------------------------------------------- */
public DelimitedFile(File file) throws Exception
{
super(file.getCanonicalPath());
initialize();
}
/*: :METHOD:005:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method creates an instance of the DelimitedFile class.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#005">View source</A>
*//*
---------------------------------------------------------------------------------------------------- */
public <Type1> DelimitedFile(File parent, Type1 child) throws Exception
{
super(parent,child);
initialize();
}
/*: :METHOD:006:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method creates an instance of the DelimitedFile class.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#006">View source</A>
*//*
---------------------------------------------------------------------------------------------------- */
public <Type1> DelimitedFile(Type1 pathname) throws Exception
{
super(pathname);
initialize();
}
/*: :METHOD:007:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method creates an instance of the DelimitedFile class.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#007">View source</A>
*//*
---------------------------------------------------------------------------------------------------- */
public <Type1,Type2> DelimitedFile(Type1 parent, Type2 child) throws Exception
{
super(parent,child);
initialize();
}
/*: :METHOD:008:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method returns true if the iteration has more elements. (In other words, returns true if next
would return an element rather than throwing an exception.)
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#008">View source</A>
@return
True if the iterator has more elements.
*//*
---------------------------------------------------------------------------------------------------- */
public boolean hasNext()
{
return (iIterator.hasNext());
}
/*: :METHOD:009:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method returns the number of columns in the delimited file. It uses the number of columns in
the header line as the number of columns in the file.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#009">View source</A>
@return
A count of the number of columns in the delimited file.
*//*
---------------------------------------------------------------------------------------------------- */
public int numCols()
{
return iHeaderLine.size();
}
/*: :METHOD:010:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method returns the column header line of the delimited file in an already tokenized form.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#010">View source</A>
@return
An ArrayList in which each element is the next sequential token from the column header line of
the delimited file.
*//*
---------------------------------------------------------------------------------------------------- */
public ArrayList<XString> headerLine()
{
return iHeaderLine;
}
/*: :METHOD:011:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method returns the most recent line read from the delimited file in an already tokenized form.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#011">View source</A>
@return
An ArrayList in which each element is the next sequential token from the current line of the
delimited file.
*//*
---------------------------------------------------------------------------------------------------- */
public ArrayList<XString> currLine()
{
return iCurrentLineTokenized;
}
/*: :METHOD:012:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method returns the most recent line read from the delimited file in its raw, single string
form.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#012">View source</A>
@return
An XString containing the most recent line read from the delimited file in its raw, single string
form.
*//*
---------------------------------------------------------------------------------------------------- */
public XString currLineRaw()
{
return iCurrentLineRaw;
}
/*: :METHOD:013:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method returns the most recent fixed-up line read from the delimited file in its raw, single
string form. A fixed-up line is one that has had spurious delimiters and quotes removed and
quote-enclosed empty strings placed into all empty fields.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#013">View source</A>
@return
An XString containing the most recent fixed-up line read from the delimited file in its raw,
single string form.
*//*
---------------------------------------------------------------------------------------------------- */
public XString currLineFixed()
{
return iCurrentLineFixed;
}
/*: :METHOD:014:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method reads the next line of the delimited file. It is really just a simplified way of calling
parsedCsvLine() because it does not return the newly read line. Instead, the calling program would
use the currLine() method to retrieve the newly read line. Alternatively, the calling program can
use the field accessor methods (like valCi(), valAssetAlias(), ...) to get the actual column values
from the newly read line.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#014">View source</A>
*//*
---------------------------------------------------------------------------------------------------- */
public void readNext() throws Exception
{
parsedLine();
}
/*: :METHOD:015:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method returns the next line of the delimited file in an already tokenized form.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#015">View source</A>
@return
An ArrayList in which each element is the next sequential token from the current line of the
delimited file.
*//*
---------------------------------------------------------------------------------------------------- */
public ArrayList<XString> parsedLine() throws Exception
{
/*.
==========================================================================================
We cache the first line of the delinmited file because it is the one that contains the
column headers. It is pre-feched in the initialization() method, which makes the first
call to this method. The second time this method is called, the cached line is returned.
Thereafter, each call to this method returns the next un-cached line from the input file.
------------------------------------------------------------------------------------------ */
if ( ++iParsedLineCallCount == 2)
{
if (! iSkipHeaderLine)
{
return iHeaderLine;
}
}
/*.
==========================================================================================
Read the next line from the file.
------------------------------------------------------------------------------------------ */
XString line = iIterator.next();
iCurrentLineRaw = line;
/*.
==========================================================================================
Replace all those insidious delimiters that sometimes are found within quotes. Replace
them with a token that can later be found and restored to the original delimiter after the
line is parsed.
------------------------------------------------------------------------------------------ */
line = replaceDelimitersInsideQuotes(line);
/*.
==========================================================================================
If the line begins with a delimeter, replace it with a iNullReplacement followed by a
delimiter
------------------------------------------------------------------------------------------ */
if (line.charAt(0)==iDelimiterChar)
{
line = iNullReplacement.concat(line);
}
/*.
==========================================================================================
Put a token into the line in all the positions that are empty and then tokenize the line.
iNullReplacement is a token that is used to replace "" because "" is ignored and passed
over by tokenizedString(). Later, after tokenization, all the iNullReplacements will be
replaced with the original "".
------------------------------------------------------------------------------------------ */
line = line.iteratingReplace
(
iDelimiterString.concat(iDelimiterString), // replace things like this ,,
iDelimiterString.concat(iNullReplacement).concat(iDelimiterString) // with things like this ,Digest("NULL_REPLACEMENT"),
);
ArrayList<XString> tempTokens = line.tokenizedString(iDelimiterString);
if (line.endsWith(iDelimiterString)) tempTokens.add(iNullReplacement);
/*.
==========================================================================================
Later (after tokenization is complete)... Replace all the iNullReplacements with nulls and
replace all the iDelimiterRepacements with iDelimiters
------------------------------------------------------------------------------------------ */
ArrayList<XString> tokens = new ArrayList<XString>();
XStringsIterator tIter = new XStringsIterator(tempTokens);
while (tIter.hasNext())
{
/*.
==========================================================================================
------------------------------------------------------------------------------------------ */
XString tkn = tIter.next().trim();
tkn = tkn.replace(iNullReplacement,"").replace(iDelimiterReplacement,iDelimiterString);
/*.
==========================================================================================
This part removes unneeded enclosing quote string from the token
------------------------------------------------------------------------------------------ */
tkn = tkn.trimLeft(iQuoteString).trimRight(iQuoteString);
if (tkn.contains(iDelimiterString))
{
tkn = iQuoteString.concat(tkn).concat(iQuoteString);
}
/*.
==========================================================================================
This part "fixes" (actually "works around") unmatched quote strings within the token.
I've already thought through the need for this so dont delete this code. The explanation
would take too long here.
------------------------------------------------------------------------------------------ */
int numQuotes = 0;
int lastQuoteIdx = 0;
for (int i=0; i<tkn.length(); i++)
{
if (tkn.charAt(i) == iQuoteChar)
{
lastQuoteIdx = i;
numQuotes++;
}
}
if (UMath.isOdd(numQuotes))
{
StringBuffer sBuf = new StringBuffer(tkn.string());
sBuf.insert(lastQuoteIdx+1,iQuoteString.string());
tkn = new XString(sBuf.toString());
}
/*.
==========================================================================================
------------------------------------------------------------------------------------------ */
tokens.add(tkn);
}
/*.
==========================================================================================
Remember what the line looks like after being cleaned up
------------------------------------------------------------------------------------------ */
int tokenNum = 0;
tIter = new XStringsIterator(tokens);
while (tIter.hasNext())
{
if (tokenNum == 0)
{
iCurrentLineFixed = tIter.next();
}
else
{
iCurrentLineFixed = iCurrentLineFixed.concat(iDelimiterString).concat(tIter.next());
}
}
/*.
==========================================================================================
Remember what the line looks like after being tokenized
------------------------------------------------------------------------------------------ */
iCurrentLineTokenized = tokens;
/*.
==========================================================================================
And return the tokenized line
------------------------------------------------------------------------------------------ */
return tokens;
}
/*: :METHOD:016:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method returns the column number of the column with the given header. If no such column is
found, it returns -1.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#016">View source</A>
@return
The column number with the given header or -1 if it is not found
@param
pHeader is the column header to search for
*//*
---------------------------------------------------------------------------------------------------- */
public <Type1> int colNum(Type1 pColHeader)
{
for (int i=0; i<iHeaderLine.size(); i++)
{
if (iHeaderLine.get(i).equals(pColHeader))
{
return i;
}
}
return (-1);
}
/*: :METHOD:017:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method returns the column number of the column with the given header ignoring case. If no such
column is found, it returns -1.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#017">View source</A>
@return
The column number with the given header or -1 if it is not found
@param
pHeader is the column header to search for
*//*
---------------------------------------------------------------------------------------------------- */
public <Type1> int colNumIgnoreCase(Type1 pColHeader)
{
for (int i=0; i<iHeaderLine.size(); i++)
{
if (iHeaderLine.get(i).equalsIgnoreCase(pColHeader))
{
return i;
}
}
return (-1);
}
/*: :METHOD:018:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method sets the quote character for this object.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#018">View source</A>
@return
A reference to this object
@param
pQuoteChar is the character to use as a quote for this object
*//*
---------------------------------------------------------------------------------------------------- */
public DelimitedFile setQuote(char pQuoteChar)
{
iQuoteChar = pQuoteChar;
iQuoteString = new XString(iQuoteChar);
return this;
}
/*: :METHOD:019:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method sets the delimiter character for this object.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#019">View source</A>
@return
A reference to this object
@param
pDelimiterChar is the character to use as a delimiter for this object
*//*
---------------------------------------------------------------------------------------------------- */
public DelimitedFile setDelimiter(char pDelimiterChar)
{
iDelimiterChar = pDelimiterChar;
iDelimiterString = new XString(iDelimiterChar);
return this;
}
/*: :METHOD:020:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method sets the flag that determines whether or not the parseLine() method wil return the
header line.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#020">View source</A>
@return
A reference to this object
@param
pSkipHeaderLines is the boolean value to turn on or off the skipping of header lines. If
pSkipHeaderLines is true, then the parsedLine() will skip the header line. If false, it will not
skip header lines.
*//*
---------------------------------------------------------------------------------------------------- */
public DelimitedFile setSkipHeaderLine(boolean pSkipHeaderLine)
{
iSkipHeaderLine = pSkipHeaderLine;
return this;
}
/*: :METHOD:021:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method tells this object whether to skip white space lines or not. A white space line is a line
of the text file that contains only white space characters.<P>
This method is a superset of setSkipBlankLines() because white space lines include blank lines.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#021">View source</A>
@return
A reference to this object
@param
pSkipWhiteSpaceLines is the boolean value to turn on or off the skipping of white space lines. If
pSkipWhiteSpaceLines is true, then the TextReaderIterator will skip white space lines. If false,
it will not skip white space lines.
*//*
---------------------------------------------------------------------------------------------------- */
public DelimitedFile setSkipWhiteSpaceLines(boolean pSkipWhiteSpaceLines)
{
iIterator.setSkipWhiteSpaceLines(pSkipWhiteSpaceLines);
return this;
}
/*: :METHOD:022:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method sets the ConsoleStream for this object.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#022">View source</A>
@return
A reference to this object
@param
pConsole is a ConsoleStream through which this objects sends its output.
*//*
---------------------------------------------------------------------------------------------------- */
public DelimitedFile setOut(ConsoleStream pConsole)
{
cOut = pConsole;
return this;
}
/*: :METHOD:023:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This boilerplate method is used for testing an instantiated object of this class and may include any
code the developer chooses.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#023">View source</A>
*//*
---------------------------------------------------------------------------------------------------- */
public DelimitedFile test() throws Exception
{
cOut.titledPrintf
(
"\"HELLO WORLD!\"",
"%s %s %s",
"I'm an object of the", CLASS_NAME, "class, and I approved this message."
);
return this;
}
/*:.
==============================================================================================================
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@[ Protected ]@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
============================================================================================================== */
/*:.
==============================================================================================================
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@[ Private ]@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
============================================================================================================== */
/*.
==========================================================================================
Class Constants
CLASS_NAME:
The name of this class
DFLT_LINE_LEN:
The default line length for word wrapping
------------------------------------------------------------------------------------------ */
private static final XString CLASS_NAME = new XString(DelimitedFile.class.getName());
private static final int DFLT_LINE_LEN = ConsoleMessage.defaultLineLength();
/*.
==========================================================================================
Class variables
cOut : console output.
------------------------------------------------------------------------------------------ */
private static ConsoleStream cOut = ConsoleStream.getSingleton();
/*.
==========================================================================================
Instance variables
<BLOCKQUOTE>
<PRE id="unindent">
iQuoteChar : the character to use as a quote
iQuoteString : an XString representation of the quote character
iDelimiterChar : the character to use as a delimiter
iDelimiterString : an XString representation of the delimiter character
iIterator : an iterator for iterating over the lines of the delimited file
iDelimiterReplacement : a uniquely recognizable string that is substituted for a delimiter character
iNullReplacement : a uniquely recognizable string that is substituted for a null character
iCurrentLineRaw : remembers the most recent line of the input file that was fetched by the iterator
iCurrentLineFixed : remembers the most recent line of the input file that was cleaned
</PRE>
</BLOCKQUOTE>
------------------------------------------------------------------------------------------ */
private char iQuoteChar = '\"';
private XString iQuoteString = new XString(iQuoteChar);
private char iDelimiterChar = ',';
private XString iDelimiterString = new XString(iDelimiterChar);
private TextReaderIterator iIterator = null;
private XString iDelimiterReplacement = null;
private XString iNullReplacement = null;
private XString iCurrentLineRaw = null;
private XString iCurrentLineFixed = null;
private ArrayList<XString> iCurrentLineTokenized = null;
private ArrayList<XString> iHeaderLine = null;
private long iParsedLineCallCount = 0L;
private boolean iSkipHeaderLine = false;
/*:.
==============================================================================================================
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@[ Public Static Methods ]@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
============================================================================================================== */
/*: :METHOD:024:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method is the common mechanism used by all constructors to initialize an object
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#024">View source</A>
*//*
---------------------------------------------------------------------------------------------------- */
private void initialize() throws Exception
{
/*.
==========================================================================================
------------------------------------------------------------------------------------------ */
iDelimiterReplacement = new XString(new Digest("DELIMITER_REPLACEMENT").string());
iNullReplacement = new XString(new Digest("NULL_REPLACEMENT").string());
/*.
==========================================================================================
------------------------------------------------------------------------------------------ */
fixEmbeddedNewlines();
trim(); // dont do this trim() until after removing embedded newlines
replaceNonPrintingCharacters();
/*.
==========================================================================================
------------------------------------------------------------------------------------------ */
iIterator = new TextReaderIterator(this);
/*.
==========================================================================================
------------------------------------------------------------------------------------------ */
iHeaderLine = parsedLine();
}
/*: :METHOD:025:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method replaces all non-printing (except tabs and newlines) and non-standard (chars 127 thru
255) ascii characters with the uptick character - '^'.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#025">View source</A>
*//*
---------------------------------------------------------------------------------------------------- */
private void replaceNonPrintingCharacters() throws Exception
{
TextFile thisFile = new TextFile(this);
String[][] strPairs = new String[30 + 129][2];
int idx = 0;
for (int i=0; i<32; i++)
{
if ((i==10) || (i==9)) continue;
strPairs[idx][0] = new Character((char)i).toString();
strPairs[idx][1] = new String("^");
idx++;
}
for (int i=127; i<256; i++)
{
strPairs[idx][0] = new Character((char)i).toString();
strPairs[idx][1] = new String("^");
idx++;
}
thisFile.replace(strPairs);
thisFile = null;
}
/*: :METHOD:026:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method eliminates the insidious delimiters that are sometimes put into quote-enclosed fields by
the csv file generators. Those delimiters obviously mess up any parsing based on delimiters.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#026">View source</A>
@return
An XString without any delimiters within quote-enclosed fields
@param
pIn is the XString to be searched for the offending quote-enclosed delimiters.
*//*
---------------------------------------------------------------------------------------------------- */
private XString replaceDelimitersInsideQuotes(XString pIn)
{
char[] chars = pIn.toCharArray();
boolean inQuote = false;
boolean fixedOne = false;
for (int i=0; i<chars.length; i++)
{
char thisChar = chars[i];
if (thisChar==iQuoteChar)
{
inQuote = !inQuote;
}
else if (inQuote)
{
if (thisChar==iDelimiterChar)
{
chars[i] = '\376'; // 376 octal == 254 decimal
fixedOne = true;
}
}
}
XString result = new XString(chars).replace("\376",iDelimiterReplacement);
return result;
}
/*: :METHOD:027:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method eliminates newlines that are embedded within quote strings in a file.
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#027">View source</A>
*//*
---------------------------------------------------------------------------------------------------- */
private void fixEmbeddedNewlines() throws Exception
{
/*.
==========================================================================================
------------------------------------------------------------------------------------------ */
File iFile = new File(this.getCanonicalPath());
/*.
==========================================================================================
This method makes no sense on directory files, this must be a normal file
------------------------------------------------------------------------------------------ */
if (iFile.isDirectory()) return;
/*.
==========================================================================================
We'll output the processed version of the file to a temp file so if the process fails
midway through, we don't corrupt the original file
------------------------------------------------------------------------------------------ */
File oFile = File.createTempFile("fixEmbeddedNewlines-",".tmp");
/*.
==========================================================================================
Process the file
------------------------------------------------------------------------------------------ */
fixEmbeddedNewlines
(
iFile.getCanonicalPath(),
oFile.getCanonicalPath()
);
/*.
==========================================================================================
Delete the original file
------------------------------------------------------------------------------------------ */
if ( ! iFile.delete())
{
throw new Exception("Cannot delete " + iFile.getCanonicalPath());
}
/*.
==========================================================================================
And replace it with the processed file.
------------------------------------------------------------------------------------------ */
if ( ! oFile.renameTo(iFile))
{
throw new Exception("Cannot rename " + oFile.getCanonicalPath());
}
}
/*: :METHOD:028:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method eliminates newlines that are embedded within quote strings in a file. To accomplish this
task, it reads the file as a stream of chars and executes the following state machine.
<BLOCKQUOTE>
<PRE id="unindent">
+-------------------+-------+----------------------------------------------------+
| Meaning of state | State | Next character |
| | +- - - - - - - - - - +- - - - - - - -+- - - - - - - -+
| | | newline | iQuoteChar | any other |
+-------------------+-------+--------------------+---------------+---------------+
| Start of line | 0 | (do nothing) | go to state 2 | go to state 1 |
+ - - - - - - -+ - - -+ - - - - - - - + - - - - - + - - - - - +
| Not inside quotes | 1 | go to state 0 | go to state 2 | (do nothing) |
+ - - - - - - -+ - - -+ - - - - - - - + - - - - - + - - - - - +
| Inside quotes | 2 | replace with blank | go to state 1 | (do nothing) |
+ - - - - - - -+ - - -+ - - - - - - - + - - - - - + - - - - - +
</PRE>
</BLOCKQUOTE>
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#028">View source</A>
@param
pFileIn is the name of the file that will be copied to the output file with embedded newlines
tossed. PFileOut must not be the same as pFileIn. PFileOut and pFileIn must not be null.
@param
pFileOut is the name that will be given to the file that is produced. PFileOut must not be the
same as pFileIn. PFileOut and pFileIn must not be null.
*//*
---------------------------------------------------------------------------------------------------- */
private <Type1,Type2> void fixEmbeddedNewlines
(
Type1 pFileIn,
Type2 pFileOut
)
throws Exception
{
/*.
==========================================================================================
Check the parameters.
------------------------------------------------------------------------------------------ */
if (pFileIn == null)
{
throw new Exception("pFileIn is null");
}
if (pFileOut == null)
{
throw new Exception("pFileOut is null");
}
if (pFileIn.equals(pFileOut))
{
throw new Exception("pFileIn and pFileOut are the same file");
}
/*.
==========================================================================================
------------------------------------------------------------------------------------------ */
XString fileIn = new XString(pFileIn);
XString fileOut = new XString(pFileOut);
/*.
==========================================================================================
Set up the files
------------------------------------------------------------------------------------------ */
CharIterator iter = (new TextFile(fileIn)).charIterator();
PrintWriter output = (new TextFile(fileOut)).printWriter();
/*.
==========================================================================================
Read each line of the source file and write a tossed version of it to the output file.
------------------------------------------------------------------------------------------ */
int state = 0;
while (iter.hasNext())
{
char c = iter.next();
switch (state)
{
case 0: // BEGIN LINE
{
if (c=='\n')
{
output.print(c);
}
else if (c==iQuoteChar)
{
state = 2;
output.print(c);
}
else
{
output.print(c);
state = 1;
}
break;
}
case 1: // OUT OF QUOTE
{
if (c=='\n')
{
state = 0;
output.print(c);
}
else if (c==iQuoteChar)
{
state = 2;
output.print(c);
}
else
{
output.print(c);
}
break;
}
case 2: // IN QUOTE
{
if (c=='\n')
{
output.print(' ');
}
else if (c==iQuoteChar)
{
state = 1;
output.print(c);
}
else
{
output.print(c);
}
break;
}
}
}
/*.
==========================================================================================
If we were still in state 2 when the end of file was reached, then that means the file
ended without a required close quote followed by a required newline. In that situation,
fix the file by finishing it off with a close quote and newline.
------------------------------------------------------------------------------------------ */
if (state == 2)
{
output.print("\"\n");
}
/*.
==========================================================================================
Flush the output buffers
------------------------------------------------------------------------------------------ */
output.flush();
/*.
==========================================================================================
Close the files
------------------------------------------------------------------------------------------ */
output.close();
}
/*: :METHOD:029:BOOKMARK:
====================================================================================================
[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][]
==================================================================================================== *//**
This method allows this class file to be unit tested as a standalone application. It's the method
that's called when the class is invoked from the command line by using the java application launcher
- "java". Main() is not a required method, but the practice of putting one in each class and
wrapping class test code within it allows easy unit testing of the class; and main does not need to
be removed when testing is complete.
<P>
<DL>
<DT>
<B>
Command line usage:
</B>
<DD>
Java cosmicabyss.com.lib.DelimitedFile
</DD>
</DT>
</DL>
<P><B>Implementation: </B><A HREF="DelimitedFile.java.html#029">View source</A>
@param
pArgs contains the command line arguments with which this class was invoked as an application.
*//*
---------------------------------------------------------------------------------------------------- */
public static void main(String[] pArgs) throws Exception
{
/*.
==========================================================================================
Greetings !
------------------------------------------------------------------------------------------ */
cOut.banner(CLASS_NAME);
/*.
==========================================================================================
Create an object and send its output to the ConsoleStream
------------------------------------------------------------------------------------------ */
DelimitedFile obj = new DelimitedFile("c:/temp/problems.csv");
// DelimitedFile obj = new DelimitedFile(pArgs[0]);
// /*
// ==========================================================================================
// the file we will read uses the double quote to enclose tokens and
// uses the tab as a delimiter.
// ------------------------------------------------------------------------------------------ */
// obj.setQuote('\"');
// obj.setDelimiter('\t');
/*.
==========================================================================================
Test code
------------------------------------------------------------------------------------------ */
obj.test();
// /*
// ==========================================================================================
// parse each line of the delimited file into its separate tokens and
// print each line of tokens out on a line with each tken separated by
// a comma.
// ------------------------------------------------------------------------------------------ */
// while (obj.hasNext())
// {
// ArrayList<XString> tokens = obj.parsedLine();
// for (XString tkn : tokens)
// {
// cOut.print(tkn.concat(","));
// }
// cOut.println();
// }
}
} // class DelimitedFile
/*:.
==============================================================================================================
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@[ Notes ]@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
============================================================================================================== */
Facebook