[Kst] branches/work/kst/portto4/kst/src/datasources/ascii
Peter Kümmel
syntheticpp at gmx.net
Sat Oct 13 17:41:11 UTC 2012
SVN commit 1320353 by kuemmel:
split out AsciiDataReader step by step
M +42 -0 asciidatareader.cpp
M +3 -3 asciidatareader.h
M +22 -52 asciisource.cpp
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciidatareader.cpp #1320352:1320353
@@ -24,6 +24,14 @@
#include <iostream>
+// Enable QASSERT in QVarLengthArray when using [] on data
+#if 0
+ #define constData constArray
+#else
+ #define constData constPointer // loads faster in debug mode
+#endif
+
+
AsciiDataReader::AsciiDataReader(AsciiSourceConfig& c) : _config(c)
{
}
@@ -135,5 +143,39 @@
}
+int AsciiDataReader::readField(const RowIndex& _rowIndex, FileBuffer* _fileBuffer, int col, int bufstart, int bufread,
+ double *v, const QString& field, int s, int n, bool& re_alloc)
+ {
+ if (_config._columnType == AsciiSourceConfig::Fixed) {
+ MeasureTime t("AsciiSource::readField: same width for all columns");
+ LexicalCast lexc;
+ lexc.setDecimalSeparator(_config._useDot);
+ // &buffer[0] points to first row at _rowIndex[0] , so if we wanna find
+ // the column in row i by adding _rowIndex[i] we have to start at:
+ const char* col_start = &_fileBuffer->constData()[0] - _rowIndex[0] + _config._columnWidth * (col - 1);
+ for (int i = 0; i < n; ++i) {
+ v[i] = lexc.toDouble(_rowIndex[i] + col_start);
+ }
+ return n;
+ } else if (_config._columnType == AsciiSourceConfig::Custom) {
+ if (_config._columnDelimiter.value().size() == 1) {
+ MeasureTime t("AsciiSource::readField: 1 custom column delimiter");
+ const AsciiDataReader::IsCharacter column_del(_config._columnDelimiter.value()[0].toLatin1());
+ return readColumns(_rowIndex, v, _fileBuffer->constData(), bufstart, bufread, col, s, n, _lineending, column_del);
+ } if (_config._columnDelimiter.value().size() > 1) {
+ MeasureTime t(QString("AsciiSource::readField: %1 custom column delimiters").arg(_config._columnDelimiter.value().size()));
+ const AsciiDataReader::IsInString column_del(_config._columnDelimiter.value());
+ return readColumns(_rowIndex, v, _fileBuffer->constData(), bufstart, bufread, col, s, n, _lineending, column_del);
+ }
+ } else if (_config._columnType == AsciiSourceConfig::Whitespace) {
+ MeasureTime t("AsciiSource::readField: whitespace separated columns");
+ const AsciiDataReader::IsWhiteSpace column_del;
+ return readColumns(_rowIndex, v, _fileBuffer->constData(), bufstart, bufread, col, s, n, _lineending, column_del);
+ }
+
+ return 0;
+}
+
+
// vim: ts=2 sw=2 et
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciidatareader.h #1320352:1320353
@@ -27,8 +27,6 @@
#include <QVarLengthArray>
class QFile;
-
-
class DataInterfaceAsciiString;
class DataInterfaceAsciiVector;
struct LexicalCast;
@@ -209,8 +207,10 @@
int readColumns(const RowIndex& rowIndex, double* v, const Buffer& buffer, int bufstart, int bufread, int col, int s, int n,
const IsLineBreak&, const ColumnDelimiter&, const CommentDelimiter&, const ColumnWidthsAreConst&);
+ bool findDataRows(RowIndex &rowIndex, int& _numFrames, bool read_completely, QFile& file, int _byteLength);
+
template<class Buffer, typename IsLineBreak, typename CommentDelimiter>
- bool findDataRows(const Buffer& buffer, int bufstart, int bufread, const IsLineBreak&, const CommentDelimiter&);
+ bool findDataRows(RowIndex &rowIndex, int& _numFrames, const Buffer& buffer, int bufstart, int bufread, const IsLineBreak&, const CommentDelimiter&);
void toDouble(const LexicalCast& lexc, const char* buffer, int bufread, int ch, double* v, int row);
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciisource.cpp #1320352:1320353
@@ -46,7 +46,6 @@
#define constData constPointer // loads faster in debug mode
#endif
-
// simulate out of memory scenario
//#define KST_TEST_OOM
@@ -232,59 +231,64 @@
return NoChange;
}
- AsciiDataReader::LineEndingType lineending = r.detectLineEndingType(file);
-
- bool new_data = false;
bool force_update = true;
if (_byteLength == file.size()) {
force_update = false;
}
_byteLength = file.size();
+ bool new_data = r.findDataRows(_rowIndex, _numFrames, read_completely, file, _byteLength);
+
+ return (!new_data && !force_update ? NoChange : Updated);
+}
+
+bool AsciiDataReader::findDataRows(RowIndex& rowIndex, int& numFrames, bool read_completely, QFile& file, int _byteLength)
+{
+ AsciiDataReader::LineEndingType lineending = detectLineEndingType(file);
+
+ bool new_data = false;
+
AsciiDataReader::FileBuffer buf;
- buf._bufferedS = _rowIndex[_numFrames];
- buf._bufferedN = 0;
do {
// Read the tmpbuffer, starting at row_index[_numFrames]
buf.clear();
buf.resize(buf.capacity());
//bufstart += bufread;
- buf._bufferedS = _rowIndex[_numFrames]; // always read from the start of a line
- buf._bufferedN = r.readFromFile(file, buf, buf._bufferedS, _byteLength - buf._bufferedS, AsciiDataReader::FileBuffer::Prealloc - 1);
+ buf._bufferedS = rowIndex[numFrames]; // always read from the start of a line
+ buf._bufferedN = readFromFile(file, buf, buf._bufferedS, _byteLength - buf._bufferedS, AsciiDataReader::FileBuffer::Prealloc - 1);
if (_config._delimiters.value().size() == 0) {
const AsciiDataReader::NoDelimiter comment_del;
if (lineending.isLF()) {
- new_data = findDataRows(buf.constData(), buf._bufferedS, buf._bufferedN, AsciiDataReader::IsLineBreakLF(lineending), comment_del);
+ new_data = findDataRows(rowIndex, numFrames, buf.constData(), buf._bufferedS, buf._bufferedN, AsciiDataReader::IsLineBreakLF(lineending), comment_del);
} else {
- new_data = findDataRows(buf.constData(), buf._bufferedS, buf._bufferedN, AsciiDataReader::IsLineBreakCR(lineending), comment_del);
+ new_data = findDataRows(rowIndex, numFrames, buf.constData(), buf._bufferedS, buf._bufferedN, AsciiDataReader::IsLineBreakCR(lineending), comment_del);
}
} else if (_config._delimiters.value().size() == 1) {
const AsciiDataReader::IsCharacter comment_del(_config._delimiters.value()[0].toLatin1());
if (lineending.isLF()) {
- new_data = findDataRows(buf.constData(), buf._bufferedS, buf._bufferedN, AsciiDataReader::IsLineBreakLF(lineending), comment_del);
+ new_data = findDataRows(rowIndex, numFrames, buf.constData(), buf._bufferedS, buf._bufferedN, AsciiDataReader::IsLineBreakLF(lineending), comment_del);
} else {
- new_data = findDataRows(buf.constData(), buf._bufferedS, buf._bufferedN, AsciiDataReader::IsLineBreakCR(lineending), comment_del);
+ new_data = findDataRows(rowIndex, numFrames, buf.constData(), buf._bufferedS, buf._bufferedN, AsciiDataReader::IsLineBreakCR(lineending), comment_del);
}
} else if (_config._delimiters.value().size() > 1) {
const AsciiDataReader::IsInString comment_del(_config._delimiters.value());
if (lineending.isLF()) {
- new_data = findDataRows(buf.constData(), buf._bufferedS, buf._bufferedS, AsciiDataReader::IsLineBreakLF(lineending), comment_del);
+ new_data = findDataRows(rowIndex, numFrames, buf.constData(), buf._bufferedS, buf._bufferedS, AsciiDataReader::IsLineBreakLF(lineending), comment_del);
} else {
- new_data = findDataRows(buf.constData(), buf._bufferedS, buf._bufferedN, AsciiDataReader::IsLineBreakCR(lineending), comment_del);
+ new_data = findDataRows(rowIndex, numFrames, buf.constData(), buf._bufferedS, buf._bufferedN, AsciiDataReader::IsLineBreakCR(lineending), comment_del);
}
}
-
} while (buf._bufferedN == AsciiDataReader::FileBuffer::Prealloc - 1 && read_completely);
- _rowIndex.resize(_numFrames+1);
+ rowIndex.resize(numFrames + 1);
- return (!new_data && !force_update ? NoChange : Updated);
+ return new_data;
}
template<class Buffer, typename IsLineBreak, typename CommentDelimiter>
-bool AsciiSource::findDataRows(const Buffer& buffer, int bufstart, int bufread, const IsLineBreak& isLineBreak, const CommentDelimiter& comment_del)
+bool AsciiDataReader::findDataRows(RowIndex& _rowIndex, int& _numFrames, const Buffer& buffer, int bufstart, int bufread, const IsLineBreak& isLineBreak, const CommentDelimiter& comment_del)
{
const AsciiDataReader::IsWhiteSpace isWhiteSpace;
@@ -433,41 +437,7 @@
return r.readField(_rowIndex, _fileBuffer, col, bufstart, bufread, v, field, s, n, re_alloc);
}
-int AsciiDataReader::readField(const RowIndex& _rowIndex, FileBuffer* _fileBuffer, int col, int bufstart, int bufread,
- double *v, const QString& field, int s, int n, bool& re_alloc)
- {
- if (_config._columnType == AsciiSourceConfig::Fixed) {
- MeasureTime t("AsciiSource::readField: same width for all columns");
- LexicalCast lexc;
- lexc.setDecimalSeparator(_config._useDot);
- // &buffer[0] points to first row at _rowIndex[0] , so if we wanna find
- // the column in row i by adding _rowIndex[i] we have to start at:
- const char* col_start = &_fileBuffer->constData()[0] - _rowIndex[0] + _config._columnWidth * (col - 1);
- for (int i = 0; i < n; ++i) {
- v[i] = lexc.toDouble(_rowIndex[i] + col_start);
- }
- return n;
- } else if (_config._columnType == AsciiSourceConfig::Custom) {
- if (_config._columnDelimiter.value().size() == 1) {
- MeasureTime t("AsciiSource::readField: 1 custom column delimiter");
- const AsciiDataReader::IsCharacter column_del(_config._columnDelimiter.value()[0].toLatin1());
- return readColumns(_rowIndex, v, _fileBuffer->constData(), bufstart, bufread, col, s, n, _lineending, column_del);
- } if (_config._columnDelimiter.value().size() > 1) {
- MeasureTime t(QString("AsciiSource::readField: %1 custom column delimiters").arg(_config._columnDelimiter.value().size()));
- const AsciiDataReader::IsInString column_del(_config._columnDelimiter.value());
- return readColumns(_rowIndex, v, _fileBuffer->constData(), bufstart, bufread, col, s, n, _lineending, column_del);
- }
- } else if (_config._columnType == AsciiSourceConfig::Whitespace) {
- MeasureTime t("AsciiSource::readField: whitespace separated columns");
- const AsciiDataReader::IsWhiteSpace column_del;
- return readColumns(_rowIndex, v, _fileBuffer->constData(), bufstart, bufread, col, s, n, _lineending, column_del);
- }
-
- return 0;
-}
-
-
//-------------------------------------------------------------------------------------------
QString AsciiSource::fileType() const
{
More information about the Kst
mailing list