[Kst] branches/work/kst/portto4/kst/src/datasources/ascii
Peter Kümmel
syntheticpp at gmx.net
Sat Oct 13 17:41:05 UTC 2012
SVN commit 1320350 by kuemmel:
split out AsciiDataReader step by step
A asciidatareader.cpp [License: GPL (v2+)]
M +106 -12 asciidatareader.h
M +1 -158 asciisource.cpp
M +0 -23 asciisource.h
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciidatareader.h #1320349:1320350
@@ -21,12 +21,14 @@
#include "asciisourceconfig.h"
#include "asciidatareader.h"
+#include "math_kst.h"
+#include "kst_inf.h"
#include <QVarLengthArray>
-#include <QFile>
-#include <QMap>
+class QFile;
+
class DataInterfaceAsciiString;
class DataInterfaceAsciiVector;
struct LexicalCast;
@@ -34,8 +36,8 @@
class AsciiDataReader
{
public:
- AsciiDataReader(AsciiSourceConfig& c) : _config(c){}
- ~AsciiDataReader() {}
+ AsciiDataReader(AsciiSourceConfig& c);
+ ~AsciiDataReader();
// TODO remove
mutable AsciiSourceConfig& _config;
@@ -216,18 +218,110 @@
};
-bool AsciiDataReader::FileBuffer::resize(int bytes)
+
+//-------------------------------------------------------------------------------------------
+template<class Buffer, typename ColumnDelimiter>
+int AsciiDataReader::readColumns(const RowIndex& rowIndex, double* v, const Buffer& buffer, int bufstart, int bufread, int col, int s, int n,
+ const LineEndingType& lineending, const ColumnDelimiter& column_del)
{
- try {
- _array->resize(bytes);
- } catch (const std::bad_alloc&) {
- // work around Qt bug
- clearFileBuffer(true);
- return false;
+ if (_config._delimiters.value().size() == 0) {
+ const NoDelimiter comment_del;
+ return readColumns(rowIndex, v, buffer, bufstart, bufread, col, s, n, lineending, column_del, comment_del);
+ } else if (_config._delimiters.value().size() == 1) {
+ const IsCharacter comment_del(_config._delimiters.value()[0].toLatin1());
+ return readColumns(rowIndex, v, buffer, bufstart, bufread, col, s, n, lineending, column_del, comment_del);
+ } else if (_config._delimiters.value().size() > 1) {
+ const IsInString comment_del(_config._delimiters.value());
+ return readColumns(rowIndex, v, buffer, bufstart, bufread, col, s, n, lineending, column_del, comment_del);
}
- return true;
+
+ return 0;
}
+template<class Buffer, typename ColumnDelimiter, typename CommentDelimiter>
+int AsciiDataReader::readColumns(const RowIndex& rowIndex, double* v, const Buffer& buffer, int bufstart, int bufread, int col, int s, int n,
+ const LineEndingType& lineending, const ColumnDelimiter& column_del, const CommentDelimiter& comment_del)
+{
+ if (_config._columnWidthIsConst) {
+ const AlwaysTrue column_withs_const;
+ if (lineending.isLF()) {
+ return readColumns(rowIndex, v, buffer, bufstart, bufread, col, s, n, IsLineBreakLF(lineending), column_del, comment_del, column_withs_const);
+ } else {
+ return readColumns(rowIndex, v, buffer, bufstart, bufread, col, s, n, IsLineBreakCR(lineending), column_del, comment_del, column_withs_const);
+ }
+ } else {
+ const AlwaysFalse column_withs_const;
+ if (lineending.isLF()) {
+ return readColumns(rowIndex, v, buffer, bufstart, bufread, col, s, n, IsLineBreakLF(lineending), column_del, comment_del, column_withs_const);
+ } else {
+ return readColumns(rowIndex, v, buffer, bufstart, bufread, col, s, n, IsLineBreakCR(lineending), column_del, comment_del, column_withs_const);
+ }
+ }
+}
+
+template<class Buffer, typename IsLineBreak, typename ColumnDelimiter, typename CommentDelimiter, typename ColumnWidthsAreConst>
+int AsciiDataReader::readColumns(const RowIndex& rowIndex, double* v, const Buffer& buffer, int bufstart, int bufread, int col, int s, int n,
+ const IsLineBreak& isLineBreak,
+ const ColumnDelimiter& column_del, const CommentDelimiter& comment_del,
+ const ColumnWidthsAreConst& are_column_widths_const)
+{
+ LexicalCast lexc;
+ lexc.setDecimalSeparator(_config._useDot);
+ const QString delimiters = _config._delimiters.value();
+
+ bool is_custom = (_config._columnType.value() == AsciiSourceConfig::Custom);
+
+ int col_start = -1;
+ for (int i = 0; i < n; i++, s++) {
+ bool incol = false;
+ int i_col = 0;
+
+ if (are_column_widths_const()) {
+ if (col_start != -1) {
+ v[i] = lexc.toDouble(&buffer[0] + rowIndex[s] + col_start);
+ continue;
+ }
+ }
+
+ v[i] = Kst::NOPOINT;
+ for (int ch = rowIndex[s] - bufstart; ch < bufread; ++ch) {
+ if (isLineBreak(buffer[ch])) {
+ break;
+ } else if (column_del(buffer[ch])) { //<- check for column start
+ if ((!incol) && is_custom) {
+ ++i_col;
+ if (i_col == col) {
+ v[i] = NAN;
+ }
+ }
+ incol = false;
+ } else if (comment_del(buffer[ch])) {
+ break;
+ } else {
+ if (!incol) {
+ incol = true;
+ ++i_col;
+ if (i_col == col) {
+ toDouble(lexc, &buffer[0], bufread, ch, &v[i], i);
+ if (are_column_widths_const()) {
+ if (col_start == -1) {
+ col_start = ch - rowIndex[s];
+ }
+ }
+ break;
+ }
+ }
+ }
+ }
+ }
+ return n;
+}
+
+
+
+
+
+
#endif
// vim: ts=2 sw=2 et
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciisource.cpp #1320349:1320350
@@ -313,24 +313,6 @@
}
-//-------------------------------------------------------------------------------------------
-AsciiDataReader::LineEndingType AsciiDataReader::detectLineEndingType(QFile& file) const
-{
- QByteArray line;
- int line_size = 0;
- while (line_size < 2 && !file.atEnd()) {
- line = file.readLine();
- line_size = line.size();
- }
- file.seek(0);
- if (line_size < 2) {
- return LineEndingType();
- }
- LineEndingType end;
- end.is_crlf = line[line_size - 2] == '\r' && line[line_size - 1] == '\n' ;
- end.character = end.is_crlf ? line[line_size - 2] : line[line_size - 1];
- return end;
-}
//-------------------------------------------------------------------------------------------
@@ -484,19 +466,9 @@
return -1;
}
-//-------------------------------------------------------------------------------------------
-void AsciiDataReader::FileBuffer::clearFileBuffer(bool forceDelete)
-{
- // force deletion of heap allocated memory if any
- if (forceDelete || _array->capacity() > AsciiDataReader::FileBuffer::Prealloc) {
- delete _array;
- _array = new Array;
- }
- _bufferedS = -10;
- _bufferedN = -10;
-}
+
//-------------------------------------------------------------------------------------------
int AsciiSource::readField(double *v, const QString& field, int s, int n)
{
@@ -615,137 +587,8 @@
return 0;
}
-//-------------------------------------------------------------------------------------------
-template<class Buffer, typename ColumnDelimiter>
-int AsciiDataReader::readColumns(const RowIndex& rowIndex, double* v, const Buffer& buffer, int bufstart, int bufread, int col, int s, int n,
- const LineEndingType& lineending, const ColumnDelimiter& column_del)
-{
- if (_config._delimiters.value().size() == 0) {
- const NoDelimiter comment_del;
- return readColumns(rowIndex, v, buffer, bufstart, bufread, col, s, n, lineending, column_del, comment_del);
- } else if (_config._delimiters.value().size() == 1) {
- const IsCharacter comment_del(_config._delimiters.value()[0].toLatin1());
- return readColumns(rowIndex, v, buffer, bufstart, bufread, col, s, n, lineending, column_del, comment_del);
- } else if (_config._delimiters.value().size() > 1) {
- const IsInString comment_del(_config._delimiters.value());
- return readColumns(rowIndex, v, buffer, bufstart, bufread, col, s, n, lineending, column_del, comment_del);
- }
- return 0;
-}
-
-template<class Buffer, typename ColumnDelimiter, typename CommentDelimiter>
-int AsciiDataReader::readColumns(const RowIndex& rowIndex, double* v, const Buffer& buffer, int bufstart, int bufread, int col, int s, int n,
- const LineEndingType& lineending, const ColumnDelimiter& column_del, const CommentDelimiter& comment_del)
-{
- if (_config._columnWidthIsConst) {
- const AlwaysTrue column_withs_const;
- if (lineending.isLF()) {
- return readColumns(rowIndex, v, buffer, bufstart, bufread, col, s, n, IsLineBreakLF(lineending), column_del, comment_del, column_withs_const);
- } else {
- return readColumns(rowIndex, v, buffer, bufstart, bufread, col, s, n, IsLineBreakCR(lineending), column_del, comment_del, column_withs_const);
- }
- } else {
- const AlwaysFalse column_withs_const;
- if (lineending.isLF()) {
- return readColumns(rowIndex, v, buffer, bufstart, bufread, col, s, n, IsLineBreakLF(lineending), column_del, comment_del, column_withs_const);
- } else {
- return readColumns(rowIndex, v, buffer, bufstart, bufread, col, s, n, IsLineBreakCR(lineending), column_del, comment_del, column_withs_const);
- }
- }
-}
-
-
-template<class Buffer, typename IsLineBreak, typename ColumnDelimiter, typename CommentDelimiter, typename ColumnWidthsAreConst>
-int AsciiDataReader::readColumns(const RowIndex& rowIndex, double* v, const Buffer& buffer, int bufstart, int bufread, int col, int s, int n,
- const IsLineBreak& isLineBreak,
- const ColumnDelimiter& column_del, const CommentDelimiter& comment_del,
- const ColumnWidthsAreConst& are_column_widths_const)
-{
- LexicalCast lexc;
- lexc.setDecimalSeparator(_config._useDot);
- const QString delimiters = _config._delimiters.value();
-
- bool is_custom = (_config._columnType.value() == AsciiSourceConfig::Custom);
-
- int col_start = -1;
- for (int i = 0; i < n; i++, s++) {
- bool incol = false;
- int i_col = 0;
-
- if (are_column_widths_const()) {
- if (col_start != -1) {
- v[i] = lexc.toDouble(&buffer[0] + rowIndex[s] + col_start);
- continue;
- }
- }
-
- v[i] = Kst::NOPOINT;
- for (int ch = rowIndex[s] - bufstart; ch < bufread; ++ch) {
- if (isLineBreak(buffer[ch])) {
- break;
- } else if (column_del(buffer[ch])) { //<- check for column start
- if ((!incol) && is_custom) {
- ++i_col;
- if (i_col == col) {
- v[i] = NAN;
- }
- }
- incol = false;
- } else if (comment_del(buffer[ch])) {
- break;
- } else {
- if (!incol) {
- incol = true;
- ++i_col;
- if (i_col == col) {
- toDouble(lexc, &buffer[0], bufread, ch, &v[i], i);
- if (are_column_widths_const()) {
- if (col_start == -1) {
- col_start = ch - rowIndex[s];
- }
- }
- break;
- }
- }
- }
- }
- }
- return n;
-}
-
-
-
//-------------------------------------------------------------------------------------------
-void AsciiDataReader::toDouble(const LexicalCast& lexc, const char* buffer, int bufread, int ch, double* v, int)
-{
- if ( isDigit(buffer[ch])
- || buffer[ch] == '-'
- || buffer[ch] == '.'
- || buffer[ch] == '+'
- || isWhiteSpace(buffer[ch])) {
- *v = lexc.toDouble(&buffer[0] + ch);
- } else if ( ch + 2 < bufread
- && tolower(buffer[ch]) == 'i'
- && tolower(buffer[ch + 1]) == 'n'
- && tolower(buffer[ch + 2]) == 'f') {
- *v = INF;
- }
-
-#if 0
- // TODO enable by option: "Add unparsable lines as strings"
- else {
- if (_rowIndex.size() > row + 1) {
- QString unparsable = QString::fromAscii(&buffer[_rowIndex[row]], _rowIndex[row + 1] - _rowIndex[row]);
- _strings[QString("Unparsable %1").arg(row)] = unparsable.trimmed();
- }
- }
-#endif
-
-}
-
-
-//-------------------------------------------------------------------------------------------
QString AsciiSource::fileType() const
{
return asciiTypeString;
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciisource.h #1320349:1320350
@@ -120,28 +120,5 @@
};
-int AsciiDataReader::readFromFile(QFile& file, AsciiDataReader::FileBuffer& buffer, int start, int bytesToRead, int maximalBytes)
-{
- if (maximalBytes == -1) {
- if (!buffer.resize(bytesToRead + 1))
- return 0;
- } else {
- bytesToRead = qMin(bytesToRead, maximalBytes);
- if (buffer.size() <= bytesToRead) {
- if (!buffer.resize(bytesToRead + 1))
- return 0;
- }
- }
- file.seek(start); // expensive?
- int bytesRead = file.read(buffer.data(), bytesToRead);
- if (buffer.size() <= bytesRead) {
- if (!buffer.resize(bytesToRead + 1))
- return 0;
- }
- buffer.data()[bytesRead] = '\0';
- return bytesRead;
-}
-
-
#endif
// vim: ts=2 sw=2 et
More information about the Kst
mailing list