[Kst] branches/work/kst/portto4/kst/src/datasources/ascii
Peter Kümmel
syntheticpp at gmx.net
Sat Oct 13 17:41:02 UTC 2012
SVN commit 1320349 by kuemmel:
split out AsciiDataReader step by step
M +4 -29 asciidatareader.h
M +17 -14 asciisource.cpp
M +4 -28 asciisource.h
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciidatareader.h #1320348:1320349
@@ -65,7 +65,7 @@
inline void clear() { _array->clear(); }
inline int size() const { return _array->size(); }
- inline void resize(int size) { _array->resize(size); }
+ inline bool resize(int size);
inline int capacity() const { return _array->capacity(); }
inline char* data() { return _array->data(); }
@@ -86,8 +86,7 @@
bool resizeBuffer(T& buffer, int bytes);
- template<class T>
- int readFromFile(QFile&, T& buffer, int start, int numberOfBytes, int maximalBytes = -1);
+ int readFromFile(QFile&, AsciiDataReader::FileBuffer&, int start, int numberOfBytes, int maximalBytes = -1);
int readField(double *v, const QString &field, int s, int n, bool& re_alloc);
@@ -217,11 +216,10 @@
};
-template<class T>
-bool AsciiDataReader::resizeBuffer(T& buffer, int bytes)
+bool AsciiDataReader::FileBuffer::resize(int bytes)
{
try {
- buffer.resize(bytes);
+ _array->resize(bytes);
} catch (const std::bad_alloc&) {
// work around Qt bug
clearFileBuffer(true);
@@ -230,29 +228,6 @@
return true;
}
-template<class T>
-int AsciiDataReader::readFromFile(QFile& file, T& buffer, int start, int bytesToRead, int maximalBytes)
-{
- if (maximalBytes == -1) {
- if (!resizeBuffer(buffer, bytesToRead + 1))
- return 0;
- } else {
- bytesToRead = qMin(bytesToRead, maximalBytes);
- if (buffer.size() <= bytesToRead) {
- if (!resizeBuffer(buffer, bytesToRead + 1))
- return 0;
- }
- }
- file.seek(start); // expensive?
- int bytesRead = file.read(buffer.data(), bytesToRead);
- if (buffer.size() <= bytesRead) {
- if (!resizeBuffer(buffer, bytesToRead + 1))
- return 0;
- }
- buffer.data()[bytesRead] = '\0';
- return bytesRead;
-}
-
#endif
// vim: ts=2 sw=2 et
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciisource.cpp #1320348:1320349
@@ -240,7 +240,7 @@
void AsciiSource::reset()
{
// forget about cached data
- clearFileBuffer();
+ _fileBuffer->clearFileBuffer();
_rowIndex.clear();
_valid = false;
@@ -346,7 +346,7 @@
MeasureTime t("AsciiSource::internalDataSourceUpdate: " + _filename);
// forget about cached data
- clearFileBuffer();
+ _fileBuffer->clearFileBuffer();
if (!_haveHeader) {
_haveHeader = initRowIndex();
@@ -396,7 +396,7 @@
//bufstart += bufread;
buf._bufferedS = _rowIndex[_numFrames]; // always read from the start of a line
- buf._bufferedN = readFromFile(file, buf, buf._bufferedS, _byteLength - buf._bufferedS, AsciiDataReader::FileBuffer::Prealloc - 1);
+ buf._bufferedN = r.readFromFile(file, buf, buf._bufferedS, _byteLength - buf._bufferedS, AsciiDataReader::FileBuffer::Prealloc - 1);
if (_config._delimiters.value().size() == 0) {
const AsciiDataReader::NoDelimiter comment_del;
@@ -485,15 +485,18 @@
}
//-------------------------------------------------------------------------------------------
-void AsciiSource::clearFileBuffer(bool forceDelete)
+void AsciiDataReader::FileBuffer::clearFileBuffer(bool forceDelete)
{
// force deletion of heap allocated memory if any
- if (forceDelete || _fileBuffer->capacity() > AsciiDataReader::FileBuffer::Prealloc) {
- delete _fileBuffer;
- _fileBuffer = new AsciiDataReader::FileBuffer;
+ if (forceDelete || _array->capacity() > AsciiDataReader::FileBuffer::Prealloc) {
+ delete _array;
+ _array = new Array;
}
+ _bufferedS = -10;
+ _bufferedN = -10;
}
+
//-------------------------------------------------------------------------------------------
int AsciiSource::readField(double *v, const QString& field, int s, int n)
{
@@ -507,12 +510,12 @@
// reading whole file into memory failed
// find a smaller allocatable size
- clearFileBuffer();
+ _fileBuffer->clearFileBuffer();
int realloc_size = n / 4;
- while (!resizeBuffer(*_fileBuffer, realloc_size) && realloc_size > 0) {
+ while (!_fileBuffer->resize(realloc_size) && realloc_size > 0) {
realloc_size /= 2;
}
- clearFileBuffer();
+ _fileBuffer->clearFileBuffer();
if (realloc_size == 0) {
QMessageBox::warning(0, "Error while reading ascii file", "File could not be read because not enough memory is available.");
return 0;
@@ -522,18 +525,18 @@
int start = s;
n_read = 0;
while (n_read < n) {
- clearFileBuffer();
+ _fileBuffer->clearFileBuffer();
int to_read = n_read + realloc_size < n ? realloc_size : n - n_read;
n_read += readField(v + start, field, n_read, to_read, re_alloc);
if (!re_alloc) {
- clearFileBuffer();
+ _fileBuffer->clearFileBuffer();
QMessageBox::warning(0, "Error while reading ascii file", "The file was only read partially not enough memory is available.");
return n_read;
}
start += to_read;
}
// don't buffer partial files
- clearFileBuffer();
+ _fileBuffer->clearFileBuffer();
return n_read;
}
@@ -573,7 +576,7 @@
r._lineending = r.detectLineEndingType(file);
- bufread = readFromFile(file, *_fileBuffer, bufstart, bufread);
+ bufread = r.readFromFile(file, *_fileBuffer, bufstart, bufread);
if (bufread == 0) {
re_alloc = false;
return 0;
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciisource.h #1320348:1320349
@@ -84,12 +84,6 @@
AsciiDataReader::FileBuffer* _fileBuffer;
AsciiDataReader::RowIndex _rowIndex;
- void clearFileBuffer(bool forceDelete = false);
-
- template<class T>
- bool resizeBuffer(T& buffer, int bytes);
-
-
friend class ConfigWidgetAscii;
mutable AsciiSourceConfig _config;
@@ -114,52 +108,34 @@
bool openValidFile(QFile &file);
static bool openFile(QFile &file);
- template<class T>
- int readFromFile(QFile&, T& buffer, int start, int numberOfBytes, int maximalBytes = -1);
-
int readField(double *v, const QString &field, int s, int n, bool& re_alloc);
template<class Buffer, typename IsLineBreak, typename CommentDelimiter>
bool findDataRows(const Buffer& buffer, int bufstart, int bufread, const IsLineBreak&, const CommentDelimiter&);
- void toDouble(const LexicalCast& lexc, const char* buffer, int bufread, int ch, double* v, int row);
-
// TODO remove
friend class DataInterfaceAsciiString;
friend class DataInterfaceAsciiVector;
};
-template<class T>
-bool AsciiSource::resizeBuffer(T& buffer, int bytes)
-{
- try {
- buffer.resize(bytes);
- } catch (const std::bad_alloc&) {
- // work around Qt bug
- clearFileBuffer(true);
- return false;
- }
- return true;
-}
-template<class T>
-int AsciiSource::readFromFile(QFile& file, T& buffer, int start, int bytesToRead, int maximalBytes)
+int AsciiDataReader::readFromFile(QFile& file, AsciiDataReader::FileBuffer& buffer, int start, int bytesToRead, int maximalBytes)
{
if (maximalBytes == -1) {
- if (!resizeBuffer(buffer, bytesToRead + 1))
+ if (!buffer.resize(bytesToRead + 1))
return 0;
} else {
bytesToRead = qMin(bytesToRead, maximalBytes);
if (buffer.size() <= bytesToRead) {
- if (!resizeBuffer(buffer, bytesToRead + 1))
+ if (!buffer.resize(bytesToRead + 1))
return 0;
}
}
file.seek(start); // expensive?
int bytesRead = file.read(buffer.data(), bytesToRead);
if (buffer.size() <= bytesRead) {
- if (!resizeBuffer(buffer, bytesToRead + 1))
+ if (!buffer.resize(bytesToRead + 1))
return 0;
}
buffer.data()[bytesRead] = '\0';
More information about the Kst
mailing list