[Kst] branches/work/kst/portto4/kst/src/datasources/ascii
Peter Kümmel
syntheticpp at gmx.net
Tue Oct 16 14:47:57 UTC 2012
SVN commit 1320707 by kuemmel:
add reading with sliding window
M +55 -13 asciifilebuffer.cpp
M +8 -2 asciifilebuffer.h
M +46 -14 asciifiledata.cpp
M +11 -4 asciifiledata.h
M +15 -29 asciisource.cpp
M +0 -3 asciisource.h
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciifilebuffer.cpp #1320706:1320707
@@ -23,7 +23,7 @@
extern size_t maxAllocate;
//-------------------------------------------------------------------------------------------
-AsciiFileBuffer::AsciiFileBuffer()
+AsciiFileBuffer::AsciiFileBuffer() : _file(0), _begin(-1), _bytesRead(0)
{
}
@@ -34,12 +34,29 @@
}
//-------------------------------------------------------------------------------------------
+void AsciiFileBuffer::setFile(QFile* file)
+{
+ delete _file;
+ _file = file;
+}
+
+//-------------------------------------------------------------------------------------------
+bool AsciiFileBuffer::openFile(QFile &file)
+{
+ // Don't use 'QIODevice::Text'!
+ // Because CR LF line ending breaks row offset calculation
+ return file.open(QIODevice::ReadOnly);
+}
+
+//-------------------------------------------------------------------------------------------
void AsciiFileBuffer::clear()
{
foreach (AsciiFileData chunk, _fileData) {
chunk.release();
}
_fileData.clear();
+ _begin = -1;
+ _bytesRead = 0;
}
//-------------------------------------------------------------------------------------------
@@ -49,9 +66,9 @@
}
//-------------------------------------------------------------------------------------------
-void AsciiFileBuffer::logData() const
+void AsciiFileBuffer::logData(const QVector<AsciiFileData>& chunks) const
{
- foreach (const AsciiFileData& chunk, _fileData) {
+ foreach (const AsciiFileData& chunk, chunks) {
chunk.logData();
}
}
@@ -97,20 +114,27 @@
chunk.setRowsRead(lastRow - rowBegin);
chunks << chunk;
}
- //qDebug() << "File splitted into " << chunks.size() << " chunks:"; logData();
+ qDebug() << "File splitted into " << chunks.size() << " chunks:"; logData(chunks);
return chunks;
}
//-------------------------------------------------------------------------------------------
-void AsciiFileBuffer::read(QFile& file, const RowIndex& rowIndex, int start, int bytesToRead, int maximalBytes)
+void AsciiFileBuffer::read(const RowIndex& rowIndex, int start, int bytesToRead, int maximalBytes)
{
- _begin = -1;
- _bytesRead = 0;
- _fileData.clear();
+ clear();
+ if (!_file) {
+ return;
+ }
+ //readWholeFile(rowIndex, start, bytesToRead, maximalBytes);
+ readFileSlidingWindow(rowIndex, start, bytesToRead, maximalBytes);
+}
+//-------------------------------------------------------------------------------------------
+void AsciiFileBuffer::readWholeFile(const RowIndex& rowIndex, int start, int bytesToRead, int maximalBytes)
+{
// first try to read the whole file into one array
AsciiFileData wholeFile;
- wholeFile.read(file, start, bytesToRead, maximalBytes);
+ wholeFile.read(*_file, start, bytesToRead, maximalBytes);
if (bytesToRead == wholeFile.bytesRead()) {
wholeFile.setRowBegin(0);
wholeFile.setRowsRead(rowIndex.size());
@@ -128,7 +152,8 @@
_bytesRead = 0;
foreach (AsciiFileData chunk, _fileData) {
// use alread set
- if (!chunk.lazyRead(file)) {
+ chunk.setFile(_file);
+ if (!chunk.read()) {
Kst::Debug::self()->log(QString("AsciiFileBuffer: error when reading into chunk"));
chunk.release();
break;
@@ -137,16 +162,33 @@
}
if (_bytesRead == bytesToRead) {
_begin = start;
- return;
} else {
_bytesRead = 0;
_fileData.clear();
Kst::Debug::self()->log(QString("AsciiFileBuffer: error while reading %1 chunks").arg(_fileData.size()));
}
+}
- // sliding window
- // TODO
+//-------------------------------------------------------------------------------------------
+void AsciiFileBuffer::readFileSlidingWindow(const RowIndex& rowIndex, int start, int bytesToRead, int maximalBytes)
+{
+ int chunkSize = qMin((size_t) 10 * MB, maxAllocate);
+ chunkSize = 2 * MB;
+ _fileData = splitFile(chunkSize, rowIndex, start, bytesToRead);
+ _bytesRead = 0;
+ AsciiFileData master;
+ if (!master.resize(chunkSize)) {
+ Kst::Debug::self()->log(QString("AsciiFileBuffer: not enough memory available for creating sliding window"));
}
+ for (int i = 0; i < _fileData.size(); i++) {
+ // use alread set
+ _fileData[i].setLazyRead(true);
+ _fileData[i].setFile(_file);
+ _fileData[i].setSharedArray(master);
+ }
+ _begin = start;
+ _bytesRead = bytesToRead;
+}
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciifilebuffer.h #1320706:1320707
@@ -31,17 +31,23 @@
void clear();
- void read(QFile&, const RowIndex& rowIndex, int start, int numberOfBytes, int maximalBytes = -1);
+ void setFile(QFile* file);
+ void read(const RowIndex& rowIndex, int start, int numberOfBytes, int maximalBytes = -1);
const QVector<AsciiFileData>& data() const;
+ static bool openFile(QFile &file);
+
private:
+ QFile* _file;
QVector<AsciiFileData> _fileData;
int _begin;
int _bytesRead;
- void logData() const;
+ void logData(const QVector<AsciiFileData>& chunks) const;
const QVector<AsciiFileData> splitFile(int chunkSize, const RowIndex& rowIndex, int start, int bytesToRead) const;
+ void readWholeFile(const RowIndex& rowIndex, int start, int bytesToRead, int maximalBytes);
+ void readFileSlidingWindow(const RowIndex& rowIndex, int start, int bytesToRead, int maximalBytes);
};
#endif
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciifiledata.cpp #1320706:1320707
@@ -94,7 +94,9 @@
}
//-------------------------------------------------------------------------------------------
-AsciiFileData::AsciiFileData() : _array(new Array), _lazyRead(false), _begin(-1), _bytesRead(0), _rowBegin(-1), _rowsRead(0)
+AsciiFileData::AsciiFileData() :
+ _array(new Array), _lazyRead(false), _file(0),
+ _begin(-1), _bytesRead(0), _rowBegin(-1), _rowsRead(0)
{
}
@@ -106,15 +108,37 @@
//-------------------------------------------------------------------------------------------
char* AsciiFileData::data()
{
+ readLazy();
return _array->data();
}
//-------------------------------------------------------------------------------------------
const char* const AsciiFileData::constPointer() const
{
+ readLazy();
return _array->data();
}
+const AsciiFileData::Array& AsciiFileData::constArray() const
+{
+ readLazy();
+ return *_array;
+}
+
+void AsciiFileData::readLazy() const
+{
+ AsciiFileData* This = const_cast<AsciiFileData*>(this);
+ if (_lazyRead) {
+ if (!_file) {
+ Kst::Debug::self()->log(QString("AsciiFileData::lazyRead error: no file"), Kst::Debug::Warning);
+ } else if ( _file->openMode() != QIODevice::ReadOnly) {
+ Kst::Debug::self()->log(QString("AsciiFileData::lazyRead error: file not open"), Kst::Debug::Warning);
+ } else if (!This->read()) {
+ Kst::Debug::self()->log(QString("AsciiFileData::lazyRead error: error while reading"), Kst::Debug::Warning);
+ }
+ }
+}
+
//-------------------------------------------------------------------------------------------
bool AsciiFileData::resize(int bytes)
{
@@ -133,8 +157,7 @@
{
// force deletion of heap allocated memory if any
if (forceDeletingArray || _array->capacity() > Prealloc) {
- delete _array;
- _array = new Array;
+ _array = QSharedPointer<Array>(new Array);
}
_begin = -1;
_bytesRead = 0;
@@ -143,12 +166,12 @@
//-------------------------------------------------------------------------------------------
void AsciiFileData::release()
{
- delete _array;
- _array = 0;
+ _array.clear();
_begin = -1;
_bytesRead = 0;
}
+
//-------------------------------------------------------------------------------------------
void AsciiFileData::read(QFile& file, int start, int bytesToRead, int maximalBytes)
{
@@ -167,21 +190,24 @@
return;
}
file.seek(start); // expensive?
- int bytesRead = file.read(data(), bytesToRead);
+ int bytesRead = file.read(_array->data(), bytesToRead);
if (!resize(bytesRead + 1))
return;
- data()[bytesRead] = '\0';
+ _array->data()[bytesRead] = '\0';
_begin = start;
_bytesRead = bytesRead;
}
//-------------------------------------------------------------------------------------------
-bool AsciiFileData::lazyRead(QFile& file)
+bool AsciiFileData::read()
{
+ if (!_file || _file->openMode() != QIODevice::ReadOnly) {
+ return false;
+ }
int start = _begin;
int bytesToRead = _bytesRead;
- read(file, start, bytesToRead);
+ read(*_file, start, bytesToRead);
if (begin() != start || bytesRead() != bytesToRead) {
clear(true);
return false;
@@ -192,12 +218,18 @@
//-------------------------------------------------------------------------------------------
void AsciiFileData::logData() const
{
- QString This = QString::fromLatin1(QByteArray((const char*)this, sizeof(AsciiFileData*)).toHex()).toUpper();
- QString array = QString::fromLatin1(QByteArray((const char*)_array, sizeof(Array*)).toHex()).toUpper();
- qDebug() << QString("%1 array %2, byte %3 ... %4, row %5 ... %6")
- .arg(This).arg(array)
+ qDebug() << QString("AsciiFileData %1, array %2, byte %3 ... %4, row %5 ... %6, lazy: %7")
+ .arg(QString::number((int)this))
+ .arg(QString::number((int)_array.data()))
.arg(begin(), 8).arg(begin() + bytesRead(), 8)
- .arg(rowBegin(), 8).arg(rowBegin() + rowsRead(), 8);
+ .arg(rowBegin(), 8).arg(rowBegin() + rowsRead(), 8)
+ .arg(_lazyRead);
}
+//-------------------------------------------------------------------------------------------
+void AsciiFileData::setSharedArray(AsciiFileData& arrayData)
+{
+ _array = arrayData._array;
+}
+
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciifiledata.h #1320706:1320707
@@ -14,6 +14,7 @@
#define ASCII_FILE_DATA_H
#include <QVector>
+#include <QSharedPointer>
class QFile;
template<class T, int Prealloc>
@@ -47,12 +48,13 @@
inline void setBegin(int begin) { _begin = begin; }
inline void setBytesRead(int read) { _bytesRead = read; }
+ inline void setFile(QFile* file) { _file = file; }
+ bool read();
void read(QFile&, int start, int numberOfBytes, int maximalBytes = -1);
- bool lazyRead(QFile&);
+
char* data();
-
const char* const constPointer() const;
- inline const Array& constArray() const { return *_array; }
+ const Array& constArray() const;
bool resize(int size);
void clear(bool forceDeletingArray = false);
@@ -65,13 +67,18 @@
void logData() const;
+ void setSharedArray(AsciiFileData&);
+
private:
- Array* _array;
+ QSharedPointer<Array> _array;
+ QFile* _file;
bool _lazyRead;
int _begin;
int _bytesRead;
int _rowBegin;
int _rowsRead;
+
+ void readLazy() const;
};
Q_DECLARE_TYPEINFO(AsciiFileData, Q_MOVABLE_TYPE);
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciisource.cpp #1320706:1320707
@@ -102,25 +102,7 @@
_strings = fileMetas();
}
-
//-------------------------------------------------------------------------------------------
-bool AsciiSource::openFile(QFile &file)
-{
- // Don't use 'QIODevice::Text'!
- // Because CR LF line ending breaks row offset calculation
- return file.open(QIODevice::ReadOnly);
-}
-
-
-//-------------------------------------------------------------------------------------------
-bool AsciiSource::openValidFile(QFile &file)
-{
- _valid = openFile(file);
- return _valid;
-}
-
-
-//-------------------------------------------------------------------------------------------
bool AsciiSource::initRowIndex()
{
_reader.clear();
@@ -128,7 +110,7 @@
if (_config._dataLine > 0) {
QFile file(_filename);
- if (!openValidFile(file)) {
+ if (!AsciiFileBuffer::openFile(file)) {
return false;
}
int header_row = 0;
@@ -192,7 +174,7 @@
}
QFile file(_filename);
- if (!openValidFile(file)) {
+ if (!AsciiFileBuffer::openFile(file)) {
// Qt: If the device is closed, the size returned will not reflect the actual size of the device.
return NoChange;
}
@@ -301,21 +283,25 @@
int begin = _reader.beginOfRow(s);
int bytesToRead = _reader.beginOfRow(s + n) - begin;
if ((begin != _fileBuffer.begin()) || (bytesToRead != _fileBuffer.bytesRead())) {
- QFile file(_filename);
- if (!openValidFile(file)) {
+ QFile* file = new QFile(_filename);
+ if (!AsciiFileBuffer::openFile(*file)) {
+ delete file;
return 0;
}
- _fileBuffer.read(file, _reader.rowIndex(), begin, bytesToRead);
+ _fileBuffer.setFile(file);
+ _fileBuffer.read(_reader.rowIndex(), begin, bytesToRead);
if (_fileBuffer.bytesRead() == 0) {
success = false;
return 0;
}
- _reader.detectLineEndingType(file);
+ _reader.detectLineEndingType(*file);
}
int sRead = 0;
- const QVector<AsciiFileData> data = _fileBuffer.data();
+ const QVector<AsciiFileData>& data = _fileBuffer.data();
+ qDebug() << "Reading vector:";
foreach (const AsciiFileData& chunk, data) {
+ chunk.logData();
sRead += _reader.readField(chunk, col, v + sRead, field, chunk.rowBegin(), chunk.rowsRead());
}
@@ -341,7 +327,7 @@
QStringList AsciiSource::scalarListFor(const QString& filename, AsciiSourceConfig*)
{
QFile file(filename);
- if (!openFile(file)) {
+ if (!AsciiFileBuffer::openFile(file)) {
return QStringList();
}
return QStringList() << "FRAMES";
@@ -352,7 +338,7 @@
QStringList AsciiSource::stringListFor(const QString& filename, AsciiSourceConfig*)
{
QFile file(filename);
- if (!openFile(file)) {
+ if (!AsciiFileBuffer::openFile(file)) {
return QStringList();
}
return QStringList() << "FILE";
@@ -384,7 +370,7 @@
QStringList AsciiSource::fieldListFor(const QString& filename, AsciiSourceConfig* cfg)
{
QFile file(filename);
- if (!openFile(file)) {
+ if (!AsciiFileBuffer::openFile(file)) {
return QStringList();
}
@@ -478,7 +464,7 @@
QStringList AsciiSource::unitListFor(const QString& filename, AsciiSourceConfig* cfg)
{
QFile file(filename);
- if (!openFile(file)) {
+ if (!AsciiFileBuffer::openFile(file)) {
return QStringList();
}
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciisource.h #1320706:1320707
@@ -90,9 +90,6 @@
DataInterfaceAsciiString* is;
DataInterfaceAsciiVector* iv;
- bool openValidFile(QFile &file);
- static bool openFile(QFile &file);
-
// TODO remove
friend class DataInterfaceAsciiString;
friend class DataInterfaceAsciiVector;
More information about the Kst
mailing list