[Kst] branches/work/kst/portto4/kst/src/datasources/ascii

[Peter Kümmel]

SVN commit 1320713 by kuemmel:

make splitting code more readable

 M  +18 -14    asciifilebuffer.cpp  
 M  +1 -0      asciifilebuffer.h  
 M  +3 -3      asciifiledata.cpp  


--- branches/work/kst/portto4/kst/src/datasources/ascii/asciifilebuffer.cpp #1320712:1320713
@@ -25,7 +25,7 @@
 //-------------------------------------------------------------------------------------------
 AsciiFileBuffer::AsciiFileBuffer() : 
   _file(0), _begin(-1), _bytesRead(0),
-  _defaultChunkSize(qMin((size_t) 10 * MB, maxAllocate))
+  _defaultChunkSize(qMin((size_t) 10 * MB, maxAllocate - 1))
 {
 }
 
@@ -73,7 +73,7 @@
 }
 
 //-------------------------------------------------------------------------------------------
-static int findRowOfPosition(const AsciiFileBuffer::RowIndex& rowIndex, int searchStart, int pos)
+int AsciiFileBuffer::findRowOfPosition(const AsciiFileBuffer::RowIndex& rowIndex, int searchStart, int pos) const
 {
   //TODO too expensive?
   const int size = rowIndex.size();
@@ -90,28 +90,32 @@
 {
   // reading whole file into one array failed, try to read into smaller arrays
   const int end = start + bytesToRead;
-  int chunkRead = 0;
-  int lastRow = 0;
+  int endsInRow = 0;
   QVector<AsciiFileData> chunks;
-  for (int pos = start; pos < end; pos += chunkRead) {
+  int pos = start;
+  while (pos < end) {
     // use for storing reading information only
     AsciiFileData chunk;
     // read complete chunk or to end of file
-    chunkRead = (pos + chunkSize < end ? chunkSize : end - pos);
+    int endRead = (pos + chunkSize < end ? pos + chunkSize : end);
     // adjust to row end: pos + chunkRead is in the middle of a row, find index of this row
-    const int rowBegin = lastRow;
-    lastRow = findRowOfPosition(rowIndex, lastRow, pos + chunkRead);
+    const int rowBegin = endsInRow;
+    endsInRow = findRowOfPosition(rowIndex, endsInRow, endRead);
     // read until the beginning of this row
-    chunkRead = (rowIndex[lastRow] - 1);
+    endRead = rowIndex[endsInRow];
     // check if it is the last row, and read remaining bytes from pos
-    chunkRead = (lastRow == rowIndex.size() - 1) ? end - pos : chunkRead - pos;
+    if (endsInRow == rowIndex.size() - 1)
+      endRead = end;
     // set information about positions in the file
     chunk.setBegin(pos);
-    chunk.setBytesRead(chunkRead);
+    chunk.setBytesRead(endRead - pos);
     // set information about rows
     chunk.setRowBegin(rowBegin);
-    chunk.setRowsRead(lastRow - rowBegin);
+    chunk.setRowsRead(endsInRow - rowBegin);
     chunks << chunk;
+    if (endsInRow ==  rowIndex.size() - 1)
+      break;
+    pos = rowIndex[endsInRow];
   }
   //qDebug() << "File splitted into " << chunks.size() << " chunks:"; logData(chunks);
   return chunks;
@@ -130,7 +134,7 @@
   wholeFile.read(*_file, start, bytesToRead, maximalBytes);
   if (bytesToRead == wholeFile.bytesRead()) {
     wholeFile.setRowBegin(0);
-    wholeFile.setRowsRead(rowIndex.size());
+    wholeFile.setRowsRead(rowIndex.size() - 1);
     _begin = start;
     _bytesRead = bytesToRead;
     _fileData << wholeFile;
@@ -172,7 +176,7 @@
     Kst::Debug::self()->log(QString("AsciiFileBuffer: not enough memory available for creating sliding window"));
   }
   for (int i = 0; i < _fileData.size(); i++) {
-    // use alread set
+    // reading from file is delayed until the data is accessed
     _fileData[i].setLazyRead(true);
     _fileData[i].setFile(_file);
     _fileData[i].setSharedArray(master);
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciifilebuffer.h #1320712:1320713
@@ -48,6 +48,7 @@
 
   void logData(const QVector<AsciiFileData>& chunks) const;
   const QVector<AsciiFileData> splitFile(int chunkSize, const RowIndex& rowIndex, int start, int bytesToRead) const;
+  int findRowOfPosition(const AsciiFileBuffer::RowIndex& rowIndex, int searchStart, int pos) const;
 };
 
 #endif
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciifiledata.cpp #1320712:1320713
@@ -39,7 +39,7 @@
 //#define KST_TEST_OOM
 
 #ifdef KST_TEST_OOM
-size_t maxAllocate = 2 * MB;
+size_t maxAllocate = 1 * MB;
 #else
 size_t maxAllocate = (size_t) -1;
 #endif
@@ -209,12 +209,12 @@
 //-------------------------------------------------------------------------------------------
 void AsciiFileData::logData() const
 {
-  qDebug() << QString("AsciiFileData %1, array %2, byte %3 ... %4, row %5 ... %6, lazy: %7")
+  qDebug() << QString("AsciiFileData %1, array %2, byte %3 ... %4 (%8), row %5 ... %6 (%9), lazy: %7")
     .arg(QString::number((int)this))
     .arg(QString::number((int)_array.data()))
     .arg(begin(), 8).arg(begin() + bytesRead(), 8)
     .arg(rowBegin(), 8).arg(rowBegin() + rowsRead(), 8)
-    .arg(_lazyRead);
+    .arg(_lazyRead).arg(bytesRead(), 8).arg(rowsRead(), 8);
 }