[Kst] branches/work/kst/portto4/kst

[Peter Kümmel]

SVN commit 1332952 by kuemmel:

asciisource: also read data with offset

 M  +1 -1      cmake/tests/datasource/ascii/CMakeLists.txt  
 M  +3 -2      src/datasources/ascii/asciidatareader.cpp  
 M  +2 -2      src/datasources/ascii/asciidatareader.h  
 M  +1 -1      src/datasources/ascii/asciifilebuffer.cpp  
 M  +7 -7      src/datasources/ascii/asciisource.cpp  
 M  +2 -2      src/datasources/ascii/asciisource.h  
 M  +15 -0     tests/datasources/ascii/asciifilebuffertest.cpp  


--- branches/work/kst/portto4/kst/cmake/tests/datasource/ascii/CMakeLists.txt #1332951:1332952
@@ -9,7 +9,7 @@
 include_directories(${kst_dir}/src/datasources/ascii)
 
 kst_add_plugin_lib(. ascii)
-#set_target_properties(kst2_datasource_ascii_lib PROPERTIES COMPILE_DEFINITIONS KST_SMALL_PRREALLOC)
+set_target_properties(kst2_datasource_ascii_lib PROPERTIES COMPILE_DEFINITIONS KST_SMALL_PRREALLOC)
 kst_init(test_asciisource "")
 kst_add_test(${kst_dir}/tests/datasources/ascii/asciifilebuffertest.cpp)
 kst_link(kst2_datasource_ascii_lib ${libcore} ${libmath} ${libwidgets})
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciidatareader.cpp #1332951:1332952
@@ -194,9 +194,10 @@
 }
 
 //-------------------------------------------------------------------------------------------
-int AsciiDataReader::readFieldFromChunk(const AsciiFileData& chunk, int col, double *v, const QString& field)
+int AsciiDataReader::readFieldFromChunk(const AsciiFileData& chunk, int col, double *v, int start, const QString& field)
 {
-  return readField(chunk, col, v + chunk.rowBegin(), field, chunk.rowBegin(), chunk.rowsRead());
+  Q_ASSERT(chunk.rowBegin() >= start);
+  return readField(chunk, col, v + chunk.rowBegin() - start, field, chunk.rowBegin(), chunk.rowsRead());
 }
 
 //-------------------------------------------------------------------------------------------
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciidatareader.h #1332951:1332952
@@ -41,8 +41,8 @@
     void detectLineEndingType(QFile& file);
     
     bool findDataRows(bool read_completely, QFile& file, qint64 _byteLength);
-    int readField(const AsciiFileData &buf, int col, double *v, const QString& field, int s, int n);
-    int readFieldFromChunk(const AsciiFileData& chunk, int col, double *v, const QString& field);
+    int readField(const AsciiFileData &buf, int col, double *v, const QString& field, int start, int n);
+    int readFieldFromChunk(const AsciiFileData& chunk, int col, double *v, int start, const QString& field);
 
     template<typename ColumnDelimiter>
     static int splitColumns(const QByteArray& line, const ColumnDelimiter& column_del, QStringList* cols = 0);
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciifilebuffer.cpp #1332951:1332952
@@ -84,7 +84,7 @@
       || bytesToRead <= 0 || start + bytesToRead > rowIndex[rowIndex.size() - 1])
     return QVector<AsciiFileData>();
 
-  qint64 nextRow = 0;
+  qint64 nextRow = findRowOfPosition(rowIndex, 0, start);
   QVector<AsciiFileData> chunks;
   chunks.reserve(bytesToRead / chunkSize);
   qint64 pos = start;
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciisource.cpp #1332951:1332952
@@ -329,9 +329,9 @@
 
     int read;
     if (useThreads())
-      read = parseWindowMultithreaded(slidingWindow[i], col, v, field);
+      read = parseWindowMultithreaded(slidingWindow[i], col, v, s, field);
     else
-      read = parseWindowSinglethreaded(slidingWindow[i], col, v, field, sampleRead);
+      read = parseWindowSinglethreaded(slidingWindow[i], col, v, s, field, sampleRead);
 
     // something went wrong abort reading
     if (read == 0) {
@@ -347,28 +347,28 @@
 
 
 //-------------------------------------------------------------------------------------------
-int AsciiSource::parseWindowSinglethreaded(QVector<AsciiFileData>& window, int col, double* v, const QString& field, int sRead)
+int AsciiSource::parseWindowSinglethreaded(QVector<AsciiFileData>& window, int col, double* v, int start, const QString& field, int sRead)
 {
   int read = 0;
   for (int i = 0; i < window.size(); i++) {
-    Q_ASSERT(sRead ==  window[i].rowBegin());
+    Q_ASSERT(sRead + start ==  window[i].rowBegin());
     if (!window[i].read() || window[i].bytesRead() == 0)
       return 0;
-    read += _reader.readFieldFromChunk(window[i], col, v, field);
+    read += _reader.readFieldFromChunk(window[i], col, v, start, field);
   }
   return read;
 }
 
 
 //-------------------------------------------------------------------------------------------
-int AsciiSource::parseWindowMultithreaded(QVector<AsciiFileData>& window, int col, double* v, const QString& field)
+int AsciiSource::parseWindowMultithreaded(QVector<AsciiFileData>& window, int col, double* v, int start, const QString& field)
 {
   if (!_fileBuffer.readWindow(window))
     return 0;
 
   QFutureSynchronizer<int> readFutures;
   foreach (const AsciiFileData& chunk, window) {
-    QFuture<int> future = QtConcurrent::run(&_reader, &AsciiDataReader::readFieldFromChunk, chunk, col, v, field);
+    QFuture<int> future = QtConcurrent::run(&_reader, &AsciiDataReader::readFieldFromChunk, chunk, col, v, start, field);
     readFutures.addFuture(future);
   }
   readFutures.waitForFinished();
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciisource.h #1332951:1332952
@@ -89,8 +89,8 @@
     bool useSlidingWindow(qint64 bytesToRead)  const;
 
     int tryReadField(double *v, const QString &field, int s, int n);
-    int parseWindowSinglethreaded(QVector<AsciiFileData>& fileData, int col, double* v, const QString& field, int sRead);
-    int parseWindowMultithreaded(QVector<AsciiFileData>& fileData, int col, double* v, const QString& field);
+    int parseWindowSinglethreaded(QVector<AsciiFileData>& fileData, int col, double* v, int start, const QString& field, int sRead);
+    int parseWindowMultithreaded(QVector<AsciiFileData>& fileData, int col, double* v, int start, const QString& field);
     
     
 
--- branches/work/kst/portto4/kst/tests/datasources/ascii/asciifilebuffertest.cpp #1332951:1332952
@@ -307,6 +307,21 @@
     }
     
 
+    void split_equal_chunkSize_is_bytesToRead()
+    {
+      int rows = 20;
+      int rowLength = 10;
+      int offset = 5;
+      int bytesToRead = (rows - offset) * rowLength;
+      initRowIndex(rows, rowLength);
+      int start = offset * rowLength;
+      QVector<AsciiFileData> c = buf.splitFile(bytesToRead, idx, start, bytesToRead);
+      QCOMPARE(c.size(), 1);
+      QCOMPARE(c[0].bytesRead(), bytesToRead);
+      QCOMPARE(c[0].begin(), idx[offset]);
+      QCOMPARE(c[0].rowsRead(), rows - offset);
+    }
+
     //void useOneWindowWithChunks(const RowIndex& rowIndex, int start, int bytesToRead, int numChunks);
 
     void useOneWindowWithChunks_small()