[Kst] branches/work/kst/portto4/kst/src/datasources/ascii

[Peter Kümmel]

SVN commit 1356717 by kuemmel:

check for complete fixed column size rows

 M  +22 -9     asciidatareader.cpp  
 M  +2 -2      asciidatareader.h  
 M  +2 -1      asciisource.cpp  


--- branches/work/kst/portto4/kst/src/datasources/ascii/asciidatareader.cpp #1356716:1356717
@@ -112,7 +112,7 @@
 }
 
 //-------------------------------------------------------------------------------------------
-bool AsciiDataReader::findDataRows(bool read_completely, QFile& file, qint64 _byteLength)
+bool AsciiDataReader::findDataRows(bool read_completely, QFile& file, qint64 _byteLength, int col_count)
 {
   detectLineEndingType(file);
 
@@ -131,23 +131,23 @@
     if (_config._delimiters.value().size() == 0) {
       const NoDelimiter comment_del;
       if (_lineending.isLF()) {
-        new_data = findDataRows(buf.checkedData(), buf.begin(), buf.bytesRead(), IsLineBreakLF(_lineending), comment_del);
+        new_data = findDataRows(buf.checkedData(), buf.begin(), buf.bytesRead(), IsLineBreakLF(_lineending), comment_del, col_count);
       } else {
-        new_data = findDataRows(buf.checkedData(), buf.begin(), buf.bytesRead(), IsLineBreakCR(_lineending), comment_del);
+        new_data = findDataRows(buf.checkedData(), buf.begin(), buf.bytesRead(), IsLineBreakCR(_lineending), comment_del, col_count);
       }
     } else if (_config._delimiters.value().size() == 1) {
       const IsCharacter comment_del(_config._delimiters.value()[0].toLatin1());
       if (_lineending.isLF()) {
-        new_data = findDataRows(buf.checkedData(), buf.begin(), buf.bytesRead(), IsLineBreakLF(_lineending), comment_del);
+        new_data = findDataRows(buf.checkedData(), buf.begin(), buf.bytesRead(), IsLineBreakLF(_lineending), comment_del, col_count);
       } else {
-        new_data = findDataRows(buf.checkedData(), buf.begin(), buf.bytesRead(), IsLineBreakCR(_lineending), comment_del);
+        new_data = findDataRows(buf.checkedData(), buf.begin(), buf.bytesRead(), IsLineBreakCR(_lineending), comment_del, col_count);
       }
     } else if (_config._delimiters.value().size() > 1) {
       const IsInString comment_del(_config._delimiters.value());
       if (_lineending.isLF()) {
-        new_data = findDataRows(buf.checkedData(), buf.begin(), buf.bytesRead(), IsLineBreakLF(_lineending), comment_del);
+        new_data = findDataRows(buf.checkedData(), buf.begin(), buf.bytesRead(), IsLineBreakLF(_lineending), comment_del, col_count);
       } else {
-        new_data = findDataRows(buf.checkedData(), buf.begin(), buf.bytesRead(), IsLineBreakCR(_lineending), comment_del);
+        new_data = findDataRows(buf.checkedData(), buf.begin(), buf.bytesRead(), IsLineBreakCR(_lineending), comment_del, col_count);
       }
     }
   } while (buf.bytesRead() == AsciiFileData::Prealloc - 1  && read_completely);
@@ -157,7 +157,7 @@
 
 //-------------------------------------------------------------------------------------------
 template<class Buffer, typename IsLineBreak, typename CommentDelimiter>
-bool AsciiDataReader::findDataRows(const Buffer& buffer, qint64 bufstart, qint64 bufread, const IsLineBreak& isLineBreak, const CommentDelimiter& comment_del)
+bool AsciiDataReader::findDataRows(const Buffer& buffer, qint64 bufstart, qint64 bufread, const IsLineBreak& isLineBreak, const CommentDelimiter& comment_del, int col_count)
 {
   const IsWhiteSpace isWhiteSpace;
   bool new_data = false;
@@ -190,6 +190,19 @@
     }
   }
   _rowIndex[_numFrames] = row_start;
+
+  if (_config._columnType == AsciiSourceConfig::Fixed) {
+    // only read complete lines, last  column could be only 1 char long
+    if (_rowIndex.size() > 1) {
+      for (int i = 1; i <= _numFrames; i++) {
+        if (_rowIndex[i] <= _rowIndex[i - 1] + col_count * (_config._columnWidth - 1) + 1) {
+        _rowIndex.resize(i);
+        _numFrames = i - 1;
+        }
+      }
+    }
+  }
+
   return new_data;
 }
 
@@ -208,7 +221,7 @@
     const LexicalCast& lexc = LexicalCast::instance();
     // &buffer[0] points to first row at _rowIndex[0] , so if we wanna find
     // the column in row i by adding _rowIndex[i] we have to start at:
-    const char* col_start = &buf.checkedData()[0] - _rowIndex[0] + _config._columnWidth * (col - 1);
+    const char*const col_start = &buf.checkedData()[0] - _rowIndex[0] + _config._columnWidth * (col - 1);
     for (int i = 0; i < n; ++i) {
       v[i] = lexc.toDouble(_rowIndex[i] + col_start);
     }
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciidatareader.h #1356716:1356717
@@ -40,7 +40,7 @@
     
     void detectLineEndingType(QFile& file);
     
-    bool findDataRows(bool read_completely, QFile& file, qint64 _byteLength);
+    bool findDataRows(bool read_completely, QFile& file, qint64 _byteLength, int col_count);
     int readField(const AsciiFileData &buf, int col, double *v, const QString& field, int start, int n);
     int readFieldFromChunk(const AsciiFileData& chunk, int col, double *v, int start, const QString& field);
 
@@ -72,7 +72,7 @@
                     const IsLineBreak&, const ColumnDelimiter&, const CommentDelimiter&, const ColumnWidthsAreConst&) const;
 
     template<class Buffer, typename IsLineBreak, typename CommentDelimiter>
-    bool findDataRows(const Buffer& buffer, qint64 bufstart, qint64 bufread, const IsLineBreak&, const CommentDelimiter&);
+    bool findDataRows(const Buffer& buffer, qint64 bufstart, qint64 bufread, const IsLineBreak&, const CommentDelimiter&, int col_count);
 
     void toDouble(const LexicalCast& lexc, const char* buffer, qint64 bufread, qint64 ch, double* v, int row) const;
 
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciisource.cpp #1356716:1356717
@@ -190,7 +190,8 @@
   _fileSize = file.size();
   _fileCreationTime_t = QFileInfo(file).created().toTime_t();
 
-  bool new_data = _reader.findDataRows(read_completely, file, _fileSize);
+  int col_count = _fieldList.size() - 1; // minus INDEX
+  bool new_data = _reader.findDataRows(read_completely, file, _fileSize, col_count);
   
   return (!new_data && !force_update ? NoChange : Updated);
 }