[Kst] branches/work/kst/portto4/kst/src/datasources/ascii

[Peter Kümmel]

SVN commit 1319572 by kuemmel:

read very big files not at once

 M  +60 -0     asciisource.cpp  
 M  +9 -1      asciisource.h  


--- branches/work/kst/portto4/kst/src/datasources/ascii/asciisource.cpp #1319571:1319572
@@ -30,6 +30,7 @@
 #include "measuretime.h"
 
 #include <QFile>
+#include <QMessageBox>
 
 #include <assert.h>
 #include <ctype.h>
@@ -484,8 +485,63 @@
 
 
 //-------------------------------------------------------------------------------------------
+void AsciiSource::clearFileBuffer()
+{
+  _tmpBuffer.clear();
+  _bufferedS = -10;
+  _bufferedN = -10;
+}
+
+//-------------------------------------------------------------------------------------------
 int AsciiSource::readField(double *v, const QString& field, int s, int n) 
 {
+  bool re_alloc;
+  int n_read = readField(v, field, s, n, re_alloc);
+  if (re_alloc) {
+    // file is now buffered in memory
+    return n_read;
+  }
+
+  // reading whole file into memory failed
+
+  // find a smaller allocatable size
+  clearFileBuffer();
+  int realloc_size = n / 2;
+  _tmpBuffer.resize(realloc_size);
+  while (_tmpBuffer.size() != realloc_size && realloc_size > 0) {
+      realloc_size /= 2;
+      _tmpBuffer.resize(realloc_size);
+  }
+  realloc_size /= 2; // while reading available memory could shrink, just be sure
+  if (realloc_size == 0) {
+    QMessageBox::warning(0, "Error while reading ascii file", "File could not be read because not enough memory is available.");
+    return 0;      
+  }
+
+  // read in 
+  int start = s;
+  n_read = 0;
+  while (n_read < n) {
+    clearFileBuffer();
+    int to_read = n_read + realloc_size < n ? realloc_size : n - n_read;
+    n_read += readField(v + start, field, n_read, to_read, re_alloc);
+    if (!re_alloc) {
+      clearFileBuffer();
+      QMessageBox::warning(0, "Error while reading ascii file", "The file was only read partially not enough memory is available.");
+      return n_read; 
+    }
+    start += to_read;
+  }
+  // don't buffer partial files
+  clearFileBuffer();
+  return n_read;
+}
+
+
+//-------------------------------------------------------------------------------------------
+int AsciiSource::readField(double *v, const QString& field, int s, int n, bool& re_alloc) 
+{
+  re_alloc = true;
   if (n < 0) {
     n = 1; /* n < 0 means read one sample, not frame - irrelevent here */
   }
@@ -518,6 +574,10 @@
 
 
     bufread = readFromFile(file, _tmpBuffer, bufstart, bufread);
+    if (bufread == 0) {
+      re_alloc = false;
+      return 0;
+    }
     _bufferedS = s;
     _bufferedN = n;
   }
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciisource.h #1319571:1319572
@@ -50,7 +50,6 @@
 
     int readField(double *v, const QString &field, int s, int n);
 
-
     QString fileType() const;
 
     void save(QXmlStreamWriter &s);
@@ -89,6 +88,7 @@
     QVarLengthArray<char, KST_PREALLOC> _tmpBuffer;
     int _bufferedS;
     int _bufferedN;
+    void clearFileBuffer();
     QVarLengthArray<int, KST_PREALLOC> _rowIndex;
 
     friend class ConfigWidgetAscii;
@@ -118,6 +118,9 @@
     template<class T>
     int readFromFile(QFile&, T& buffer, int start, int numberOfBytes, int maximalBytes = -1);
 
+    int readField(double *v, const QString &field, int s, int n, bool& re_alloc);
+
+
     struct LineEndingType {
       bool is_crlf;
       char character;
@@ -250,12 +253,17 @@
 template<class T>
 int AsciiSource::readFromFile(QFile& file, T& buffer, int start, int bytesToRead, int maximalBytes)
 {    
+  const int oldSize = buffer.size();
   if (maximalBytes == -1) {
     buffer.resize(bytesToRead + 1);
+    if (buffer.size() == oldSize)
+      return 0;
   } else {
     bytesToRead = qMin(bytesToRead, maximalBytes);
     if (buffer.size() <= bytesToRead) {
       buffer.resize(bytesToRead + 1);
+      if (buffer.size() == oldSize)
+        return 0;
     }
   }
   file.seek(start); // expensive?