[Kst] branches/work/kst/portto4/kst/src/datasources/ascii

[Peter Kümmel]

SVN commit 1320705 by kuemmel:

split out AsciiFileData

 M  +5 -160    asciifilebuffer.cpp  
 M  +3 -52     asciifilebuffer.h  
 A             asciifiledata.cpp   asciifilebuffer.cpp#1320704 [License: GPL (v2+)]
 A             asciifiledata.h   asciifilebuffer.h#1320704 [License: GPL (v2+)]


--- branches/work/kst/portto4/kst/src/datasources/ascii/asciifilebuffer.cpp #1320704:1320705
@@ -10,173 +10,19 @@
  *                                                                         *
  ***************************************************************************/
 
-// needed to track memeory usage
-#include "qplatformdefs.h"
-#include <stdlib.h>
-void* fileBufferMalloc(size_t bytes);
-void fileBufferFree(void* ptr);
-#define malloc fileBufferMalloc
-#define qMalloc fileBufferMalloc
-#define free fileBufferFree
-#define qFree fileBufferFree
-#include <QVarLengthArray>
-#undef malloc
-#undef qMalloc
-#undef free
-#undef qFree
-
 #include "asciifilebuffer.h"
 #include "debug.h"
 
 #include <QFile>
 #include <QDebug>
+#include <QVarLengthArray>
 
 
-static int MB = 1024*1024;
-
-// Simulate out of memory scenario
-//#define KST_TEST_OOM
-
-#ifdef KST_TEST_OOM
-static size_t maxAllocate = 2 * MB;
-#else
-static size_t maxAllocate = (size_t) -1;
-#endif
-
-#define KST_MEMORY_DEBUG if(1)
-
 //-------------------------------------------------------------------------------------------
-static QMap<void*, size_t> allocatedMBs;
+extern int MB;
+extern size_t maxAllocate;
 
 //-------------------------------------------------------------------------------------------
-static void logMemoryUsed()
-{
-  size_t sum = 0;
-  QMapIterator<void*, size_t> it(allocatedMBs);
-  while (it.hasNext()) {
-    it.next();
-    sum +=  it.value();
-  }
-  Kst::Debug::self()->log(QString("AsciiFileData: %1 MB used").arg(sum / MB), Kst::Debug::Warning);
-  KST_MEMORY_DEBUG qDebug() << "AsciiFileData: " << sum / MB<< "MB used";
-}
-
-//-------------------------------------------------------------------------------------------
-void* fileBufferMalloc(size_t bytes)
-{
-  void* ptr = 0;
-#ifdef KST_TEST_OOM
-  if (bytes <= maxAllocate)
-#endif
-    ptr = malloc(bytes);
-  if (ptr)  {
-    allocatedMBs[ptr] = bytes;
-    KST_MEMORY_DEBUG qDebug() << "AsciiFileBuffer: " << bytes / MB << "MB allocated";
-    KST_MEMORY_DEBUG logMemoryUsed();
-  } else {
-    Kst::Debug::self()->log(QString("AsciiFileData: failed to allocate %1 MBs").arg(bytes / MB), Kst::Debug::Warning);
-    logMemoryUsed();
-    KST_MEMORY_DEBUG qDebug() << "AsciiFileData: error when allocating " << bytes / MB << "MB";
-  }
-  return ptr;
-}
-
-//-------------------------------------------------------------------------------------------
-void fileBufferFree(void* ptr)
-{
-  if (allocatedMBs.contains(ptr)) {
-    KST_MEMORY_DEBUG qDebug() << "AsciiFileData: " << allocatedMBs[ptr] / MB << "MB freed";
-    allocatedMBs.remove(ptr);
-  }
-  KST_MEMORY_DEBUG logMemoryUsed();
-  free(ptr);
-}
-
-//-------------------------------------------------------------------------------------------
-AsciiFileData::AsciiFileData() : _array(new Array), _begin(-1), _bytesRead(0), _rowBegin(-1), _rowsRead(0)
-{
-}
-
-//-------------------------------------------------------------------------------------------
-AsciiFileData::~AsciiFileData()
-{
-}
-
-//-------------------------------------------------------------------------------------------
-char* AsciiFileData::data()
-{
-  return _array->data();
-}
-
-//-------------------------------------------------------------------------------------------
-const char* const AsciiFileData::constPointer() const
-{
-  return _array->data();
-}
-
-//-------------------------------------------------------------------------------------------
-bool AsciiFileData::resize(int bytes)
-{ 
-  try {
-    _array->resize(bytes);
-  } catch (const std::bad_alloc&) {
-    // work around Qt bug
-    clear(true);
-    return false;
-  }
-  return true;
-}
-
-//-------------------------------------------------------------------------------------------
-void AsciiFileData::clear(bool forceDeletingArray)
-{
-  // force deletion of heap allocated memory if any
-  if (forceDeletingArray || _array->capacity() > Prealloc) {
-    delete _array;
-    _array = new Array;
-  }
-  _begin = -1;
-  _bytesRead = 0;
-}
-
-//-------------------------------------------------------------------------------------------
-void AsciiFileData::release()
-{
-  delete _array;
-  _array = 0;
-  _begin = -1;
-  _bytesRead = 0;
-}
-
-//-------------------------------------------------------------------------------------------
-void AsciiFileData::read(QFile& file, int start, int bytesToRead, int maximalBytes)
-{
-  _begin = -1;
-  _bytesRead = 0;
-
-  if (bytesToRead <= 0)
-    return;
-
-  if (maximalBytes == -1) {
-    if (!resize(bytesToRead + 1))
-      return;
-  } else {
-    bytesToRead = qMin(bytesToRead, maximalBytes);
-    if (!resize(bytesToRead + 1))
-      return;
-  }
-  file.seek(start); // expensive?
-  int bytesRead = file.read(data(), bytesToRead);
-  if (!resize(bytesRead + 1))
-    return;
-
-  data()[bytesRead] = '\0';
-  _begin = start;
-  _bytesRead = bytesRead;
-}
-
-
-//-------------------------------------------------------------------------------------------
 AsciiFileBuffer::AsciiFileBuffer()
 {
 }
@@ -188,7 +34,7 @@
 }
 
 //-------------------------------------------------------------------------------------------
-void AsciiFileBuffer::clear(bool forceDeletingArray)
+void AsciiFileBuffer::clear()
 {
   foreach (AsciiFileData chunk, _fileData) {
     chunk.release();
@@ -215,7 +61,7 @@
 //-------------------------------------------------------------------------------------------
 static int findRowOfPosition(const AsciiFileBuffer::RowIndex& rowIndex, int searchStart, int pos)
 {
-  //TODO too expensive
+  //TODO too expensive?
   const int size = rowIndex.size();
   for (int row = searchStart; row != size; row++) {
     if (rowIndex[row] > pos)
@@ -246,7 +92,6 @@
     wholeFile.release();
   }
 
-
   // reading whole file into one array failed, try to read into smaller arrays
   int chunkSize = qMin((size_t) 10 * MB, maxAllocate);
   int end = start + bytesToRead;
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciifilebuffer.h #1320704:1320705
@@ -13,60 +13,11 @@
 #ifndef ASCII_FILE_BUFFER_H
 #define ASCII_FILE_BUFFER_H
 
+#include "asciifiledata.h"
+
 #include <QVector>
 
-template<class T, int Prealloc>
-class QVarLengthArray;
 
-class QFile;
-
-class AsciiFileData
-{
-public:
-
-  enum SizeOnStack
-  {
-    Prealloc =
-#if defined(__ANDROID__) || defined(__QNX__) // Some mobile systems really do not like you allocating 1MB on the stack.
-    1 * 1024
-#else
-    1 * 1024 * 1024
-#endif
-  };
-
-  typedef QVarLengthArray<char, Prealloc> Array;
-  
-  AsciiFileData();
-  ~AsciiFileData();
-
-  inline int begin() const { return _begin; }
-  inline int bytesRead() const { return _bytesRead; }
-  void read(QFile&, int start, int numberOfBytes, int maximalBytes = -1);
-  char* data();
-
-  const char* const constPointer() const;
-  inline const Array& constArray() const { return *_array; }
-
-  bool resize(int size);
-  void clear(bool forceDeletingArray = false);
-  void release();
-
-  inline int rowBegin() const { return _rowBegin; }
-  inline int rowsRead() const { return _rowsRead; }
-  inline void setRowBegin(int begin) { _rowBegin = begin; }
-  inline void setRowsRead(int read) { _rowsRead = read; }
-
-private:
-  Array* _array;
-  int _begin;
-  int _bytesRead;
-  int _rowBegin;
-  int _rowsRead;
-};
-
-Q_DECLARE_TYPEINFO(AsciiFileData, Q_MOVABLE_TYPE);
-
-
 class AsciiFileBuffer
 {
 public:
@@ -78,7 +29,7 @@
   inline int begin() const { return _begin; }
   inline int bytesRead() const { return _bytesRead; }
 
-  void clear(bool forceDeletingArray = false);
+  void clear();
   
   void read(QFile&, const RowIndex& rowIndex, int start, int numberOfBytes, int maximalBytes = -1);