[Kst] branches/work/kst/portto4/kst
Peter Kümmel
syntheticpp at gmx.net
Sun Oct 21 15:22:30 UTC 2012
SVN commit 1321749 by kuemmel:
add unit test for AsciiFileBuffer
and fix the found bug
M +34 -7 cmake/modules/KstMacros.cmake
M +0 -3 cmake/src/datasources/CMakeLists.txt
M +1 -1 cmake/tests/CMakeLists.txt
A cmake/tests/datasource (directory)
A cmake/tests/datasource/ascii (directory)
A cmake/tests/datasource/ascii/CMakeLists.txt
M +6 -6 src/datasources/ascii/asciidatareader.cpp
M +46 -24 src/datasources/ascii/asciifilebuffer.cpp
M +2 -0 src/datasources/ascii/asciifilebuffer.h
M +8 -7 src/datasources/ascii/asciifiledata.cpp
M +2 -1 src/datasources/ascii/asciifiledata.h
A tests/datasources (directory)
A tests/datasources/ascii (directory)
A tests/datasources/ascii/asciifilebuffertest.cpp [License: GPL (v2+)]
--- branches/work/kst/portto4/kst/cmake/modules/KstMacros.cmake #1321748:1321749
@@ -84,6 +84,26 @@
endmacro()
+macro(kst_add_test _source_file)
+ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${kst_build_dir}/bin)
+ get_filename_component(_file_name ${_source_file} NAME)
+ if(CMAKE_GENERATOR MATCHES Ninja)
+ set(_moced ${CMAKE_CURRENT_BINARY_DIR}/moc_${_file_name}) # ninja generator bug
+ else()
+ set(_moced moc_${_file_name})
+ endif()
+ qt4_generate_moc(${_source_file} ${_moced})
+ set_source_files_properties(${_source_file} PROPERTIES OBJECT_DEPENDS ${_moced}) # moc on source file changes
+ add_executable(${kst_name} ${_source_file})
+ set_target_properties(${kst_name} PROPERTIES DEPEND ${_moced})
+ target_link_libraries(${kst_name} ${QT_QTTEST_LIBRARY})
+ set_property(TARGET ${kst_name} PROPERTY DEBUG_POSTFIX ${kst_debug_postfix})
+ kst_link(${libcore} ${libmath} ${libapp} ${libwidgets} ${QT_QTTEST_LIBRARY})
+ set_target_properties(${kst_name} PROPERTIES DEBUG_POSTFIX ${kst_debug_postfix})
+ add_test(NAME ${kst_name} COMMAND ${kst_name})
+endmacro()
+
+
macro(kst_install_executable)
install(TARGETS ${kst_name}
RUNTIME DESTINATION bin COMPONENT Runtime
@@ -153,7 +173,7 @@
endmacro()
-macro(kst_add_plugin folder name)
+macro(kst_add_plugin_internal folder name libtype postfix)
set(_name _${kst_plugin_prefix}_${folder}_${name})
string(REPLACE . _ _name ${_name})
string(REPLACE / _ _name ${_name})
@@ -161,12 +181,7 @@
string(REPLACE __ _ _name ${_name})
kst_init(${kst_binary_name} ${_name})
kst_files_find(${kst_plugin_dir}/${folder}/${name})
- add_library(${kst_name} MODULE ${kst_${kst_name}_sources} ${kst_${kst_name}_headers})
- kst_link(${libcore} ${libmath} ${libwidgets})
- if(NOT APPLE)
- install(TARGETS ${kst_name} LIBRARY DESTINATION ${kst_install_plugins})
-# kst_find_install_desktop_file(${kst_plugin_dir}/${folder}/${name})
- endif()
+ add_library(${kst_name}${postfix} ${libtype} ${kst_${kst_name}_sources} ${kst_${kst_name}_headers})
add_dependencies(${kst_binary_name} ${kst_name})
kst_flat_source_group(${kst_${kst_name}_headers} ${kst_${kst_name}_sources_not_generated})
if(kst_verbose)
@@ -174,6 +189,18 @@
endif()
endmacro()
+macro(kst_add_plugin folder name)
+ kst_add_plugin_internal(${folder} ${name} MODULE "")
+ kst_link(${libcore} ${libmath} ${libwidgets})
+ if(NOT APPLE)
+ install(TARGETS ${kst_name} LIBRARY DESTINATION ${kst_install_plugins})
+ endif()
+endmacro()
+
+macro(kst_add_plugin_lib folder name)
+ kst_add_plugin_internal(${folder} ${name} STATIC _lib)
+endmacro()
+
macro(kst_add_dependency name)
add_dependencies(${kst_name} ${name})
endmacro()
--- branches/work/kst/portto4/kst/cmake/src/datasources/CMakeLists.txt #1321748:1321749
@@ -56,6 +56,3 @@
kst_add_plugin(. matlab) # Directory named after the file format, not the lib
kst_link(${MATIO_LIBRARIES})
endif()
-
-
-
--- branches/work/kst/portto4/kst/cmake/tests/CMakeLists.txt #1321748:1321749
@@ -42,5 +42,5 @@
# like done with qmake
#add_subdirectory(allinone)
add_subdirectory(realloc)
+add_subdirectory(datasource/ascii)
-
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciidatareader.cpp #1321748:1321749
@@ -54,7 +54,7 @@
void AsciiDataReader::clear()
{
_rowIndex.clear();
- setRow0Begin(-1);
+ setRow0Begin(0);
_numFrames = 0;
}
@@ -121,8 +121,8 @@
// Read the tmpbuffer, starting at row_index[_numFrames]
buf.clear();
- // always read from the start of a line
- buf.read(file, _rowIndex[_numFrames], _byteLength - buf.begin(), AsciiFileData::Prealloc - 1);
+ int bufstart = _rowIndex[_numFrames]; // always read from the start of a line
+ buf.read(file, bufstart, _byteLength - bufstart, AsciiFileData::Prealloc - 1);
if (buf.bytesRead() == 0) {
return false;
}
@@ -151,8 +151,6 @@
}
} while (buf.bytesRead() == AsciiFileData::Prealloc - 1 && read_completely);
- _rowIndex.resize(_numFrames + 1);
-
return new_data;
}
@@ -175,7 +173,9 @@
_rowIndex[_numFrames] = row_start;
++_numFrames;
if (_numFrames >= _rowIndex.size()) {
- _rowIndex.resize(_rowIndex.size() + AsciiFileData::Prealloc - 1);
+ if (_rowIndex.capacity() < _numFrames + 1)
+ _rowIndex.reserve(_numFrames + AsciiFileData::Prealloc);
+ _rowIndex.resize(_numFrames + 1);
}
new_data = true;
row_start = row_offset+i;
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciifilebuffer.cpp #1321748:1321749
@@ -60,50 +60,72 @@
//-------------------------------------------------------------------------------------------
int AsciiFileBuffer::findRowOfPosition(const AsciiFileBuffer::RowIndex& rowIndex, int searchStart, int pos) const
{
- //TODO too expensive?
- const int size = rowIndex.size();
- for (int row = searchStart; row != size; row++) {
- if (rowIndex[row] > pos)
+ if (rowIndex.isEmpty() ||
+ pos < 0 || pos >= rowIndex[rowIndex.size() - 1] || // within the file
+ searchStart > rowIndex.size()-1 || pos < rowIndex[searchStart]) //within the search region
+ return -1;
+
+ // is expensive for large index with searchStart == 0
+ const int indexOfLastRow = rowIndex.size() - 2;
+ for (int row = searchStart; row <= indexOfLastRow; row++) {
+ if (pos < rowIndex[row])
return row - 1;
}
- // must be the last row
- return size - 1;
+ if (pos < rowIndex[indexOfLastRow + 1]) // length of file in the last element
+ return indexOfLastRow;
+ return -1;
}
//-------------------------------------------------------------------------------------------
const QVector<AsciiFileData> AsciiFileBuffer::splitFile(int chunkSize, const RowIndex& rowIndex, int start, int bytesToRead) const
{
- // reading whole file into one array failed, try to read into smaller arrays
- const int end = start + bytesToRead;
- int endsInRow = 0;
+ const int end = start + bytesToRead; // position behind last valid seekable byte in file
+ if (chunkSize <= 0 || rowIndex.isEmpty() || start >= end || start < 0
+ || bytesToRead <= 0 || start + bytesToRead > rowIndex[rowIndex.size() - 1])
+ return QVector<AsciiFileData>();
+
+ int nextRow = 0;
QVector<AsciiFileData> chunks;
- if (chunkSize == 0)
- return chunks;
chunks.reserve(bytesToRead / chunkSize);
int pos = start;
+ int rows = rowIndex.size();
while (pos < end) {
// use for storing reading information only
AsciiFileData chunk;
+ // error if chunkSize is too small for one row
+ if (nextRow + 1 < rows && rowIndex[nextRow + 1] - rowIndex[nextRow] > chunkSize)
+ return QVector<AsciiFileData>();
// read complete chunk or to end of file
int endRead = (pos + chunkSize < end ? pos + chunkSize : end);
// adjust to row end: pos + chunkRead is in the middle of a row, find index of this row
- const int rowBegin = endsInRow;
- endsInRow = findRowOfPosition(rowIndex, endsInRow, endRead);
- // read until the beginning of this row
- endRead = rowIndex[endsInRow];
- // check if it is the last row, and read remaining bytes from pos
- if (endsInRow == rowIndex.size() - 1)
+ const int rowBegin = nextRow;
+ nextRow = findRowOfPosition(rowIndex, nextRow, endRead - 1);
+ if (nextRow == -1 || nextRow >= rows)
+ return QVector<AsciiFileData>();
+ // read until the beginning of the found row
+ if (nextRow == rows - 2) { // last valid row
+ // if exactly at the end of the row, read this row
+ if (endRead == rowIndex[rows - 1]) {
+ nextRow++;
endRead = end;
+ } else {
+ // find complete last row next time
+ endRead = end - 1;
+ }
+ } else {
+ // if exactly at the end of the row, read this row
+ if (endRead == rowIndex[nextRow + 1])
+ nextRow++;
+ endRead = rowIndex[nextRow];
+ }
// set information about positions in the file
- chunk.setBegin(pos);
- chunk.setBytesRead(endRead - pos);
+ chunk.setBegin(rowIndex[rowBegin]);
+ chunk.setBytesRead(rowIndex[nextRow] - rowIndex[rowBegin]);
// set information about rows
chunk.setRowBegin(rowBegin);
- chunk.setRowsRead(endsInRow - rowBegin);
+ chunk.setRowsRead(nextRow - rowBegin);
chunks << chunk;
- if (endsInRow == rowIndex.size() - 1)
- break;
- pos = rowIndex[endsInRow];
+ pos = rowIndex[nextRow];
}
//qDebug() << "File splitted into " << chunks.size() << " chunks:"; AsciiFileData::logData(chunks);
return chunks;
@@ -134,7 +156,7 @@
if (!_file)
return;
- if (bytesToRead == 0 || numWindowChunks == 0 || windowSize == 0)
+ if (bytesToRead <= 0 || numWindowChunks <= 0 || windowSize <= 0)
return;
int chunkSize = windowSize / numWindowChunks;
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciifilebuffer.h #1321748:1321749
@@ -52,6 +52,8 @@
const QVector<AsciiFileData> splitFile(int chunkSize, const RowIndex& rowIndex, int start, int bytesToRead) const;
int findRowOfPosition(const AsciiFileBuffer::RowIndex& rowIndex, int searchStart, int pos) const;
void useSlidingWindowWithChunks(const RowIndex& rowIndex, int start, int bytesToRead, int windowSize, int numWindowChunks, bool reread);
+
+ friend class AsciiSourceTest;
};
#endif
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciifiledata.cpp #1321748:1321749
@@ -60,8 +60,8 @@
it.next();
sum += it.value();
}
- Kst::Debug::self()->log(QString("AsciiFileData: %1 MB used").arg(sum / MB), Kst::Debug::Warning);
- KST_MEMORY_DEBUG qDebug() << "AsciiFileData: " << sum / MB<< "MB used";
+ if(sum / MB != 0) Kst::Debug::self()->log(QString("AsciiFileData: %1 MB used").arg(sum / MB), Kst::Debug::Warning);
+ KST_MEMORY_DEBUG if(sum / MB != 0) qDebug() << "AsciiFileData: " << sum / MB<< "MB used";
}
//-------------------------------------------------------------------------------------------
@@ -74,8 +74,8 @@
ptr = malloc(bytes);
if (ptr) {
allocatedMBs[ptr] = bytes;
- KST_MEMORY_DEBUG qDebug() << "AsciiFileBuffer: " << bytes / MB << "MB allocated";
- KST_MEMORY_DEBUG logMemoryUsed();
+ KST_MEMORY_DEBUG if(bytes / MB != 0) qDebug() << "AsciiFileBuffer: " << bytes / MB << "MB allocated";
+ KST_MEMORY_DEBUG if(bytes / MB != 0) logMemoryUsed();
} else {
Kst::Debug::self()->log(QString("AsciiFileData: failed to allocate %1 MBs").arg(bytes / MB), Kst::Debug::Warning);
logMemoryUsed();
@@ -88,7 +88,7 @@
void fileBufferFree(void* ptr)
{
if (allocatedMBs.contains(ptr)) {
- KST_MEMORY_DEBUG qDebug() << "AsciiFileData: " << allocatedMBs[ptr] / MB << "MB freed";
+ KST_MEMORY_DEBUG if(allocatedMBs[ptr] / MB != 0) qDebug() << "AsciiFileData: " << allocatedMBs[ptr] / MB << "MB freed";
allocatedMBs.remove(ptr);
}
KST_MEMORY_DEBUG logMemoryUsed();
@@ -157,7 +157,7 @@
_begin = -1;
_bytesRead = 0;
- if (bytesToRead <= 0)
+ if (bytesToRead <= 0 || start < 0)
return;
if (maximalBytes == -1) {
@@ -168,7 +168,8 @@
if (!resize(bytesToRead + 1))
return;
}
- file.seek(start); // expensive?
+ if (!file.seek(start)) // expensive?
+ return;
int bytesRead = file.read(_array->data(), bytesToRead);
if (!resize(bytesRead + 1))
return;
--- branches/work/kst/portto4/kst/src/datasources/ascii/asciifiledata.h #1321748:1321749
@@ -28,7 +28,8 @@
enum SizeOnStack
{
Prealloc =
-#if defined(__ANDROID__) || defined(__QNX__) // Some mobile systems really do not like you allocating 1MB on the stack.
+#if defined(__ANDROID__) || defined(__QNX__) || defined(KST_SMALL_PRREALLOC)
+ // Some mobile systems really do not like you allocating 1MB on the stack.
1 * 1024
#else
1 * 1024 * 1024
More information about the Kst
mailing list