What should I do to solve this error in Qvector?

[Yash001]

My Software is crashing only on windows. whenever the large amount of data is available.

I am using Qt 5.12.3 with visual studio 2017.

Here the crashing point I mention.

template <typename T>
void QVector<T>::reallocData(const int asize, const int aalloc, QArrayData::AllocationOptions options)
{
0039CFA8  add         byte ptr [ecx+4D89F065h],cl  
0039CFAE  call        MainWindowUI::CreateNewDataTabWidget+36C6h (0424D36h)  
    Q_ASSERT(asize >= 0 && asize <= aalloc);
0039CFB3  jl          QVector<double>::reallocData+3Fh (039CFBFh)  
0039CFB5  mov         eax,dword ptr [asize]  
0039CFB8  cmp         eax,dword ptr [aalloc]  
0039CFBB  jg          QVector<double>::reallocData+3Fh (039CFBFh)  
0039CFBD  jmp         QVector<double>::reallocData+57h (039CFD7h)  
0039CFBF  push        22Ah  
0039CFC4  push        549E24h  
0039CFC9  push        549E54h  
0039CFCE  call        dword ptr [__imp_qt_assert (0E8A168h)]  
0039CFD4  add         esp,0Ch  
    Data *x = d;
0039CFD7  mov         ecx,dword ptr [this]  
0039CFDA  mov         edx,dword ptr [ecx]  
0039CFDC  mov         dword ptr [x],edx  

    const bool isShared = d->ref.isShared();
0039CFDF  mov         eax,dword ptr [this]  
0039CFE2  mov         ecx,dword ptr [eax]  
0039CFE4  call        QtPrivate::RefCount::isShared (0341B13h)  
0039CFE9  mov         byte ptr [isShared],al  

    if (aalloc != 0) {
0039CFEC  cmp         dword ptr [aalloc],0  
0039CFF0  je          $LN83+110h (039D4B8h)  
        if (aalloc != int(d->alloc) || isShared) {
0039CFF6  mov         ecx,dword ptr [this]  
0039CFF9  mov         edx,dword ptr [ecx]  
0039CFFB  mov         eax,dword ptr [edx+8]  
0039CFFE  and         eax,7FFFFFFFh  
0039D003  cmp         dword ptr [aalloc],eax  
0039D006  jne         QVector<double>::reallocData+94h (039D014h)  
        if (aalloc != int(d->alloc) || isShared) {
0039D008  movzx       ecx,byte ptr [isShared]  
0039D00C  test        ecx,ecx  
0039D00E  je          $LN83+34h (039D3DCh)  
            QT_TRY {
0039D014  mov         dword ptr [ebp-4],0  
                // allocate memory
                x = Data::allocate(aalloc, options);
0039D01B  mov         edx,dword ptr [options]  
0039D01E  push        edx  
0039D01F  mov         eax,dword ptr [aalloc]  
0039D022  push        eax  
0039D023  call        QTypedArrayData<double>::allocate (0347A40h)  
0039D028  add         esp,8  
0039D02B  mov         dword ptr [ebp-44h],eax  
0039D02E  mov         ecx,dword ptr [ebp-44h]  
0039D031  mov         dword ptr [x],ecx  
                Q_CHECK_PTR(x);
0039D034  cmp         dword ptr [x],0  
0039D038  jne         QVector<double>::reallocData+0C0h (039D040h)  
0039D03A  call        dword ptr [__imp_qBadAlloc (0E8AA54h)]  
0039D040  xor         edx,edx                     // at this line software is crash 
0039D042  jne         QVector<double>::reallocData+0B4h (039D034h)  

call stack:

compiler output:

Exception thrown at 0x778A35D2 in _SquidStat.exe: Microsoft C++ exception: std::bad_alloc at memory location 0x012FB8CC.
Exception thrown at 0x778A35D2 in _SquidStat.exe: Microsoft C++ exception: [rethrow] at memory location 0x00000000.
Unhandled exception at 0x778A35D2 in _SquidStat.exe: Microsoft C++ exception: std::bad_alloc at memory location 0x012FB8CC.

similar problem I found Here.
https://www.qtcentre.org/threads/67925-qvector-h-line-538-Out-of-memory-std-bad_alloc

Here some code which is read and write.

"AisDataStore.h"

#pragma once
#include <qvector.h>
#include <qlist.h>
#include <qmap.h>


class  AisDataStore {
public:
	AisDataStore();
	void addDataPoint(qreal value);
	void addDataPoint(QString data);
	qreal getMinValue();
	void setMinValue(qreal value);
	void setMinMaxValue(qreal value);
	
	qreal getMaxValue();
	void setMaxValue(qreal value);
	
	const QVector<qreal>& getAllDataPoints() const;
	const QStringList& getAllStringDataPoints() const;

	bool isDataListEmpty();
	bool isStringDataListEmpty();

	qreal firstDataPoint();
	qreal lastDataPoint();
	
	QString firstStringData();
	QString lastStringData();

	void setSingleDataPoint(qreal value);

	int numberOfDataPoints();
	void removeAllDataPoints();

private:
	QVector<qreal> data;
	QStringList string_data;
	qreal min;
	qreal max;
};

typedef QMap<QString, AisDataStore> AisDataMap;

"AisDataStore.cpp"

#include "AisDataStore.h"

AisDataStore::AisDataStore() : min(0), max(0) {}

void AisDataStore::addDataPoint(qreal value) {
	if (data.isEmpty()) {
		min = value;
		max = value;
	}
	else {
		if (value > max) {
			max = value;
		}
		if (value < min) {
			min = value;
		}
	}
	data.push_back(value);
}

void AisDataStore::addDataPoint(QString data)
{
	string_data.push_back(data);
}

qreal AisDataStore::getMinValue() {
	return min;
}

void AisDataStore::setMinValue(qreal value) {
	min = value;
}
void AisDataStore::setMinMaxValue(qreal value)
{
	min = value;
	max = value;
}
qreal AisDataStore::getMaxValue() {
	return max;
}

void AisDataStore::setMaxValue(qreal value) {
	max = value;
}

const QVector<qreal>& AisDataStore::getAllDataPoints() const {
	return data;
}
const QStringList& AisDataStore::getAllStringDataPoints() const {
	return string_data;
}

bool AisDataStore::isDataListEmpty() {
	return data.isEmpty();
}
bool AisDataStore::isStringDataListEmpty() {
	return string_data.isEmpty();
}

qreal AisDataStore::firstDataPoint() {
	if (!isDataListEmpty()) {
		return data.first();
	}
	return 0;
}

qreal AisDataStore::lastDataPoint() {
	if (!isDataListEmpty()) {
		return data.last();
	}
	return 0;
}


QString AisDataStore::firstStringData() {
	if (!string_data.isEmpty()) {
		return string_data.first();
	}
	return "";
}

QString AisDataStore::lastStringData() {
	if (!string_data.isEmpty()) {
		return string_data.last();
	}
	return "";
}

void AisDataStore::setSingleDataPoint(qreal value) {
	data.insert(0, value);
	if (data.count() > 1) {
		for (int i = 1; i < data.count(); i++) {
			data.removeAt(i);
		}
	}
}
/*always only one list will fill with data */
int AisDataStore::numberOfDataPoints() {
	int ret = 0;
	if (!isDataListEmpty()) {
		ret = data.count();
	}
	else {
		ret = string_data.count();
	}
	return ret;
}
void AisDataStore::removeAllDataPoints() {
	data.clear();
	string_data.clear();
	return;
}

data (Qvector) is fill while data point is arrived.

any suggestion to solve this error?

[JonB]

@Yash001

Yes, I am building application in x86

If you are building for x86 you are building a 32-bit application. Even though your Windows is 64-bit you will be limited to 32-bit address space. You may have 8GB of physical RAM, but your program will only access at most about 3.4GB of it. [ @Christian-Ehrlicher has just told me it will actually be only 2GB!]

If you want to access more RAM than that, you need to build for 64-bit, x64. Or, you could look at what @VRonin offered earlier on, but that may mean work from you.

[Christian Ehrlicher]

How much elements does the vector contain?

[Yash001]

Large amount of data points (double) are store in QVectore. In 42 min I am getting 8 million data point for single container.

[aha_1980]

@Yash001 said in What should I do to solve this error in Qvector?:

std::bad_alloc

That means you run out of memory. Install more RAM in your computer.

Regards

[J.Hilk]

Or download some more.
One can never have enough ram 😎

https://downloadmoreram.com

[SGaist]

Hi,

How many of these datastore do you have ?
Also, are you building a 32bit application ?

[VRonin]

Are you sure you need 8m data points all in ram at once?

If the answer is yes then, have another think about it. If it's still yes for inspiration on how to solve this you can have a look at this header only library there I implemented HugeContainers::HugeHash and HugeContainers::HugeMap that are versions of QHash and QMap respectively that are not restricted by memory size. Shouldn't be hard to implement a QVector replacement using the same philosophy.

[Yash001]

@aha_1980
I do have 8 gb ram. Is it not enough?
Same software is run in Macbook air (OS macOs 10.14) for more than 40 hours.

[SGaist]

You still haven't answered the question about whether you are building in 32 or 64 bit on Windows.

[Yash001]

@SGaist said in What should I do to solve this error in Qvector?:

How many of these datastore do you have ?

Single experiment contain 9 Qvector.

@SGaist said in What should I do to solve this error in Qvector?:

Also, are you building a 32bit application ?

Yes, I am building application in x86

[Yash001]

@SGaist said in What should I do to solve this error in Qvector?:

You still haven't answered the question about whether you are building in 32 or 64 bit on Windows.

64 bit windows

[JonB]

@Yash001

Yes, I am building application in x86

If you are building for x86 you are building a 32-bit application. Even though your Windows is 64-bit you will be limited to 32-bit address space. You may have 8GB of physical RAM, but your program will only access at most about 3.4GB of it. [ @Christian-Ehrlicher has just told me it will actually be only 2GB!]

If you want to access more RAM than that, you need to build for 64-bit, x64. Or, you could look at what @VRonin offered earlier on, but that may mean work from you.

[Yash001]

@VRonin

@VRonin said in What should I do to solve this error in Qvector?:

Are you sure you need 8m data points all in ram at once?

Yes. I am plotting the graph in real time.

Thank you for providing nice implementation for large dataset. I do have few questions regarding your code.

using ItemMapType = typename std::conditional<sorted, QMap<KeyType, ContainerObject<ValueType> >, QHash<KeyType, ContainerObject<ValueType> > >::type;
            std::unique_ptr<ItemMapType> m_itemsMap;

I think HugeContainerData::std::unique_ptr<ItemMapType> m_itemsMap is storing all the key and position of value in "temp/"HugeContainerDataXXXXXX" file.

std::unique_ptr<QMap<qint64, bool> > m_memoryMap; is provide the information for value like value is store that particular location or not in "temp/"HugeContainerDataXXXXXX" file.

Could you correct me if i understood wrong in above two use?

if I understood correctly then I am missing something.
How do you save the space in RAM because of size of HugeContainerData::std::unique_ptr<ItemMapType> m_itemsMap is increase as per number of elements are increase.

Another few Question for creating Hugevector.

1. Is it good idea to create the HugeContainerData::Hugevector with help of HugeContainerData::HugeMap.?

like HugeContainerData::HugeMap <int, double>
Here "key" = index of Vector.
Value = vectore value.

or

2. Are you recommend to create separate implementation for Hugevector?

something like below:
modify the HugeContainerData::std::unique_ptr<ItemMapType>

using ItemMapType = typename std::conditional<sorted, QMap<KeyType, ContainerObject<ValueType> >, QHash<KeyType, ContainerObject<ValueType> > >::type;

to

using ItemMapType = typename std::conditional<sorted, QVectore<ContainerObject<ValueType> >, QList<ContainerObject<ValueType> > >::type;

ItemMapType will store only position of elements in temp/"HugeContainerDataXXXXXX" file.

[Yash001]

Here I implemented My HugeVector with help of https://github.com/VSRonin/QtHugeContainer

#pragma once
#ifndef hugecontainer_h__
#define hugecontainer_h__

#include <QDataStream>
#include <QDir>
#include <QDirIterator>
#include <qvector.h>
#include <QSharedData>
#include <QSharedDataPointer>
#include <QExplicitlySharedDataPointer>
#include <QTemporaryFile>
#include <QDebug>
#include <memory>
#include <initializer_list>


namespace HugeContainers {
	template <class ValueType>
	class HugeContainer;
}

template <class ValueType>
QDataStream& operator<<(QDataStream &out, const HugeContainers::HugeContainer<ValueType>& cont);

template <class ValueType>
QDataStream& operator>>(QDataStream &in, HugeContainers::HugeContainer<ValueType>& cont);


namespace HugeContainers {
	//! Removes any leftover data from previous crashes
	inline void cleanUp() {
		QDirIterator cleanIter{ QDir::tempPath(), QStringList(QStringLiteral("HugeContainerData*")), QDir::Files | QDir::Writable | QDir::CaseSensitive | QDir::NoDotAndDotDot };
		while (cleanIter.hasNext()) {
			cleanIter.next();
			QFile::remove(cleanIter.filePath());
		}

	}

	template <class ValueType>
	class HugeContainer
	{
		static_assert(std::is_default_constructible<ValueType>::value, "ValueType must provide a default constructor");
		static_assert(std::is_copy_constructible<ValueType>::value, "ValueType must provide a copy constructor");
	private:
		template <class ValueType>
		struct ContainerObjectData : public QSharedData
		{
			bool m_isAvailable;
			union ObjectData
			{
				explicit ObjectData(qint64 fp)
					:m_fPos(fp)
				{}
				explicit ObjectData(ValueType* v)
					:m_val(v)
				{}
				qint64 m_fPos;
				ValueType* m_val;
			} m_data;
			explicit ContainerObjectData(qint64 fp)
				:QSharedData()
				, m_isAvailable(false)
				, m_data(fp)
			{}
			explicit ContainerObjectData(ValueType* v)
				:QSharedData()
				, m_isAvailable(true)
				, m_data(v)
			{
				Q_ASSERT(v);
			}
			~ContainerObjectData()
			{
				if (m_isAvailable)
					delete m_data.m_val;
			}
			ContainerObjectData(const ContainerObjectData& other)
				:QSharedData(other)
				, m_isAvailable(other.m_isAvailable)
				, m_data(other.m_data.m_fPos)
			{
				if (m_isAvailable)
					m_data.m_val = new ValueType(*(other.m_data.m_val));
			}
		};

		template <class ValueType>
		class ContainerObject
		{
			QExplicitlySharedDataPointer<ContainerObjectData<ValueType> > m_d;
		public:
			explicit ContainerObject(qint64 fPos)
				:m_d(new ContainerObjectData<ValueType>(fPos))
			{}
			explicit ContainerObject(ValueType* val)
				:m_d(new ContainerObjectData<ValueType>(val))
			{}
			ContainerObject(const ContainerObject& other) = default;
			bool isAvailable() const { return m_d->m_isAvailable; }
			qint64 fPos() const { return m_d->m_data.m_fPos; }
			const ValueType* val() const { Q_ASSERT(m_d->m_isAvailable); return m_d->m_data.m_val; }
			ValueType* val() { Q_ASSERT(m_d->m_isAvailable); m_d.detach(); return m_d->m_data.m_val; }
			void setFPos(qint64 fp)
			{
				if (!m_d->m_isAvailable && m_d->m_data.m_fPos == fp)
					return;
				m_d.detach();
				if (m_d->m_isAvailable)
					delete m_d->m_data.m_val;
				m_d->m_data.m_fPos = fp;
				m_d->m_isAvailable = false;
			}
			void setVal(ValueType* vl)
			{
				m_d.detach();
				if (m_d->m_isAvailable)
					delete m_d->m_data.m_val;
				m_d->m_data.m_val = vl;
				m_d->m_isAvailable = true;
			}
		};


		template <class ValueType>
		class HugeContainerData : public QSharedData
		{
		public:
			using ItemMapType = QVector<ContainerObject<ValueType>>;
			std::unique_ptr<ItemMapType> m_itemsMap;
			std::unique_ptr<QMap<qint64, bool> > m_memoryMap;
			std::unique_ptr<QTemporaryFile> m_device;

			HugeContainerData()
				: QSharedData()
				, m_device(std::make_unique<QTemporaryFile>(QDir::tempPath() + QDir::separator() + QStringLiteral("HugeContainerDataXXXXXX")))
				, m_memoryMap(std::make_unique<QMap<qint64, bool> >())
				, m_itemsMap(std::make_unique<ItemMapType>())
			{
				if (!m_device->open())
					Q_ASSERT_X(false, "HugeContainer::HugeContainer", "Unable to create a temporary file");
				m_memoryMap->insert(0, true);
			}
			~HugeContainerData() = default;
			
			HugeContainerData(HugeContainerData& other)
				: QSharedData(other)
				, m_device(std::make_unique<QTemporaryFile>(QDir::tempPath() + QDir::separator() + QStringLiteral("HugeContainerDataXXXXXX")))
				, m_memoryMap(std::make_unique<QMap<qint64, bool> >(*(other.m_memoryMap)))
				, m_itemsMap(std::make_unique<ItemMapType>(*(other.m_itemsMap)))
			{
				if (!m_device->open())
					Q_ASSERT_X(false, "HugeContainer::HugeContainer", "Unable to create a temporary file");
				other.m_device->seek(0);
				qint64 totalSize = other.m_device->size();
				for (; totalSize > 1024; totalSize -= 1024)
					m_device->write(other.m_device->read(1024));
				m_device->write(other.m_device->read(totalSize));
			}

		};

		QExplicitlySharedDataPointer<HugeContainerData<ValueType>> m_d;


		qint64 writeInMap(const QByteArray& block) const
		{
			if (!m_d->m_device->isWritable())
				return -1;

			auto i = m_d->m_memoryMap->end()-1; // last value iterator
			if (i.value()) {
				m_d->m_memoryMap->insert(i.key() + block.size(), true);		
				i.value() = false;
				m_d->m_device->seek(i.key());
				if (m_d->m_device->write(block) >= 0)
					return i.key();
				return -1;
			}
			Q_UNREACHABLE();
			return 0;
		}

		void removeFromMap(qint64 pos) const {
			auto fileIter = m_d->m_memoryMap->find(pos);
			Q_ASSERT(fileIter != m_d->m_memoryMap->end());
			if (fileIter.value())
				return;
			fileIter.value() = true;
			m_d->m_memoryMap->erase(fileIter);
		}


	
		qint64 writeElementInMap(const ValueType& val) const
		{
			QByteArray block;
			{
				QDataStream writerStream(&block, QIODevice::WriteOnly);
				writerStream << val;
			}

			const qint64 result = writeInMap(block);
			return result;
		}


		bool saveQueue(const uint& index) const {
			bool allOk = false;
			auto valToWrite = m_d->m_itemsMap->begin() + index; 
			
			const qint64 result = writeElementInMap(*(valToWrite->val()));
			if (result >= 0) {
				valToWrite->setFPos(result);
				allOk = true;
			}
			return allOk;
		}

		bool enqueueValue(std::unique_ptr<ValueType>& val) const
		{
			m_d->m_itemsMap->push_back(ContainerObject<ValueType>(val.release()));
			if (saveQueue(size()-1)) {
				return true;
			}
		    return false;
		}

		std::unique_ptr<ValueType> valueFromBlock(const uint& index) const
		{
			QByteArray block = readBlock(index);
			if (block.isEmpty())
				return nullptr;
			auto result = std::make_unique<ValueType>();
			QDataStream readerStream(block);
			readerStream >> *result;
			return result;
		}

		QByteArray readBlock(const uint& index ) const
		{
			if (Q_UNLIKELY(!m_d->m_device->isReadable()))
				return QByteArray();
			m_d->m_device->setTextModeEnabled(false);
			
			auto itemIter = m_d->m_itemsMap->begin() + index;       //  get iterator at particular position
			Q_ASSERT(itemIter != m_d->m_itemsMap->end());
			Q_ASSERT(!itemIter->isAvailable());
			
			auto fileIter = m_d->m_memoryMap->constFind(itemIter->fPos());
			Q_ASSERT(fileIter != m_d->m_memoryMap->constEnd());
			if (fileIter.value())
				return QByteArray();
			
			auto nextIter = fileIter + 1;
			m_d->m_device->seek(fileIter.key());
			
			QByteArray result;
			if (nextIter == m_d->m_memoryMap->constEnd())
				result = m_d->m_device->readAll();
			else
				result = m_d->m_device->read(nextIter.key() - fileIter.key());
			
			return result;
		}


	public:

		HugeContainer()
			:m_d(new HugeContainerData<ValueType>{})
		{
		}

		HugeContainer(const HugeContainer& other) = default;
		HugeContainer& operator=(const HugeContainer& other) = default;
		HugeContainer& operator=(HugeContainer&& other) Q_DECL_NOTHROW {
			swap(other);
			return *this;
		}


		void swap(HugeContainer<ValueType>& other) Q_DECL_NOTHROW {
			std::swap(m_d, other.m_d);
		}

		
		void push_back(const ValueType &val) {
			m_d.detach();
			auto tempval = std::make_unique<ValueType>(val);
			enqueueValue(tempval);
		}
		
		void push_back(ValueType* val)
		{
			if (!val)
				return;
			m_d.detach();
			std::unique_ptr<ValueType> tempval(val);
			enqueueValue(tempval);
			
		}

		/*if index is not correct then append to the vector*/
		void insert(uint index, const ValueType &val) {

			if (!correctIndex(index))
				index = size();
		
			m_d.detach();
			auto tempval = std::make_unique<ValueType>(val);
			m_d->m_itemsMap->insert(index, ContainerObject<ValueType>(tempval.release()));
			saveQueue(index);
		}


		void insert(const uint& index, ValueType* val)
		{
			if (!val)
				return;

			if (!correctIndex(index))
				return;

			m_d.detach();
			std::unique_ptr<ValueType> tempval(val);

			m_d->m_itemsMap->insert(index, ContainerObject<ValueType>(tempval.release()));
			saveQueue(index);
		}


		/* Must be put correct index for finding value */
		const ValueType& at(const uint& index)
		{
			Q_ASSERT(!isEmpty());

			auto valueIter = m_d->m_itemsMap->begin() + index;
			Q_ASSERT(valueIter != m_d->m_itemsMap->end());
			
			auto result = valueFromBlock(index);
			Q_ASSERT(result);
			
			m_d.detach();
			return *(result.release());
		}

		bool removeAt(const uint& index)
		{
			if (!correctIndex(index))
				return false;
			m_d.detach();
			auto itemIter = m_d->m_itemsMap->begin() + index;
			Q_ASSERT(itemIter != m_d->m_itemsMap->end());
			removeFromMap(itemIter->fPos());
			m_d->m_itemsMap->erase(itemIter);
			return true;
		}

		void clear()
		{
			if (isEmpty())
				return;
			m_d.detach();
			if (!m_d->m_device->resize(0)) {
				Q_ASSERT_X(false, "HugeContainer::HugeContainer", "Unable to resize temporary file");
			}
			m_d->m_itemsMap->clear();
			m_d->m_memoryMap->clear();
			m_d->m_memoryMap->insert(0, true);
		}

		int count() const
		{
			return size();
		}
		int size() const
		{
			return m_d->m_itemsMap->size();
		}
		bool isEmpty() const
		{
			return m_d->m_itemsMap->isEmpty();
		}

		bool correctIndex(const uint& index) {
			return ((index >= 0) && (m_d->m_itemsMap->size() > index));
		}

		int memMapsize() const
		{
			return m_d->m_memoryMap->size();
		}

		inline const ValueType& first() const
		{
			Q_ASSERT(!isEmpty());
			return at(0);
		}

		inline ValueType& first()
		{
			Q_ASSERT(!isEmpty());
			return const_cast<ValueType&>(at(0));
		}

		inline const ValueType& last() const
		{
			Q_ASSERT(!isEmpty());
			return at(size() - 1);
		}

		inline ValueType& last()
		{
			Q_ASSERT(!isEmpty());
			return const_cast<ValueType&>(at(size() - 1));
		}
	    
	};

}
#endif // hugecontainer_h__

I test this code with help of

int main(int argc, char *argv[])
{
	QCoreApplication a(argc, argv);

	/*  create the funcation for checking the size of memory map*/
	HugeContainers::HugeContainer<qreal> obj;
	std::cout << "start executation";
	//QVector<qreal> obj;
	for (int i = 0; i < 9000000; i++) {
		try {
			obj.push_back(10.23);
			obj.push_back(10.23);
			obj.push_back(10.23);
			obj.push_back(10.23);
			obj.push_back(10.23);
			obj.push_back(10.23);
			obj.push_back(10.23);
			obj.push_back(10.23);
			obj.push_back(10.23);
		}
		catch (...) {
			std::cout << "value of i : " << i;
		}
	}
	std::cout << "stop executation";

	
	return a.exec();
}

I got same error bad_allocation.

I knew it is slow because of I did not add m_maxCache as @VRonin used in his code. I will add later if it is work fine.

anything do I need to modify?

[Christian Ehrlicher]

@Yash001 said in What should I do to solve this error in Qvector?:

9000000

You try to allocate 9000000 * 9 * 8Bytes = 648000000Bytes = ~617MB continuous memory which fails because your OS can't allocate it. Don't see what Qt can do against this. Esp.when your executable is a 32 Bit executable which seems to be the case.

[JonB]

@Yash001
So now the (probably better) question is why do you need 9 million elements in a QVector?

[Yash001]

@JonB said in What should I do to solve this error in Qvector?:

So now the (probably better) question is why do you need 9 million elements in a QVector?

Thank you for your time and help @JonB @Christian-Ehrlicher @VRonin @aha_1980 @J-Hilk

Just for testing purpose I took 9 million elements in Qvector.

My application could be run more than 2 hours on the user end. I don't know exact time. currently I can reach at near about 8 million * 9 (QVector) data points in 42 min.

one Qvector is pointing to X axis data on plot.
another Qvector is pointing to Y axis data on plot.

User can select the another vector for X and Y axis.

Just want to give more detail:

1. currently I am getting data point from device.
2. store into container.
3. read the last value from container.
4. put data point into csv file and plot.
5. User can select different container anytime.

any other better approach should I try?

something like change the design or something else.

is It worth to create temp file for memory map?

[JonB]

@Yash001
Well you now know that with your 32-bit compiled application you're not going to have enough contiguous memory for 9 million or whatever data points in memory. So what are you going to do about it? :) 9M is a lot of plot points!? Maybe you can discard old points, maybe you can thin out with sampling. Maybe you should make your app 64-bit and also buy more memory, but if you don't hit the limit at 9M maybe you'll find your data is even bigger and then what?

[Yash001]

@JonB said in What should I do to solve this error in Qvector?:

Maybe you should make your app 64-bit and also buy more memory,

I trying to build application in 64 bits and test it same things occur.

Is it worth to create the temp file for memory map and data?

1. memory map has position of value, which store in data file.
2. whenever I need to read the value , at that time with help memory-map file I do have address of data, and after that read the value from temp file.

to make more speed operation I should use large Queue<qreal>.
in starting phase few data point will store in queue if queue is full then write the full queue in files.

I did not have clear though about reading operation for making speed up from index.

[Christian Ehrlicher]

Nobody will take a look at 9 Million data points... thin them out with sampling.