QDataStream fail to send large strings

[Dariusz]

Hey

I'm on pyside2 5.15 qt. I'm trying to socket-QDataStream send string that is 500mb big...

When I try to send it it just sends 4 bytes.

Is the string size limited to socket buffer? Do I need to enable some kind of chunking?

Regards
Dariusz

[Christian Ehrlicher]

Why should a string be 500MB large?
How do you send it/receive it?

[Dariusz]

Sender

port = 500

socket = QTcpSocket()

# Connect to the server
socket.connectToHost("localhost", port)
msg = str(uuid.uuid4())
for a in range(0, 28):
    msg += msg
print(len(msg) / 1024 / 1024)

# Wait for the connection to be established
if socket.waitForConnected(5000):
    print("sending...")
    # Create a QByteArray to hold the data
    data = QByteArray("Hello World!".encode())
    data = json.dumps({"hello": "world", "this": [{"this": "is"}, {"awe": "some"}],
                       "stupidLongStuff": msg,
                       "END": 1})
    # Create a QDataStream to write the data
    stream = QDataStream(socket)
    dataSize = len(data)
    print("sending data size : ", dataSize)
    # Write the data to the stream
    stream.writeInt32(dataSize)
    stream.writeString(str(data))
    # Wait for the data to be sent
    socket.waitForBytesWritten()
    print("Data sent successfully")
else:
    print("Error:", socket.errorString())

Receiver is just

    def _handleReadyRead(self):
        client_socket: QTcpSocket = self.sender()
        stream = QDataStream(client_socket)
        jobSize = client_socket.bytesAvailable()
        print("Thread dat : ", self.thread(), jobSize)
        size =  stream.readInt32()
        data = stream.readString()

The int works fine, but string after is fail. No matter waht I try, dataStream wont push it.

[Christian Ehrlicher]

readRead() is called when data is available at the socket, not when all data is received (what a socket can't know) so you have to reassemble your data by yourself.

[Dariusz]

@Christian-Ehrlicher Should DataStream not deal with it? Handle chunking/etc/etc?

Regards
Dariusz

[Christian Ehrlicher]

@Dariusz said in QDataStream fail to send large strings:

Should DataStream not deal with it? Handle chunking/etc/etc?

Why? "The QDataStream class provides serialization of binary data to a QIODevice" - nothing more and nothing less.

You can look at the transaction support for QDataStream though.

[Dariusz]

@Christian-Ehrlicher Hmmm I think because its class is responsible for serializing/deserializing/managing data so I thought it would handle the chunking/etc with socket. Else it feels like half-measure and I still have to solve the chunking/disconnection/etc.

[SGaist]

Hi,

QDataStream serializes something to a QIODevice. What happens after that is not its concern. For example, why would it need to know how to write to a socket more than knowing how to write to a file ? That's what the QIODevice are responsible for. If you need higher level structure management, make your own QIODevice.

[Christian Ehrlicher]

Please read about the transactions.
Also writing this as a string is nonsense - it just doubles the size of transferred bytes for no reason. Just sent a bytearray directly. Also there is no need to send the dataSize since QDataStream adds this information to the stream by itself.

[Dariusz]

@Christian-Ehrlicher Hmmm I'm not having any luck with this method so far :/
Code below is self contained test. The DataStream is not commitTransaction and thus still waits for more data but never reads it/deals with it.
Socket dies, no more data comes in, handle_ready_read never gets called again, even if I force call again it wont read any more data... :/ Where am I hitting wall here?

import json
import uuid
from functools import partial

from PySide2.QtNetwork import *
from PySide2.QtCore import *
from PySide2.QtWidgets import *


class BasicServer(QTcpServer):
    s_newJob = Signal(object)

    def __init__(self, port: int):
        QTcpServer.__init__(self)
        self._mStatus = self.listen(QHostAddress.LocalHost, port)
        self._mSocketList = {}
        self._mJobs = {}
        self.newConnection.connect(self._handleNewConnection)
        print("Starting logic server on port : ", port, " Status : ", self._mStatus)
        self._mTimer = QElapsedTimer()
        self._mTimer.start()

    def getStatus(self) -> int:
        return self._mStatus

    def _handleNewConnection(self):
        client_socket: QTcpSocket = self.nextPendingConnection()
        client_socket.readyRead.connect(partial(self.handle_ready_read, client_socket))
        stream = QDataStream(client_socket)
        self._mSocketList[client_socket] = [stream, QByteArray()]
        client_socket.disconnected.connect(partial(self._handleSocketDisconnect, client_socket))

    def handle_ready_read(self, client_socket):
        streamItem = self._mSocketList.get(client_socket, None)
        if streamItem is None:
            return
        print("reading?")
        stream = streamItem[0]
        stream.startTransaction()
        streamData = streamItem[1]
        stream >> streamData
        print(len(streamData))
        self._mSocketList[client_socket][1] = streamData
        if not stream.commitTransaction():
            return
        print("IM FINISH! ", streamData)

    def _handleSocketDisconnect(self, socketPtr: QTcpSocket):
        print("Socket is gone...")
        return  # for now keep socket "alive"
        killPort = socketPtr.peerPort()
        if killPort in self._mJobs:
            if not self._mJobs[killPort]["emitted"]:
                if self._mJobs[killPort]["toDownload"] == 0:
                    pass
            del self._mJobs[killPort]

        if socketPtr in self._mSocketList:
            del self._mSocketList[socketPtr]
        print("Killing socket", killPort, socketPtr)
        socketPtr.deleteLater()


def clientTest():
    port = 500

    socket = QTcpSocket()

    # Connect to the server
    socket.connectToHost("localhost", port)
    msg = str(uuid.uuid4())
    for a in range(0, 10):
        msg += msg
    print(len(msg) / 1024 / 1024)

    # Wait for the connection to be established
    if socket.waitForConnected(5000):
        print("sending...")
        # Create a QByteArray to hold the data
        data = QByteArray("Hello World!".encode())
        jDumped = json.dumps({"hello": "world", "this": [{"this": "is"}, {"awe": "some"}],
                              "stupidLongStuff": msg,
                              "END": 1})
        data = QByteArray(jDumped.encode())

        # Create a QDataStream to write the data
        stream = QDataStream(socket)
        # stream.setVersion(QDataStream.Qt_5_15)
        print("Data size : ", len(data))
        stream << data
        # stream.writeString("hellooo")
        # dataSize = len(data)

        # print("sending data size : ", dataSize)
        # Write the data to the stream
        # stream.writeInt64(dataSize)
        # socket.waitForBytesWritten()
        # socket.write(struct.pack("l", len(data)))
        # socket.write(data.encode())
        socket.waitForBytesWritten()
        print("Data sent successfully")
    else:
        print("Error:", socket.errorString())


app = QApplication([])
s = BasicServer(500)

clientTest()

app.exec_()