SERVER/CLIENT cannot send and recieve data on the same socket, server close server after first attempt

[Lightshadown]

Hi im trying to create a backup function for my program, the idea is to copy and send all the files from one especific folder from my client to the server, the issue here is everytime i send the file name and size of every file as a list the server close the conecction after its recieve but i need the server to ask the client for the data of the files itself, the main is like follows

make a list of all files on folder -------> send the list of Files and size via sockets ------> recieve and ask back the file i want to copy one by one ----->send the data of the actual file -----> repeat until the list is over

it supose i handle the data send using the ready read signal but for some reason i cant send the actual data, the conecction is closed

i add my actual code, the client is in C++ and the server is in python, any help is welcome

Server.py

def servidorMk3():
    host = socket.gethostname()
    host_port = 8000
    host_ip = socket.gethostbyname(host)
    buffer = 1024
    cnt = 0
    code_end = "<END>"
    code_ok = "<OK>"
    code_fin = "<FIN>"
    code_bad = "<BAD>"
    nombre = []
    tamano = []
    
    print (f"Servidor Mk3")
    print (f"Ip del servidor: " + host_ip)
    log("Inicio del servidorMk3 " + host_ip)
    
    # while True:
    #     send_UDP_broadcast(host_ip)
    #     ip = lisent_UDP_broadcast()
    #     if ip:
    #         break
    #     else:
    #         continue
    
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as tcp_conn:
        tcp_conn.bind((host_ip, host_port))
        tcp_conn.listen(2)
        con, adrs = tcp_conn.accept()
        with con:
            cnt += 1
            print (f"Conecion Exitosa")
            log("Connecion exitosa con el server: " + adrs[0])
            if cnt > 20:
                log("Fin del ciclo")
            else:
                while True:
                    # start to recieve the list of files
                    while True:         # start to fill the socket
                        data = con.recv(buffer)
                        if not data:     # if theres no data close the loop
                            break
                        if code_end.encode("utf8") in data:  # if found <END>, ends the current element
                            break
                    data = data.decode("utf8")
                    elemt = []
                    elemt = data.split("<SEPARATOR>")
                    if code_fin in data:  # if found <FIN>, the list is over
                         break     
                    
                    ##  Erease the empty elements
                    ##  **************************************************
                    if code_end in elemt: elemt.remove(code_end)
                    #print("antes de borrar elementos")
                    #for a in range(len(elemt)):
                    #    print(elemt[a])
                    elemt = list(filter(None, elemt))
                    print ("despues de borrar")
                    for a in range(len(elemt)):
                        print(elemt[a])
                    if not elemt:
                        nombre, tamano = elemt[:2]
                    else:
                    #    print("Elemento vacio")
                        break
                    ## **************************************************
                    
                    #  now i ask what file i want to recieve
                    #request_files(nombres, adrs, host_port)
                # creates 2 independent list then iterate and asign names and size respectively
                
                for a in range (len(nombre)):
                   # here i ask the names and start asking the client for it
                    con.send(nombre[a].encode("utf8"))
                    print (nombre[a])
                    while True:    # recive the data file
                        file_data = con.recv(buffer)      
                        if not file_data:
                            break
                    #     # revisar como comparar el tamaño que ya tengo vs el tamaño que recibo
                    #     #https://stackoverflow.com/questions/449560/how-do-i-determine-the-size-of-an-object-in-python
                    #     if int(nombres[1]) == sys.getsizeof(file_data):
                    #         break
                    
                    # # Here i save the files to disk
                    directorio = crear_directorio()
                    if directorio == "<BAD>":
                        print(" no se puede crear la ubicacion")
                        print(type(directorio))  # revisar el tipo que regresa esta funcion¡¡¡¡
                        print(directorio)
                    else:
                        file_name = directorio + nombre[a]  # path to the file,   
                        file_size = int(tamano[a])   # size of file
                        file_create = open(file_name, 'wb')  # open file for writing in binary 
                        while (file_data):   # creacion y escritura del archivo
                            file_create.write(file_data)   # writes to the file
                        file_create.close()     #close the file
                        check = os.path.getsize(file_name)  # returns the size of the file
                    
                        if check == file_size:
                            #con.send(code_ok.encode())
                            log("archivo " + nombre[a] + " creado, tamaño: " + tamano[a])
                        else:
                            #con.sendall(code_bad.encode())
                            log("No se pudo crear el archivo: " + nombre[a])
            
            log("Respaldo de Archivos completado")
            print(f"Respaldo de archivos terminada")

client.c++

Server::Server() {    // server constructor
     Gen gen;  // Logs 

     QObject::connect(TCP_socket, &QTcpSocket::readyRead,  [&]() {  //  the idea is to handle here the data of the file
            QString dataread;
            gen.Log
            QByteArray data, block_file;
            QDataStream salida(TCP_Socket);
            QStringList File_Name;
            QDir directorio("Data/");
            bool yes;

            File_Name = directorio.entryList (QStringList() << "*.db" << "*.DB" << "*.Db" << "*.dB" , QDir::Files  );
            data = TCP_socket->readAll();
            dataread.append(data);

            foreach (QString lista, File_Name) {
                     block_file.clear();
                     if (lista == dataread) {
                         QFile Archivo("Data/" + lista);
                         Archivo.open(QIODevice::ReadOnly);
                         block_file = Archivo.readAll();
                         gen.Log(Lista);
                      }
                    if (TCP_socket->isValid() ) {
                          gen.Log("antes de escribir");
                          salida.writeRawData ( (char*)(block_file.data()), block_file.size() );                    
                          yes = TCP_socket->flush();
                          TCP_socket->waitForBytesWritten();
                          yes == true ? gen.Log("Datos Enviados: " + lista) : gen.Log("No se envio la data del archivo: " + lista);
                      }
               }
      });


void Server::TCP_Request_Send() {     // actual backup function
      Gen L; 
      QString Ip ("192.168.1.123");     // need to add a broadcast function to retrieve the ip
       qint64 filesize;
       bool yes = false;
       QByteArray block, block_file; 
      QStringList File_name;
       QDir directorio("Data/");

       File_Name = directorio.entryList (QStringList() << "*.db" << "*.DB" << "*.Db"<< "*.db", QDir::Files );
       File_Name.append ("<FIN>");
       L.Log ("..........Lista de Archivos");

       if (TCP_socket->state() == QAbstractSocket::UnconnectedState ) {
               L.Log("Before connection");
               TCP_socket->connectToHost( Ip, 8000, QIODevice::WriteOnly);
               if (TCP_socket->waitForConnected(5000) ) {
                       QDataStream salida(TCP_socket);
                       foreach ( QString files, File_Name) {
                             QFile Archivo ("Data/" + files);
                             std::string nombre = files.toStdString();
                             L.Log (QString::fromStdString(nombre) );
                             Archivo.open (QIODevice::ReadOnly);
                             block_file = Archivo.readAll();
                             filesize = Archivo.size();
                             block.clear();
                         
                             if (files == "<FIN>") {
                                    block.append ("<FIN><SEPARATOR><END>");
                             } else {
                                    block.append (QString::fromStdString(nombre) + "<SEPARATOR>"  + QString::number(filesize) + "<SEPARATOR>" + "<END>" );
                                    if (block.isEmpty() == true) {
                                             continue;    // if empty the for continues
                                        }
                             }

                           L.Log ("despues de block");
                           if (TCP_socket->isValid() ) {
                                 L.Log ("antes de escribir");
                                 salida.writeRawData ( (char*)(block.data()), block.size() );
                                 yes = TCP_socket->waitForBytesWritten();
                                L.Log (QString::fromUtf8(block.constData(), block.size()  ) );
                                yes == true ? L.Log ("nombre " + files + " enviado") : L.Log ("no se pudo enviar el nombre" + files );
                          }
               Archivo.close();
          };
       }
     }
  }

[VRonin]

@Lightshadown said in SERVER/CLIENT cannot send and recieve data on the same socket, server close server after first attempt:

the server close the conecction after its recieve

I think this is unrelated to Qt and just a Python problem. when you reach log("Respaldo de Archivos completado"), con might get deleted by the garbage collector

[Lightshadown]

@VRonin yes it was a loop related issue, i just erased one while true and i fix it, now i ran into the issue that data is parsed incorrectlly, the first element is never read on the server side, but its send on the client, i do a print i should had this for the firts elemnt

0 ..  Ciclo_OI.db
1 ..  25698

But i got the second instead, and the rest of the list in this way, and im unable to figure it out whats wrong meabe the socket is not recieving the full data, any idea

 Conecion Exitosa
 despues de borrar
 0 .. <END>Ciclo_OI_2020_2021.db
 1 .. 32768
 Tamaño esperado: 32768 ... tamaño real 324 

 despues de borrar
  0 .. <END>Ciclo_PV_2020_2021.db
  1 .. 32768
  Tamaño esperado: 32768 ... tamaño real 66 

this code kinda works but still missing data, im sending Filename <SEP> Size <SEP> SqliteData <END> and it supose im spliting the data using <SEP> and erasing <END>, its just a way to separate de data from each file, working code atm

with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as tcp_conn:
    tcp_conn.bind((host_ip, host_port))
    tcp_conn.listen(1)
    con, adrs = tcp_conn.accept()
    print (f"Conecion Exitosa")
    log("Connecion exitosa con el server: " + adrs[0])
    
    while True:
        #empieza a recibir la lista de los archivos
        while True:         # recibe la info en general
            data = con.recv(buffer)
            if not data:     # si no hay informacion se cierra el ciclo
                break
            if code_end.encode("utf8") in data:  # si llega <END>, termino el elemento actual
                break
        
        if code_fin.encode("utf8") in data:  # si llega <FIN>, se acabo la lista
            break     
        elemt = []
        elemt = data.split(b'<SEP>')

        
        ##  Elimina los elementos vacios
        ##  **************************************************
        
        # for a in range(len(elemt)):
        #     if code_end.encode("utf8") in elemt[a]:
        #         elemt[a] = elemt[a].strip(b'<END>')
                # elemt[a] = elemt[a].remove(code_end.encode("utf8"))
        #print("antes de borrar elementos")
        #elemt = list(filter(None, elemt))   # erase empty spaces
        if len(elemt) < 3:
            nombre = elemt[0].decode("utf8")
            print(nombre)
            size = elemt[1].decode("utf8")
            dataFile =  elemt[2]
            print (f" menor a {len(elemt)}")
        if len(elemt) > 3:
            nombre = elemt[1].decode("utf8")
            size = elemt[2].decode("utf8")
            dataFile =  elemt[3]
            print (f" mayor a {len(elemt)}")
        #for a in range(len(elemt)):  # remove <END>
        #    if code_end.encode("utf8") in elemt[a]:
        nombre.replace("<END>","")     # get rid of end
        nombre = nombre.strip()   # erase empty spaces
        size = size.strip()
        #size = list(filter(None, size))
        
        # there should be 3 elemnts now name, size, datafile
        print ("despues de borrar")
        #for a in range(len(elemt)):
        #    print(elemt[a].decode("utf8"))
        # if not elemt:
        #     nombre, tamano = elemt[:2]
        # else:
        # #    print("Elemento vacio")
        #     break
        ## **************************************************
        
        #  ahora ahy que pedir cual archivo nos mande
        #request_files(nombres, adrs, host_port)
    
    # crear 2 listas independientes y luego iterarlas, asignarles los nombres y tamanos respectivamente
    # for a in range (len(nombre)):
    #     #  aqui tomar los nombres y pedirlos    
    #     #con.send(nombre[a].encode("utf8"))
    #     print (nombre[a])
    #     while True:    # recibe la data del archivo
    #         file_data = con.recv(buffer)      
    #         if not file_data:
    #             break
        #     # revisar como comparar el tamaño que ya tengo vs el tamaño que recibo
        #     #https://stackoverflow.com/questions/449560/how-do-i-determine-the-size-of-an-object-in-python
        #     if int(nombres[1]) == sys.getsizeof(file_data):
        #         break
        
        # # Aqui guardo los archivos a disco revisar la funcion mk2
        directorio = crear_directorio()
        if directorio == "<BAD>":
            print(" no se puede crear la ubicacion")
            print(type(directorio))  # revisar el tipo que regresa esta funcion¡¡¡¡
            print(directorio)
        else:
            print(f"0 .. {nombre}")
            print(f"1 .. {size}")
            #print (dataFile)
            file_name = directorio + nombre  # path to the file,   
            file_size = int(size)   # size of file
            
                   # actual data of the file
            file_create = open(file_name, 'wb')  # open file for writing in binary 
            #while (dataFile):   # creacion y escritura del archivo
            file_create.write(dataFile)   # writes to the file
            file_create.close()     #close the file
            check = os.path.getsize(file_name)  # returns the size of the file
            print (f"Tamaño esperado: {file_size} ... tamaño real {check} ")
            
            if check == file_size:
                #con.send(code_ok.encode())
                log("archivo " + file_name + " creado, tamaño: " + file_size)
            else:
                #con.sendall(code_bad.encode())
                log("No se pudo crear el archivo: " + file_name)
    
    log("Respaldo de Archivos completado")
    print(f"Respaldo de archivos terminada")

[SGaist]

Hi,

Wouldn't it be simpler to integrate rsync in your application ? It seems that this tool would do the work you are looking to implement.

[Lightshadown]

@SGaist can you do that on qt c++? as far as i know it only works for linux and python, my client would live on windows and my server inside a RPi 3

[SGaist]

librsync might fit your needs. Note that it does not implement the rsync wire protocol.

[Lightshadown]

well at the end i simply sended all the data and split everything inside a list, not the best thing to do but it works, heres the code

def servidorMk3():
    host = socket.gethostname()
    host_port = 8000
    host_ip = socket.gethostbyname(host)
    buffer = 1024
    code_end = "<END>"
    code_ok = "<OK>"
    code_fin = "<FIN>"
    code_sep = "<SEP>"
    nombre, size, dataFile, elemt = [], [], [], []
    
    print (f"Servidor Mk3")
    print (f"Ip del servidor: " + host_ip)
    log("Inicio del servidorMk3 " + host_ip)
   
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as tcp_conn:
        tcp_conn.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        tcp_conn.bind((host_ip, host_port))
        tcp_conn.listen(1)
        con, adrs = tcp_conn.accept()
        print (f"Conecion Exitosa")
        log("Connecion exitosa con el server: " + adrs[0])
        
        while True:
            alldata = con.recv(buffer)
            while True:         # recibe la info en general
                alldata = alldata + con.recv(buffer)
                if code_fin.encode("utf8") in alldata:  # si llega <FIN>, se acabo la lista
                    break     

            data = alldata.split(b'<SEP>')    #separar manualmente
            while code_fin.encode("utf8") in data: data.remove(code_fin.encode("utf8"))

            for i in range(0, len(data),3):
                bloque = data[i:i+3]
                nombre.append(bloque[0].decode("utf8"))
                print(nombre)
                size.append(bloque[1].decode("utf8"))
                dataFile.append(bloque[2])

            directorio = crear_directorio()
            for x in range(len(nombre)):
                if directorio == "<BAD>":
                    print(" no se puede crear la ubicacion")
                    print(type(directorio))  # revisar el tipo que regresa esta funcion¡¡¡¡
                    print(directorio)
                    break
                else:
                    print(f"0 .. {nombre[x]}")
                    print(f"1 .. {size[x]}")
                    #print (dataFile)
                    file_name = directorio + nombre[x]  # path to the file,   
                    file_size = int(size[x])   # size of file
                 
                    file_create = open(file_name, 'wb')  # open file for writing in binary
                    file_create.write(dataFile[x])   # writes to the file
                    file_create.close()     #close the file
                    check = os.path.getsize(file_name)  # returns the size of the file
                    print (f"Tamaño esperado: {file_size} ... tamaño real {check} ")
                    
                    if check == file_size:
                        log("archivo " + file_name + " creado, tamaño: " + str(file_size))
                    else:
                        log("No se pudo crear el archivo: " + file_name)
                if x == len(nombre):
                    log("Respaldo de Archivos completado")                
            
    tcp_conn.close()
    print(f"Respaldo de archivos terminada")