Issue with Worker Class when used with QTables

SGaist · 24 Jul 2024, 19:36

Hi,

This is more than 600 lines of pretty convoluted code that uses way too many globals.
If you want an answer please reduce it so that it can be used to reproduce your issue.

imissthecommandline · S SGaist 24 Jul 2024, 19:15

@SGaist
ouch, sorry about that

here's a version with pretty much all but the essentials removed:

# -*- coding: utf-8 -*-
"""
Created on Wed Jul 24 15:22:55 2024

@author: pierre
"""

#libraries for array management and graphing
import pandas as pd
import numpy as np
import matplotlib as plt

#libraries for system access and gui foundation
import sys 
from PyQt6.QtWidgets import (
    QApplication,
    QLabel,
    QMainWindow,
    QStatusBar,
    QToolBar,
    QStackedWidget,
    QStackedLayout,
    QWidget,
    QTabWidget,
    QVBoxLayout,
    QGridLayout,
    QPushButton,
    QLineEdit,
    QTableView
)

from PyQt6 import QtCore, QtGui, QtWidgets
from PyQt6.QtGui import *
from PyQt6.QtWidgets import *
from PyQt6.QtCore import *


#placeholde for stuff i haven't implemented in full yet
placeholder = "<unimplemented val!>"

#Library Imports for core management program
import socket
import threading
import time
import pickle

global window

#allows for utilization of threads in gui backend
class Worker(QRunnable):
    def __init__(self, fn, *args):
        super(Worker, self).__init__()
        # Store constructor arguments (re-used for processing)
        self.fn = fn
        self.args = args
        self.signals = WorkerSignals()

    @pyqtSlot()
    def run(self):
        '''
        Initialise the runner function with passed args, kwargs.
        '''
        self.fn(self, *self.args)
        
#implementation of Qobject for handling signals sent from worker threads
class WorkerSignals(QObject):
    operations = pyqtSignal(tuple)
    

#code taken from pyqt tutorial (link below)
#https://www.pythonguis.com/tutorials/pyqt6-qtableview-modelviews-numpy-pandas/
class table_model(QtCore.QAbstractTableModel):
    def __init__(self, data):
        super(table_model, self).__init__()
        self._data = data

    def data(self, index, role):
        if role == Qt.ItemDataRole.DisplayRole:
            value = self._data.iloc[index.row(), index.column()]
            return str(value)

    def rowCount(self, index):
        return self._data.shape[0]

    def columnCount(self, index):
        return self._data.shape[1]

    def headerData(self, section, orientation, role):
        # section is the index of the column/row.
        if role == Qt.ItemDataRole.DisplayRole:
            if orientation == Qt.Orientation.Horizontal:
                return str(self._data.columns[section])

            if orientation == Qt.Orientation.Vertical:
                return str(self._data.index[section])
    
    def flags(self, index):
        return Qt.ItemFlag.ItemIsSelectable | Qt.ItemFlag.ItemIsEnabled | Qt.ItemFlag.ItemIsEditable
        
    #append the dataframe
    def appendSelf(self,new_val):
        self._data = pd.concat([self._data,new_val])
        self.layoutChanged.emit()
        return 0
        
    #edit a specific value
    def editSelf(self,new_val,index,column):
        self._data.at[index,column] = new_val
        self.layoutChanged.emit()
        return 0
    
    #remove a line
    def removeSelf(self,index):
        self._data.set_index(index)
        self._data.reset_index(drop=True)
        self.layoutChanged.emit()
        return 0
    
#Main GUI window coding
class Window(QMainWindow):
    def __init__(self):
        super().__init__(parent=None)
        #values for window resolution
        self.x_res = 640
        self.y_res = 480
        self.setWindowTitle("S.T.A.R.F.I.S.H")
        #various set-up functions for the gui
        self.stack_init()
        self.setGeometry(0, 30, self.x_res, self.y_res)
        #setup for thread manager
        self.threadpool = QThreadPool()

        
    def stack_init(self):
        #all possible screens are organized through tabs at the top
        #all sub-tabs are part of a stack
        
        #basic setup
        self.layout = QVBoxLayout(self)
        self.main_tabs = QTabWidget(self)
        self.main_tabs.resize(self.x_res - 70, self.y_res - 70) 
        self.main_tabs.move(10, 40) 
        
        #custom tab init
        self.home_tab = QWidget(self)
        self.main_tabs.addTab(self.home_tab,"Home")
        self.config_tab = QWidget(self)
        self.main_tabs.addTab(self.config_tab,"Configuration")
        self.conn_tab = QWidget(self)
        self.main_tabs.addTab(self.conn_tab,"Connections")
        self.conn_man_tab = QWidget(self)
        self.main_tabs.addTab(self.conn_man_tab,"Connection Management")
        self.run_tab = QWidget(self)
        self.main_tabs.addTab(self.run_tab,"Run")
        self.layout.addWidget(self.main_tabs) 
        self.setLayout(self.layout) 
        
        #home tab setup
        #label 1 formatting
        self.home_tab.layout = QVBoxLayout(self)
        self.top_label = QLabel()
        self.top_label.setText("BlaBlaBla this tab isn't the problem") 
        self.home_tab.layout.addWidget(self.top_label)

        self.home_tab.setLayout(self.home_tab.layout)
        
        #options tab setup
        #each adjustable setting has a qlabel in column 0 saying what it is
        #this what the value is and the current value
        #a input box in column 4 lets you change the value
        
        self.config_tab.layout = QGridLayout(self)
        self.upload_button = QPushButton("&Upload custom config", self)
        self.config_tab.layout.addWidget(self.upload_button,0,0,1,5)
        #columns 1,3,5 are thin for l'aesthétique :)
        for i in range(3):
            self.config_tab.layout.setColumnMinimumWidth(1 + (i*2), 10)
            
        #network settings start
        self.config_ops_1 = QLabel("Go to the connections tab", self)
        self.config_ops_1.setStyleSheet("border: 2px solid blue;") 
        self.config_tab.layout.addWidget(self.config_ops_1,1,0)
        
        self.config_tab.setLayout(self.config_tab.layout)
        
        #connections tab setup

        
        self.conn_tab.layout = QGridLayout(self)
        self.start_connecting = QPushButton("&Start a thread", self)
        self.start_connecting.clicked.connect(lambda: self.threadstarter(arbitrary_task))
        self.conn_tab.layout.addWidget(self.start_connecting,0,0,1,2)
        #initially array for connected devices
        self.conn_devices_table = QTableView()
        self.connected_devices = pd.DataFrame([
            ["{server_ip}", "{server_port}", "Connections Tab","{current_mode}"],
            ],columns = ["Ipv4 Address", "Port", "Device State","Current Program"],
            index = ["Server"]
            )
        self.device_list = table_model(self.connected_devices)
        self.conn_devices_table.setModel(self.device_list)
        self.conn_tab.layout.addWidget(self.conn_devices_table,2,0,5,5)
        self.conn_tab.setLayout(self.conn_tab.layout)
        
        #connection management tab setup
        
        #button for authorizing all connections, button for clearing network
        self.conn_man_tab.layout = QGridLayout(self)
        self.authorize_connections = QPushButton("&Start a thread", self)
        self.authorize_connections.clicked.connect(lambda: self.connection_authorizer(arbitrary_task))
        self.conn_man_tab.layout.addWidget(self.authorize_connections,0,0,1,2)
        self.conn_man_tab.setLayout(self.conn_man_tab.layout)
        
        #run tab setup
        self.run_tab.layout = QGridLayout(self)
        self.run_device_label = QLabel("Other Table I Guess",self)
        self.run_device_label.setStyleSheet("border: 2px solid blue;")
        self.run_tab.layout.addWidget(self.run_device_label,0,0,1,1)
        
        self.run_device_table = QTableView()
        self.connected_devices_run = pd.DataFrame([
            ["Running","{current_mode}","N/A"],
            ],columns = ["Device State","Current Program","Last Ping"],
            index = ["Server"]
        )
        self.device_list_run = table_model(self.connected_devices_run)
        self.run_device_table.setModel(self.device_list_run)
        self.run_tab.layout.addWidget(self.run_device_table,4,0,5,5)
        
        self.run_tab.setLayout(self.run_tab.layout)
    
    def threadstarter(self, function, *args):
        new_thread = Worker(function, *args)
        new_thread.signals.operations.connect(self.threadhandler)
        self.threadpool.start(new_thread)
        
    #note for any additions, is set to expect a tuple
    #sending an int closes the thread, unless the int is in a tuple (duh)
    def threadhandler(self, command_list):
        pass

def arbitrary_task(*args):
    print("Started Thread")
    while(True):
        pass

def main():
    global window
    app = QApplication([])
    window = Window()
    window.show()
    sys.exit(app.exec())
    
#call main to start program
main()

it seems to only lag on tabs with tables, even though we aren't interacting with them. any ideas as to why this is?

SGaist · 24 Jul 2024, 21:50

What is the procedure to reproduce your issue with that version ?
By the way it crashes on line 210 when clicking "Start a thread" on the "Connection management" tab.

imissthecommandline · S SGaist 24 Jul 2024, 20:49

@SGaist the connected function on line 210 should be

lambda: self.threadstarter(arbitrary_task))

I messed up switching it out for the simplified version.

If you click the start thread button a bunch of times, the program lags, but only when going to a tab with a table or from a tab with a table removing the connections tab table removes the lag.

To reproduce:
Run Program
Click on Connections Tab
Click the start thread button 8 or so times
Click around the tabs

I think pyqt6 might have a hard time managing threads and tables, but it's probably my implementation, as this is my first time with pyqt6

Thank you

imissthecommandline · wrote on 26 Jul 2024, 14:38

Ok so i've figured out some interesting things about this:

Using the .hide() function on the table removes the lag. using .hide() on both of the tables before starting a new thread then .show() after the thread is started moderately reduces the lag.
Completely reinitializing the table by removing the widget from the layout then adding it again after starting a new thread reduces the lag more than method 1, but there's still some noticeable lag
3.Adding the tables AFTER starting the threads has no impact at all, the lag is just as bad

Still not sure what the issue is though

Here's the code for anyone insane enough to try and help

# -*- coding: utf-8 -*-
"""
Created on Wed Jul 24 15:22:55 2024

@author: pierre
"""

#libraries for array management and graphing
import pandas as pd
import numpy as np
import matplotlib as plt

#libraries for system access and gui foundation
import sys 
from PyQt6.QtWidgets import (
    QApplication,
    QLabel,
    QMainWindow,
    QStatusBar,
    QToolBar,
    QStackedWidget,
    QStackedLayout,
    QWidget,
    QTabWidget,
    QVBoxLayout,
    QGridLayout,
    QPushButton,
    QLineEdit,
    QTableView
)

from PyQt6 import QtCore, QtGui, QtWidgets
from PyQt6.QtGui import *
from PyQt6.QtWidgets import *
from PyQt6.QtCore import *


#placeholde for stuff i haven't implemented in full yet
placeholder = "<unimplemented val!>"

#Library Imports for core management program
import socket
import threading
import time
import pickle

global window

#allows for utilization of threads in gui backend
class Worker(QRunnable):
    def __init__(self, fn, *args):
        super(Worker, self).__init__()
        # Store constructor arguments (re-used for processing)
        self.fn = fn
        self.args = args
        self.signals = WorkerSignals()

    @pyqtSlot()
    def run(self):
        '''
        Initialise the runner function with passed args, kwargs.
        '''
        self.fn(self, *self.args)
        
#implementation of Qobject for handling signals sent from worker threads
class WorkerSignals(QObject):
    operations = pyqtSignal(tuple)
    

#code taken from pyqt tutorial (link below)
#https://www.pythonguis.com/tutorials/pyqt6-qtableview-modelviews-numpy-pandas/
class table_model(QtCore.QAbstractTableModel):
    def __init__(self, data):
        super(table_model, self).__init__()
        self._data = data

    def data(self, index, role):
        if role == Qt.ItemDataRole.DisplayRole:
            value = self._data.iloc[index.row(), index.column()]
            return str(value)

    def rowCount(self, index):
        return self._data.shape[0]

    def columnCount(self, index):
        return self._data.shape[1]

    def headerData(self, section, orientation, role):
        # section is the index of the column/row.
        if role == Qt.ItemDataRole.DisplayRole:
            if orientation == Qt.Orientation.Horizontal:
                return str(self._data.columns[section])

            if orientation == Qt.Orientation.Vertical:
                return str(self._data.index[section])
    
    def flags(self, index):
        return Qt.ItemFlag.ItemIsSelectable | Qt.ItemFlag.ItemIsEnabled | Qt.ItemFlag.ItemIsEditable
        
    #append the dataframe
    def appendSelf(self,new_val):
        self._data = pd.concat([self._data,new_val])
        self.layoutChanged.emit()
        return 0
        
    #edit a specific value
    def editSelf(self,new_val,index,column):
        self._data.at[index,column] = new_val
        self.layoutChanged.emit()
        return 0
    
    #remove a line
    def removeSelf(self,index):
        self._data.set_index(index)
        self._data.reset_index(drop=True)
        self.layoutChanged.emit()
        return 0
    
#Main GUI window coding
class Window(QMainWindow):
    def __init__(self):
        super().__init__(parent=None)
        #values for window resolution
        self.x_res = 640
        self.y_res = 480
        self.setWindowTitle("S.T.A.R.F.I.S.H")
        #various set-up functions for the gui
        self.stack_init()
        self.setGeometry(0, 30, self.x_res, self.y_res)
        #setup for thread manager
        self.threadpool = QThreadPool()

        
    def stack_init(self):
        #all possible screens are organized through tabs at the top
        #all sub-tabs are part of a stack
        
        #basic setup
        self.layout = QVBoxLayout(self)
        self.main_tabs = QTabWidget(self)
        self.main_tabs.resize(self.x_res - 70, self.y_res - 70) 
        self.main_tabs.move(10, 40) 
        
        #custom tab init
        self.home_tab = QWidget(self)
        self.main_tabs.addTab(self.home_tab,"Home")
        self.config_tab = QWidget(self)
        self.main_tabs.addTab(self.config_tab,"Configuration")
        self.conn_tab = QWidget(self)
        self.main_tabs.addTab(self.conn_tab,"Connections")
        self.conn_man_tab = QWidget(self)
        self.main_tabs.addTab(self.conn_man_tab,"Connection Management")
        self.run_tab = QWidget(self)
        self.main_tabs.addTab(self.run_tab,"Run")
        self.layout.addWidget(self.main_tabs) 
        self.setLayout(self.layout) 
        
        #home tab setup
        #label 1 formatting
        self.home_tab.layout = QVBoxLayout(self)
        self.top_label = QLabel()
        self.top_label.setText("BlaBlaBla this tab isn't the problem") 
        self.home_tab.layout.addWidget(self.top_label)

        self.home_tab.setLayout(self.home_tab.layout)
        
        #options tab setup
        #each adjustable setting has a qlabel in column 0 saying what it is
        #this what the value is and the current value
        #a input box in column 4 lets you change the value
        
        self.config_tab.layout = QGridLayout(self)
        self.upload_button = QPushButton("&Upload custom config", self)
        self.config_tab.layout.addWidget(self.upload_button,0,0,1,5)
        #columns 1,3,5 are thin for l'aesthétique :)
        for i in range(3):
            self.config_tab.layout.setColumnMinimumWidth(1 + (i*2), 10)
            
        #network settings start
        self.config_ops_1 = QLabel("Go to the connections tab", self)
        self.config_ops_1.setStyleSheet("border: 2px solid blue;") 
        self.config_tab.layout.addWidget(self.config_ops_1,1,0)
        
        self.config_tab.setLayout(self.config_tab.layout)
        
        #connections tab setup

        
        self.conn_tab.layout = QGridLayout(self)
        self.start_connecting = QPushButton("&Start a thread", self)
        self.start_connecting.clicked.connect(lambda: self.threadstarter(arbitrary_task))
        self.conn_tab.layout.addWidget(self.start_connecting,0,0,1,2)
        #initial array for connected devices
        self.conn_devices_table = QTableView()
        self.connected_devices = pd.DataFrame([
            ["{server_ip}", "{server_port}", "Connections Tab","{current_mode}"],
            ],columns = ["Ipv4 Address", "Port", "Device State","Current Program"],
            index = ["Server"]
            )
        self.device_list = table_model(self.connected_devices)
        self.conn_devices_table.setModel(self.device_list)
        self.conn_tab.layout.addWidget(self.conn_devices_table,2,0,5,5)
        self.conn_tab.setLayout(self.conn_tab.layout)
        
        #connection management tab setup
        
        #button for authorizing all connections, button for clearing network
        self.conn_man_tab.layout = QGridLayout(self)
        self.authorize_connections = QPushButton("&Tabletime", self)
        self.authorize_connections.clicked.connect(lambda: self.testingthing())
        self.conn_man_tab.layout.addWidget(self.authorize_connections,0,0,1,2)
        self.conn_man_tab.setLayout(self.conn_man_tab.layout)
        
        #run tab setup
        self.run_tab.layout = QGridLayout(self)
        self.run_device_label = QLabel("Other Table I Guess",self)
        self.run_device_label.setStyleSheet("border: 2px solid blue;")
        self.run_tab.layout.addWidget(self.run_device_label,0,0,1,1)
        
        self.run_device_table = QTableView()
        self.connected_devices_run = pd.DataFrame([
            ["Running","{current_mode}","N/A"],
            ],columns = ["Device State","Current Program","Last Ping"],
            index = ["Server"]
        )
        self.device_list_run = table_model(self.connected_devices_run)
        self.run_device_table.setModel(self.device_list_run)
        self.run_tab.layout.addWidget(self.run_device_table,4,0,5,5)
        self.run_tab.setLayout(self.run_tab.layout)
    
    def threadstarter(self, function, *args):
        #the mechanics of this frighten me
        self.conn_tab.layout.removeWidget(self.conn_devices_table)
        self.conn_tab.setLayout(self.conn_tab.layout)
        self.run_tab.layout.removeWidget(self.run_device_table)
        self.run_tab.setLayout(self.run_tab.layout)
        
        ##############################
        #normal thread starting stuff#
        ##############################
        new_thread = Worker(function, *args)
        new_thread.signals.operations.connect(self.threadhandler)
        self.threadpool.start(new_thread)
        #####################################
        #end of normal thread starting stuff#
        #####################################
        
        self.conn_devices_table = QTableView()
        self.device_list = table_model(self.connected_devices)
        self.conn_devices_table.setModel(self.device_list)
        self.conn_tab.layout.addWidget(self.conn_devices_table,2,0,5,5)
        self.conn_tab.setLayout(self.conn_tab.layout)
        
        self.run_device_table = QTableView()
        self.device_list_run = table_model(self.connected_devices_run)
        self.run_device_table.setModel(self.device_list_run)
        self.run_tab.layout.addWidget(self.run_device_table,4,0,5,5)
        self.run_tab.setLayout(self.run_tab.layout)
        
    def testingthing(self):
        #for adding the table after the threads
        
        #This is dumb
        self.conn_devices_table = QTableView()
        self.connected_devices = pd.DataFrame([
            ["{server_ip}", "{server_port}", "Connections Tab","{current_mode}"],
            ],columns = ["Ipv4 Address", "Port", "Device State","Current Program"],
            index = ["Server"]
            )
        self.device_list = table_model(self.connected_devices)
        self.conn_devices_table.setModel(self.device_list)
        self.conn_tab.layout.addWidget(self.conn_devices_table,2,0,5,5)
        self.conn_tab.setLayout(self.conn_tab.layout)
        
        
        self.run_device_table = QTableView()
        self.connected_devices_run = pd.DataFrame([
            ["Running","{current_mode}","N/A"],
            ],columns = ["Device State","Current Program","Last Ping"],
            index = ["Server"]
        )
        self.device_list_run = table_model(self.connected_devices_run)
        self.run_device_table.setModel(self.device_list_run)
        self.run_tab.layout.addWidget(self.run_device_table,4,0,5,5)
        self.run_tab.setLayout(self.run_tab.layout)

        
    #note for any additions, is set to expect a tuple
    #sending an int closes the thread, unless the int is in a tuple (duh)
    def threadhandler(self, command_list):
        pass

def arbitrary_task(*args):
    print("Started Thread")
    while(True):
        pass

def main():
    global window
    app = QApplication([])
    window = Window()
    window.show()
    sys.exit(app.exec())
    
#call main to start program
main()

SGaist · 31 Jul 2024, 13:21

Is your code based on that tutorial ?

imissthecommandline · S SGaist 27 Jul 2024, 19:36

@SGaist in part yes. Do you have a better tutorial or resource you reccomend?

imissthecommandline · wrote on 31 Jul 2024, 13:58

sorry, i used this one: https://www.pythonguis.com/tutorials/multithreading-pyqt6-applications-qthreadpool/

imissthecommandline · wrote on 31 Jul 2024, 14:11

ok, so what is happening is that every thread is making several calls to the table model every time the tab is opened. is there a way you recommend to prevent this? @SGaist

SGaist · wrote on 2 Aug 2024, 18:57

You can implement a debouncer. For example, accumulate a certain amount of changes and only then signal that the model has changed to minimize the amount of time the views will query the model.
Also, don't forget to properly use lock mechanism to avoid writing to the same object from multiple different threads.