Filter checked checkboxes - qtableview

By this I tried to make a minimal reproducible example of what I now have.
Using QSortFilterProxyModel() to find text and a custom SortFilterProxyModel to show all the selected checkboxes.
It looks like both proxy's conflicting eachother and giving the wrong row number back. (There is also an option for adding value with a dialog and also this goes to the wrong row with the custom proxy enabled, but works without the custom proxy).
class winConfigurator(QtWidgets.QMainWindow):
def __init__(self, data=None):
super(winConfigurator,self).__init__()
uic.loadUi(os.path.join(os.path.dirname(__file__), 'winConfiguratorView.ui'), self)
self.leSearch.returnPressed.connect(self.searchField)
self.chMatchSelected.toggled.connect(self.showSelected)
def readFile(self, filename):
self.model = TableModel([headers, newRows])
self.proxy_model = QSortFilterProxyModel()
self.proxy_model.setSourceModel(self.model)
"""
Proxy for 'Show Selected'
"""
self.custom_proxy_model = SortFilterProxyModel()
self.custom_proxy_model.setSourceModel(self.proxy_model)
self.tableView.setModel(self.custom_proxy_model)
def searchField(self):
self.proxy_model.setFilterFixedString(self.leSearch.text())
def showSelected(self, state = None):
self.custom_proxy_model.clearFilter()
checkstate_items = self.model.checks.items()
if state == True:
self.custom_proxy_model.setFilterByCheckbox(checkstate_items)
class SortFilterProxyModel(QSortFilterProxyModel):
def __init__(self, *args, **kwargs):
QSortFilterProxyModel.__init__(self, *args, **kwargs)
self.filters = {}
def setFilterByCheckbox(self, checkstates = {}):
self.filters = checkstates
self.invalidateFilter()
def clearFilter(self):
self.filters = {}
self.invalidateFilter()
def filterAcceptsRow(self, source_row, source_parent):
"""
Check if checkbox is checked and show this row
Slow, 7 seconds for 50k rows.
"""
try:
values = []
if self.filters:
for index, is_checked in self.filters:
if is_checked:
row = index.row()
model = self.sourceModel()
if hasattr(model, 'mapToSource'):
index = model.index(source_row, 0, source_parent)
if not index.parent().isValid():
modelIndex = model.mapToSource(index)
source_row = modelIndex.row()
if row == source_row:
values.append(index)
return any(values)
return True
except Exception as e:
# print(e)
return True

Related

connection in a different thread can't read tables created in another thread?

In a testing suite I have a fixture that drop all the tables in an engine, then start fresh and create all the tables. After this fixture logic, my test case runs, using the newly created table.
The fixture and the test case are run in the MainThread, while the database consumer is a web application server run in another thread.
However, I keep getting: sqlite3.OperationalError: no such table: ***
I've checked that they are using the same in-memory engine, but different connections(this is correct). And I've checked that the fixture does run before the consumer thread starts running.
What could be possible cause?
My code is as below:
import os
import pytest
import cherrypy
class DAL:
def __init__(self,
path="database",
filename=None,
conn_string=None,
echo=False):
if filename is None and conn_string is None:
conn_string = "sqlite:///:memory:"
elif conn_string is not None:
conn_string = conn_string
else:
conn_string = f'sqlite:///{os.path.abspath(path)}/{filename}'
self.conn_string = conn_string
engine = create_engine(conn_string, echo=echo)
Session_Factory = sessionmaker(bind=engine)
self.Session = sqlalchemy.orm.scoped_session(Session_Factory)
def __str__(self):
return f"<DAL>object: {self.conn_string}, at {hex(id(self))}"
def get_a_dbsession(self):
opened_db = self.Session()
return opened_db
def __enter__(self):
return self.get_a_dbsession()
def __exit__(self, exception_type, exception_value, exception_traceback):
opened_db = self.Session()
try:
opened_db.commit()
except:
opened_db.rollback()
raise
finally:
self.Session.remove()
else:
opened_db.close()
raise exception_type
def create_schema(self):
SchemaBase.metadata.create_all(self.Session().connection().engine)
class SAEnginePlugin(cherrypy.process.plugins.SimplePlugin):
def __init__(self, bus, dal):
"""
The plugin is registered to the CherryPy engine.
"""
cherrypy.process.plugins.SimplePlugin.__init__(self, bus)
self.dal = dal
def start(self):
self.bus.subscribe("bind-session", self.bind)
def stop(self):
self.bus.unsubscribe("bind-session", self.bind)
if self.dal:
del self.dal
def bind(self):
"""
Whenever this plugin receives the 'bind-session' message, it applies
this method and bind the received session to the engine.
"""
# self.dal.Session.configure(bind=self.dal.engine)
session = self.dal.get_a_dbsession()
return session
class SATool(cherrypy.Tool):
def __init__(self):
"""
This tool binds a session to the engine each time
a requests starts and commits/rollbacks whenever
the request terminates.
"""
cherrypy.Tool.__init__(self,
'on_start_resource',
self.bind_session,
priority=20)
def _setup(self):
cherrypy.Tool._setup(self)
cherrypy.request.hooks.attach('on_end_resource',
self.close_session,
priority=80)
def bind_session(self):
"""
Attaches a session to the request's scope by requesting
the SA plugin to bind a session to the SA engine.
"""
session = cherrypy.engine.publish('bind-session').pop()
cherrypy.request.db = session
def close_session(self):
"""
Commits the current transaction or rollbacks if an error occurs.
In all cases, the current session is unbound and therefore
not usable any longer.
"""
if not hasattr(cherrypy.request, 'db'):
return
try:
cherrypy.request.db.commit()
except:
cherrypy.request.db.rollback()
raise
finally:
cherrypy.request.db.close()
cherrypy.request.db = None
# Register the SQLAlchemy tool
cherrypy.tools.db = SATool()
class UnitServer:
...
#cherrypy.expose
#cherrypy.tools.json_in()
def list_filtered_entries(self):
...
queryOBJ = cherrypy.request.db.query(classmodel_obj)
...
############# main module code below ############:
# mocking 'db':
dal = database.DAL()
# configure cherrypy:
SAEnginePlugin(cherrypy.engine, dal).subscribe()
#pytest.fixture(autouse=True) # automatically run before every test case
def mocked_dal(request):
# first, clean the database by dropping all tables:
database.SchemaBase.metadata.drop_all(dal.Session().connection().engine)
# second, create the schema from blank:
dal.create_schema()
# third, insert some dummy data record:
...
db.commit()
class TestMyUnitServer(cherrypy.test.helper.CPWebCase):
#staticmethod
def setup_server():
...
server_app = UnitServer()
cherrypy.tree.mount(server_app, '', {'/': {'tools.db.on': True}})
def test_list_filtered_entries_allentries(self):
...
self.getPage('/list_filtered_entries',
headers=[("Accept", "application/json"),
('Content-type', 'application/json'),
('Content-Length',
str(len(json.dumps(query_params)))),
("Connection", "keep-alive"),
("Cache-Control", "max-age=0")],
body=serialized_query_params,
method="POST")
self.assertStatus('200 OK')

pyqt4 QTableView in QMainWindow with csv input and headers

I am working with a QMainWindow and adding a QTableView widget. The table is to be filled with data from a csv file. The csv file first row has the headers, but I cannot find how to write that row into the headers. Even inputting a test header list does not work.
Also I want to reverse sort on the "time" column.
Here is code restricted to mostly the table:
import sys
import csv
from PyQt4 import QtGui
from PyQt4.QtCore import *
from array import *
class UserWindow(QtGui.QMainWindow):
def __init__(self, parent=None):
super(UserWindow, self).__init__()
self.specModel = QtGui.QStandardItemModel(self)
self.specList = self.createSpecTable()
self.initUI()
def specData(self):
with open('testFile.csv', 'rb') as csvInput:
for row in csv.reader(csvInput):
if row > 0:
items = [QtGui.QStandardItem(field) for field in row]
self.specModel.appendRow(items)
def createSpecTable(self):
self.specTable = QtGui.QTableView()
# This is a test header - different from what is needed
specHdr = ['Test', 'Date', 'Time', 'Type']
self.specData()
specM = specTableModel(self.specModel, specHdr, self)
self.specTable.setModel(specM)
self.specTable.setShowGrid(False)
vHead = self.specTable.verticalHeader()
vHead.setVisible(False)
hHead = self.specTable.horizontalHeader()
hHead.setStretchLastSection(True)
self.specTable.sortByColumn(3, Qt.DescendingOrder)
return self.specTable
def initUI(self):
self.ctr_frame = QtGui.QWidget()
self.scnBtn = QtGui.QPushButton("Sample")
self.refBtn = QtGui.QPushButton("Reference")
self.stpBtn = QtGui.QPushButton("Blah")
# List Window
self.specList.setModel(self.specModel)
# Layout of Widgets
pGrid = QtGui.QGridLayout()
pGrid.setSpacing(5)
pGrid.addWidget(self.scnBtn, 3, 0, 1, 2)
pGrid.addWidget(self.refBtn, 3, 2, 1, 2)
pGrid.addWidget(self.stpBtn, 3, 4, 1, 2)
pGrid.addWidget(self.specList, 10, 0, 20, 6)
self.ctr_frame.setLayout(pGrid)
self.setCentralWidget(self.ctr_frame)
self.statusBar()
self.setGeometry(300, 300, 400, 300)
self.setWindowTitle('Test')
class specTableModel(QAbstractTableModel):
def __init__(self, datain, headerdata, parent=None, *args):
QAbstractTableModel.__init__(self, parent, *args)
self.arraydata = datain
self.headerdata = headerdata
def rowCount(self, parent):
return len(self.arraydata)
def columnCount(self, parent):
return len(self.arraydata[0])
def data(self, index, role):
if not index.isValid():
return QVariant()
elif role != Qt.DisplayRole:
return QVariant()
return QVariant(self.arraydata[index.row()][index.column()])
def headerData(self, col, orientation, role):
if orientation == Qt.Horizontal and role == Qt.DisplayRole:
return self.headerdata[col]
return None
def main():
app = QtGui.QApplication(sys.argv)
app.setStyle(QtGui.QStyleFactory.create("plastique"))
ex = UserWindow()
ex.show()
sys.exit(app.exec_())
if __name__ == '__main__':
main()
and here is a really short csv file:
Run,Date,Time,Comment
data1,03/03/2014,00:04,Reference
data2,03/03/2014,02:00,Reference
data5,03/03/2014,02:08,Sample
data6,03/03/2014,13:57,Sample
Also the rowCount & columnCount definitions do not work.
Worked out answers to what I posted: Wrote a 'getHeader' function simply to read the first line of the csv file and returned the list. Added the following to the createSpecTable function:
specHdr = self.getHeader()
self.specModel.setHorizontalHeaderLabels(specHdr)
self.specModel.sort(2, Qt.DescendingOrder)
The last statement solved the reverse sort problem. The header line from the csv file was removed from the table by adding a last line to the specData function:
self.specModelremoveRow(0).
Finally the rowCount and columnCount were corrected with:
def rowCount(self, parent):
return self.arraydata.rowCount()
def columnCount(self, parent):
return self.arraydata.columnCount()

Tastypie API return only one object for given parameters

I am overwriting the get_obj_list function. Following some parameters and a random function I would like to return an object related to the actual object that follows the parameters. This works fine. How can I return only this obj instead of a one-entry-list? Is there another function that better fits my purpose?
class SentenceRandomResource(ModelResource):
class Meta:
queryset = Sentence.objects.filter()
resource_name = 'sentence/random'
always_return_data = True
authorization = ReadOnlyAuthorization()
filtering = {'internal': ALL}
def obj_get_list(self, bundle, **kwargs):
if 'case' in bundle.request.GET.keys() and 'lemma' in bundle.request.GET.keys() :
if 'number' in bundle.request.GET.keys() :
words = Word.objects.filter(case = bundle.request.GET['case'], number = bundle.request.GET['number'], lemma = bundle.request.GET['lemma'])
else :
words = Word.objects.filter(case = bundle.request.GET['case'], lemma = bundle.request.GET['lemma'])
number_of_words = len(words)
if number_of_words > 0 :
random_index = int(random.random()*number_of_words)+0
random_word = words[random_index]
sentence = random_word.sentence
return [sentence]
else: ...
else: ...
Thanks to method prepend_url you may add some special functionality not included in RESTful principles.
import random
from tastypie.http import HttpBadRequest
class SentenceRandomResource(ModelResource):
class Meta:
queryset = Sentence.objects.filter()
resource_name = 'sentence/random'
always_return_data = True
authorization = ReadOnlyAuthorization()
filtering = {'internal': ALL}
def prepend_urls(self, *args, **kwargs):
name = 'get_one_random'
return [url(r"^(?P<resource_name>%s)/%s%s$" %
(self._meta.resource_name, name, trailing_slash()),
self.wrap_view(name), name="api_%s" % name)]
def get_one_random(self, request, **kwargs):
"""
Gets one random sentence of sentences with provided `case` and `lemma`
params.
"""
case = request.GET.get('case')
lemma = request.GET.get('lemma')
number = request.GET.get('number')
if case and lemma:
query_params = {'case': case, 'lemma': lemma}
if number is not None:
query_params['number'] = number
words = Word.objects.filter(**query_params)
word = random.choice(words)
return self.create_response(request, {'sentence': word.sentence.__dict__})
else:
return self.error_response(request, {'error': 'lemma and case are required.'},
response_class=HttpBadRequest)
Example use:
GET ..../sentence/random/get_one_random/?case=1&lemma=2
{'sentence': 'asdfasdf'}

Writing items to a MySQL database in Scrapy

I am new to Scrapy, I had the spider code
class Example_spider(BaseSpider):
name = "example"
allowed_domains = ["www.example.com"]
def start_requests(self):
yield self.make_requests_from_url("http://www.example.com/bookstore/new")
def parse(self, response):
hxs = HtmlXPathSelector(response)
urls = hxs.select('//div[#class="bookListingBookTitle"]/a/#href').extract()
for i in urls:
yield Request(urljoin("http://www.example.com/", i[1:]), callback=self.parse_url)
def parse_url(self, response):
hxs = HtmlXPathSelector(response)
main = hxs.select('//div[#id="bookshelf-bg"]')
items = []
for i in main:
item = Exampleitem()
item['book_name'] = i.select('div[#class="slickwrap full"]/div[#id="bookstore_detail"]/div[#class="book_listing clearfix"]/div[#class="bookstore_right"]/div[#class="title_and_byline"]/p[#class="book_title"]/text()')[0].extract()
item['price'] = i.select('div[#id="book-sidebar-modules"]/div[#class="add_to_cart_wrapper slickshadow"]/div[#class="panes"]/div[#class="pane clearfix"]/div[#class="inner"]/div[#class="add_to_cart 0"]/form/div[#class="line-item"]/div[#class="line-item-price"]/text()').extract()
items.append(item)
return items
And pipeline code is:
class examplePipeline(object):
def __init__(self):
self.dbpool = adbapi.ConnectionPool('MySQLdb',
db='blurb',
user='root',
passwd='redhat',
cursorclass=MySQLdb.cursors.DictCursor,
charset='utf8',
use_unicode=True
)
def process_item(self, spider, item):
# run db query in thread pool
assert isinstance(item, Exampleitem)
query = self.dbpool.runInteraction(self._conditional_insert, item)
query.addErrback(self.handle_error)
return item
def _conditional_insert(self, tx, item):
print "db connected-=========>"
# create record if doesn't exist.
tx.execute("select * from example_book_store where book_name = %s", (item['book_name']) )
result = tx.fetchone()
if result:
log.msg("Item already stored in db: %s" % item, level=log.DEBUG)
else:
tx.execute("""INSERT INTO example_book_store (book_name,price)
VALUES (%s,%s)""",
(item['book_name'],item['price'])
)
log.msg("Item stored in db: %s" % item, level=log.DEBUG)
def handle_error(self, e):
log.err(e)
After running this I am getting the following error
exceptions.NameError: global name 'Exampleitem' is not defined
I got the above error when I added the below code in process_item method
assert isinstance(item, Exampleitem)
and without adding this line I am getting
**exceptions.TypeError: 'Example_spider' object is not subscriptable
Can anyone make this code run and make sure that all the items saved into database?
Try the following code in your pipeline
import sys
import MySQLdb
import hashlib
from scrapy.exceptions import DropItem
from scrapy.http import Request
class MySQLStorePipeline(object):
def __init__(self):
self.conn = MySQLdb.connect('host', 'user', 'passwd',
'dbname', charset="utf8",
use_unicode=True)
self.cursor = self.conn.cursor()
def process_item(self, item, spider):
try:
self.cursor.execute("""INSERT INTO example_book_store (book_name, price)
VALUES (%s, %s)""",
(item['book_name'].encode('utf-8'),
item['price'].encode('utf-8')))
self.conn.commit()
except MySQLdb.Error, e:
print "Error %d: %s" % (e.args[0], e.args[1])
return item
Your process_item method should be declared as: def process_item(self, item, spider): instead of def process_item(self, spider, item): -> you switched the arguments around.
This exception: exceptions.NameError: global name 'Exampleitem' is not defined indicates you didn't import the Exampleitem in your pipeline.
Try adding: from myspiders.myitems import Exampleitem (with correct names/paths ofcourse).
I think this way is better and more concise:
#Item
class pictureItem(scrapy.Item):
topic_id=scrapy.Field()
url=scrapy.Field()
#SQL
self.save_picture="insert into picture(`url`,`id`) values(%(url)s,%(id)s);"
#usage
cur.execute(self.save_picture,dict(item))
It's just like
cur.execute("insert into picture(`url`,`id`) values(%(url)s,%(id)s)" % {"url":someurl,"id":1})
Cause (you can read more about Items in Scrapy)
The Field class is just an alias to the built-in dict class and doesn’t provide any extra functionality or attributes. In other words, Field objects are plain-old Python dicts.

SQLAlchemy session voes in unittest

I've just started using SQLAlchemy a few days ago and right now I'm stuck with a problem that I hope anyone can shed some light on before I loose all my hair.
When I run a unittest, see snippet below, only the first test in the sequence is passing. The test testPhysicalPrint works just fine, but testRecordingItem fails with NoResultFound exception - No row was found for one(). But if I remove testPhysicalPrint from the test class, then testRecordingItem works.
I assume that the problem has something to do with the session, but I can't really get a grip of it.
In case anyone wonders, the setup is as follows:
Python 3.1 (Ubuntu 10.04 package)
SQLAlchemy 0.7.2 (easy_install:ed)
PostgreSQL 8.4.8 (Ubuntu 10.04 package)
PsycoPG2 2.4.2 (easy_installed:ed)
Exemple test:
class TestSchema(unittest.TestCase):
test_items = [
# Some parent class products
PrintItem(key='p1', title='Possession', dimension='30x24'),
PrintItem(key='p2', title='Andrzej Żuławski - a director', dimension='22x14'),
DigitalItem(key='d1', title='Every Man His Own University', url='http://www.gutenberg.org/files/36955/36955-h/36955-h.htm'),
DigitalItem(key='d2', title='City Ballads', url='http://www.gutenberg.org/files/36954/36954-h/36954-h.htm'),
]
def testPrintItem(self):
item = self.session.query(PrintItem).filter(PrintItem.key == 'p1').one()
assert item.title == 'Possession', 'Title mismatch'
def testDigitalItem(self):
item2 = self.session.query(DigitalItem).filter(DigitalItem.key == 'd2').one()
assert item2.title == 'City Ballads', 'Title mismatch'
def setUp(self):
Base.metadata.create_all()
self.session = DBSession()
self.session.add_all(self.test_items)
self.session.commit()
def tearDown(self):
self.session.close()
Base.metadata.drop_all()
if __name__ == '__main__':
unittest.main()
UPDATE
Here is the working code snippet.
# -*- coding: utf-8 -*-
import time
import unittest
from sqlalchemy import *
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import *
Base = declarative_base()
engine = create_engine('sqlite:///testdb', echo=False)
DBSession = sessionmaker(bind=engine)
class ItemMixin(object):
"""
Commons attributes for items, ie books, DVD:s...
"""
__tablename__ = 'testitems'
__table_args__ = {'extend_existing':True}
id = Column(Integer, autoincrement=True, primary_key=True)
key = Column(Unicode(16), unique=True, nullable=False)
title = Column(UnicodeText, default=None)
item_type = Column(Unicode(20), default=None)
__mapper_args__ = {'polymorphic_on': item_type}
def __init__(self, key, title=None):
self.key = key
self.title = title
class FooItem(Base, ItemMixin):
foo = Column(UnicodeText, default=None)
__mapper_args__ = {'polymorphic_identity':'foo'}
def __init__(self, foo=None, **kwargs):
ItemMixin.__init__(self, **kwargs)
self.foo = foo
class BarItem(Base, ItemMixin):
bar = Column(UnicodeText, default=None)
__mapper_args__ = {'polymorphic_identity':'bar'}
def __init__(self, bar=None, **kwargs):
ItemMixin.__init__(self, **kwargs)
self.bar = bar
# Tests
class TestSchema(unittest.TestCase):
# Class variables
is_setup = False
session = None
metadata = None
test_items = [
FooItem(key='f1', title='Possession', foo='Hello'),
FooItem(key='f2', title='Andrzej Żuławsk', foo='World'),
BarItem(key='b1', title='Wikipedia', bar='World'),
BarItem(key='b2', title='City Ballads', bar='Hello'),
]
def testFooItem(self):
print ('Test Foo Item')
item = self.__class__.session.query(FooItem).filter(FooItem.key == 'f1').first()
assert item.title == 'Possession', 'Title mismatch'
def testBarItem(self):
print ('Test Bar Item')
item = self.__class__.session.query(BarItem).filter(BarItem.key == 'b2').first()
assert item.title == 'City Ballads', 'Title mismatch'
def setUp(self):
if not self.__class__.is_setup:
self.__class__.session = DBSession()
self.metadata = Base.metadata
self.metadata.bind = engine
self.metadata.drop_all() # Drop table
self.metadata.create_all() # Create tables
self.__class__.session.add_all(self.test_items) # Add data
self.__class__.session.commit() # Commit
self.__class__.is_setup = True
def tearDown(self):
if self.__class__.is_setup:
self.__class__.session.close()
# Just for Python >=2.7 or >=3.2
#classmethod
def setUpClass(cls):
pass
#Just for Python >=2.7 or >=3.2
#classmethod
def tearDownClass(cls):
pass
if __name__ == '__main__':
unittest.main()
The most likely reason for this behavior is the fact that that data is not properly cleaned up between the tests. This explains why when you run only one test, it works.
setUp is called before every test, and tearDown - after.
Depending on what you would like to achieve, you have two options:
create data only once for all test.
In this case you if you had Python-2.7+ or Python-3.2+, you could use tearDownClass method. In your case you can handle it with a boolean class variable to prevent the code you have in setUp running more then once.
re-create data before every test
In this case you need to make sure that in the tearDown you delete all the data. This is what you are not doing right now, and I suspect that when the second test is ran, the call to one() fails not because it does not find an object, but because it finds more two objects matching the criteria.
Check the output of this code to understand the call sequence:
import unittest
class TestSchema(unittest.TestCase):
def testOne(self):
print '==testOne'
def testTwo(self):
print '==testTwo'
def setUp(self):
print '>>setUp'
def tearDown(self):
print '<<tearDown'
#classmethod
def setUpClass():
print '>>setUpClass'
#classmethod
def tearDownClass():
print '<<tearDownClass'
if __name__ == '__main__':
unittest.main()
Output:
>>setUp
==testOne
<<tearDown
>>setUp
==testTwo
<<tearDown
I have this as my tearDown method and it does work fine for my tests:
def tearDown (self):
"""Cleans up after each test case."""
sqlalchemy.orm.clear_mappers()