can't pickle instancemethod objects - pickle

I met a problem of pickle, Code is that:
import cPickle
class A(object):
def __init__(self):
self.a = 1
def methoda(self):
print(self.a)
class B(object):
def __init__(self):
self.b = 2
a = A()
self.b_a = a.methoda
def methodb(self):
print(self.b)
if __name__ == '__main__':
b = B()
with open('best_model1.pkl', 'w') as f:
cPickle.dump(b, f)
Error is that:
File "/usr/lib/python2.7/copy_reg.py", line 70, in _reduce_ex
raise TypeError, "can't pickle %s objects" % base.name TypeError: can't pickle instancemethod objects

You can if you use dill instead of cPickle.
>>> import dill
>>>
>>> class A(object):
... def __init__(self):
... self.a = 1
... def methods(self):
... print(self.a)
...
>>>
>>> class B(object):
... def __init__(self):
... self.b = 2
... a = A()
... self.b_a = a.methods
... def methodb(self):
... print(self.b)
...
>>> b = B()
>>> b_ = dill.dumps(b)
>>> _b = dill.loads(b_)
>>> _b.methodb()
2
>>>
Also see:
Can't pickle <type 'instancemethod'> when using python's multiprocessing Pool.map()

Also, when dill is installed pickle will work but as usual not cPickle.
import cPickle, pickle
class A(object):
def __init__(self):
self.a = 1
def methoda(self):
print(self.a)
class B(object):
def __init__(self):
self.b = 2
a = A()
self.b_a = a.methoda
def methodb(self):
print(self.b)
# try using cPickle
try:
c = cPickle.dumps(b)
d = cPickle.loads(c)
except Exception as err:
print('Unable to use cPickle (%s)'%err)
else:
print('Using cPickle was successful')
print(b)
print(d)
# try using pickle
try:
c = pickle.dumps(b)
d = pickle.loads(c)
except Exception as err:
print('Unable to use pickle (%s)'%err)
else:
print('Using pickle was successful')
print(b)
print(d)
>>> Unable to use cPickle (can't pickle instancemethod objects)
>>> Using pickle was successful
>>> <__main__.B object at 0x10e9b84d0>
>>> <__main__.B object at 0x13df07190>
for whatever reason, cPickle is not simply a C version of pickle 100 times faster but there are some differences

Related

Calling Class Function Without Period

New to python and it seems like a simple question but I haven't been able to find an answer. I found this code on github. I can't understand why it doesn't say model.forward(x).
import torch
from torch import nn
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self).__init__()
self.flatten = nn.Flatten()
def forward(self, x):
tensor = self.flatten(x)
return tensor
x = torch.rand(2, 2, 2, 2, 2, 2)
print(x)
model = NeuralNet()
output = model(x)
print(output)
Refere to this Why there are different output between model.forward(input) and model(input) and also read about
__call__
method.

The PyQt5 Application Stop completely

I have a problem here with my Pyqt5 app when I try to add the user input to mysql database in the Add_car_info function. When I press the button to add the info the app stop working and gives me this error message. tested on windows.
here is the code:
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
from PyQt5.QtCore import *
from PyQt5.uic import loadUiType
from PyQt5 import QtWidgets, QtGui
import threading
import sys
import os
from os import path
import mysql.connector
import MySQLdb
#get the ui file
FORM_CLASS,_=loadUiType(path.join(path.dirname(__file__),"main.ui"))
class MainApp(QMainWindow, FORM_CLASS):
def __init__(self, parent=None):
super(MainApp,self).__init__(parent)
QMainWindow.__init__(self)
self.setupUi(self)
self.Handel_Ui()
self.Handel_DB_Connect()
self.Handel_Buttons()
self.Handel_DB_Connect()
def Handel_Ui(self):
self.setWindowTitle('Car_System')
self.tabWidget.tabBar().setVisible(False)
def Handel_DB_Connect(self):
db = mysql.connector.connect(database='mydb',
host='localhost',
user='root',
port='3309',
password='toor')
self.cur = db.cursor()
QApplication.processEvents()
def Handel_Buttons(self):
self.pushButton.clicked.connect(self.Add_car_info)
self.pushButton_2.clicked.connect(self.Update_car_info)
self.pushButton_3.clicked.connect(self.Delete_car_info)
self.pushButton_4.clicked.connect(self.Add_fuel_info)
self.pushButton_9.clicked.connect(self.Update_fuel_info)
self.pushButton_5.clicked.connect(self.Add_maintenance_info)
self.pushButton_6.clicked.connect(self.Update_maintenance_info)
self.pushButton_7.clicked.connect(self.Add_Licence_info)
self.pushButton_8.clicked.connect(self.Update_Licence_info)
self.pushButton_17.clicked.connect(self.Add_Revenus_info)
self.pushButton_18.clicked.connect(self.Update_Revenus_info)
self.pushButton_19.clicked.connect(self.Add_Rents_info)
self.pushButton_20.clicked.connect(self.Update_Rents_info)
self.pushButton_34.clicked.connect(self.Add_elewater_info)
self.pushButton_33.clicked.connect(self.Update_elewater_info)
def Add_car_info(self):
car_number = self.lineEdit_12.text()
owner_company = self.lineEdit_10.text()
branch = self.lineEdit_8.text()
service_mode = self.comboBox.currentIndex()
shaceh_number = self.lineEdit_4.text()
motor_number = self.lineEdit_2.text()
fuel_type = self.comboBox_2.currentIndex()
car_type = self.lineEdit_11.text()
car_model = self.lineEdit_9.text()
car_load = self.lineEdit_7.text()
car_wight = self.lineEdit_5.text
car_shape = self.lineEdit_3.text()
car_color = self.lineEdit.text()
self.cur.execute('''INSERT INTO car_info(car_number, owner_company, branch, service_mode, shaceh_number, motor_number, fuel_type, car_type, car_model, car_load, car_weight, car_shape, car_color)'
VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)''' , (car_number,owner_company,branch,service_mode,shaceh_number,motor_number,fuel_type,car_type,car_model,car_load,car_wight,car_shape,car_color))
QApplication.processEvents()
print('Done')
def Update_car_info(self):
pass
def Delete_car_info(self):
pass
def Add_fuel_info(self):
pass
def Update_fuel_info(self):
pass
def Add_maintenance_info(self):
pass
def Update_maintenance_info(self):
pass
def Add_Licence_info(self):
pass
def Update_Licence_info(self):
pass
def Add_Revenus_info(self):
pass
def Update_Revenus_info(self):
pass
def Add_Rents_info(self):
pass
def Update_Rents_info(self):
pass
def Add_elewater_info(self):
pass
def Update_elewater_info(self):
pass
def main():
app = QApplication(sys.argv)
window = MainApp()
window.show()
app.exec_()
if __name__ == '__main__':
main()
thank you for your attantion and have a good day.

Save Nested Objects to File in Python3

How can I save this structure of Python objects into a file (preferably JSON)? And how can I load this structure from the file again?
class Nested(object):
def __init__(self, n):
self.name = "Nested Object: " + str(n)
self.state = 3.14159265359
class Nest(object):
def __init__(self):
self.x = 1
self.y = 2
self.objects = []
tree = []
tree.append(Nest())
tree.append(Nest())
tree.append(Nest())
tree[0].objects.append(Nested(1))
tree[0].objects.append(Nested(2))
tree[1].objects.append(Nested(1))
tree[2].objects.append(Nested(7))
tree[2].objects.append(Nested(8))
tree[2].objects.append(Nested(9))
Thanks to the reference to "pickle" I found a well working very simple solution to save my array of objects:
pickle
import pickle
pickle.dump( tree, open( "save.p", "wb" ) )
loaded_objects = pickle.load( open( "save.p", "rb" ) )
jsonpickle
import jsonpickle
frozen = jsonpickle.encode(tree)
with open("save.json", "w") as text_file:
print(frozen, file=text_file)
file = open("save.json", "r")
loaded_objects = jsonpickle.decode(file.read())
If you don't want pickle, nor want to use an external library you can always do it the hard way:
import json
class NestEncoder(json.JSONEncoder):
def default(self, obj):
entry = dict(obj.__dict__)
entry['__class__'] = obj.__class__.__name__
return entry
class NestDecoder(json.JSONDecoder):
def __init__(self):
json.JSONDecoder.__init__(self, object_hook=self.dict_to_object)
def dict_to_object(self, dictionary):
if dictionary.get("__class__") == "Nested":
obj = Nested.__new__(Nested)
elif dictionary.get("__class__") == "Nest":
obj = Nest.__new__(Nest)
else:
return dictionary
for key, value in dictionary.items():
if key != '__class__':
setattr(obj, key, value)
return obj
with open('nest.json', 'w') as file:
json.dump(tree, file, cls=NestEncoder)
with open('nest.json', 'r') as file:
tree2 = json.load(file, cls=NestDecoder)
print("Smoke test:")
print(tree[0].objects[0].name)
print(tree2[0].objects[0].name)
Assigning the the attributes to the classes doesn't have to be done dynamically with setattr() you can also do it manually.
There are probably plenty of pitfalls with doing it like this, so be careful.

Does IOError: [Errno 2] No such file or directory: mean the file hasn't been written?

I'm using Tweepy for the first time. Currently getting this error
---------------------------------------------------------------------------
IOError Traceback (most recent call last)
<ipython-input-11-cdd7ebe0c00f> in <module>()
----> 1 data_json = io.open('raw_tweets.json', mode='r', encoding='utf-8').read() #reads in the JSON file
2 data_python = json.loads(data_json)
3
4 csv_out = io.open('tweets_out_utf8.csv', mode='w', encoding='utf-8') #opens csv file
IOError: [Errno 2] No such file or directory: 'raw_tweets.json'
I've got a feeling that the code I've got isn't working. For example print(status) doesn't print anything. Also I see no saved CSV or JSON file in the directory.
I'm a newbie so any help/documentation you can offer would be great!
import time
from tweepy import Stream
from tweepy import OAuthHandler
from tweepy.streaming import StreamListener
import os
import json
import csv
import io
from pymongo import MongoClient
ckey = 'blah'
consumer_secret = 'blah'
access_token_key = 'blah'
access_token_secret = 'blah'
#start_time = time.time() #grabs the system time
keyword_list = ['keyword'] #track list
#Listener Class Override
class listener(StreamListener):
def __init__(self, start_time, time_limit=60):
self.time = start_time
self.limit = time_limit
self.tweet_data = []
def on_data(self, data):
saveFile = io.open('raw_tweets.json', 'a', encoding='utf-8')
while (time.time() - self.time) < self.limit:
try:
self.tweet_data.append(data)
return True
except BaseException, e:
print 'failed ondata,', str(e)
time.sleep(5)
pass
saveFile = io.open('raw_tweets.json', 'w', encoding='utf-8')
saveFile.write(u'[\n')
saveFile.write(','.join(self.tweet_data))
saveFile.write(u'\n]')
saveFile.close()
exit()
def on_error(self, status):
print status
class listener(StreamListener):
def __init__(self, start_time, time_limit=10):
self.time = start_time
self.limit = time_limit
def on_data(self, data):
while (time.time() - self.time) < self.limit:
print(data)
try:
client = MongoClient('blah', 27017)
db = client['blah']
collection = db['blah']
tweet = json.loads(data)
collection.insert(tweet)
return True
except BaseException as e:
print('failed ondata,')
print(str(e))
time.sleep(5)
pass
exit()
def on_error(self, status):
print(status)
data_json = io.open('raw_tweets.json', mode='r', encoding='utf-8').read() #reads in the JSON file
data_python = json.loads(data_json)
csv_out = io.open('tweets_out_utf8.csv', mode='w', encoding='utf-8') #opens csv file
UPDATED: Creates file but file is empty
import tweepy
import datetime
auth = tweepy.OAuthHandler('xxx', 'xxx')
auth.set_access_token('xxx', 'xxx')
class listener(tweepy.StreamListener):
def __init__(self, timeout, file_name, *args, **kwargs):
super(listener, self).__init__(*args, **kwargs)
self.start_time = None
self.timeout = timeout
self.file_name = file_name
self.tweet_data = []
def on_data(self, data):
if self.start_time is None:
self.start_time = datetime.datetime.now()
while (datetime.datetime.now() - self.start_time).seconds < self.timeout:
with open(self.file_name, 'a') as data_file:
data_file.write('\n')
data_file.write(data)
def on_error(self, status):
print status
l = listener(60, 'stack_raw_tweets.json')
mstream = tweepy.Stream(auth=auth, listener=l)
mstream.filter(track=['python'], async=True)
You are not creating a Stream for the listener. The last but one line of the code below does that. Followed by that you have to start the Stream, which is the last line. I must warn you that storing this in mongodb is the right thing to do as the file that I am storing it seems to grow easily to several GB. Also the file is not exactly a json. Each line in the file is a json. You must tweak it to your needs.
import tweepy
import datetime
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
class listener(tweepy.StreamListener):
def __init__(self, timeout, file_name, *args, **kwargs):
super(listener, self).__init__(*args, **kwargs)
self.start_time = None
self.timeout = timeout
self.file_name = file_name
self.tweet_data = []
def on_data(self, data):
if self.start_time is None:
self.start_time = datetime.datetime.now()
while (datetime.datetime.now() - self.start_time).seconds < self.timeout:
with open(self.file_name, 'a') as data_file:
data_file.write('\n')
data_file.write(data)
def on_error(self, status):
print status
l = listener(60, 'raw_tweets.json')
mstream = tweepy.Stream(auth=auth, listener=l)
mstream.filter(track=['python'], async=True)

Python 3 Accessing variable result from another Class

I have a little problem with a variable update.
I have my variable declared in my first function as such self.TestVar = 0
then if a certain count ==2 self.TestVar = 2
in a second function (in the same class) but called from within another class I want returning self.TestVar. no way.
AttributeError: 'ThndClass' object has no attribute 'TestVar'
I am most certainly not doing the good way, all I want is accessing self.TestVar = 2 from my other class that's it's but I can't find a proper way to do so in Python.
It looks like my issue is that I get my self.TestVar = 2 in a "if" statement which make it live in another scope (or I might be wrong).
import sys
from PIL import Image
from PyQt4 import QtCore, QtGui
class MainWindow(QtGui.QWidget):
def __init__(self):
super(MainWindow, self).__init__()
self.initUI()
def initUI(self):
self.TestVar = 0
self.TheCount = 2
if self.TheCount ==2:
self.TestVar = 2
ThndClass()
def Getit(self):
print("called correctly")
print(self.TestVar)
return self.TestVar
def main():
app = QtGui.QApplication([])
mw = MainWindow()
sys.exit(app.exec_())
class ThndClass(QtGui.QWidget):
def __init__(self):
super(ThndClass, self).__init__()
self.initUI2()
def initUI2(self):
print("Class Called")
print(MainWindow.Getit(self))
if __name__ == '__main__':
main()
If I remove the 2nd Class call :
import sys
from PIL import Image
from PyQt4 import QtCore, QtGui
class MainWindow(QtGui.QWidget):
def __init__(self):
super(MainWindow, self).__init__()
self.initUI()
def initUI(self):
self.TestVar = 0
self.TheCount = 2
if self.TheCount ==2:
self.TestVar = 2
self.Getit()
def Getit(self):
print("called correctly")
print(self.TestVar)
return self.TestVar
def main():
app = QtGui.QApplication([])
mw = MainWindow()
sys.exit(app.exec_())
if __name__ == '__main__':
main()
This works correctly, but I want to be able to call def Getit() from another class and get my result. Or simply get a way to directly access self.TestVar from my other class.
When you call
MainWindow.Getit(self)
in ThndClass.initUI2, you are treating MainWindow and ThndClass interchangeably, when they do not have the same attributes. Here is an actual minimal example:
class Parent():
def __init__(self):
pass
class Child1(Parent):
def __init__(self):
super().__init__()
self.foo = "foo"
def method(self):
print(type(self))
print(self.foo)
class Child2(Parent):
def __init__(self):
super().__init__()
self.bar = "bar"
c1 = Child1()
Child1.method(c1) # pass Child1 instance to Child1 instance method
c2 = Child2()
Child1.method(c2) # pass Child2 instance to Child1 instance method
and full output:
<class '__main__.Child1'> # gets a Child1 instance
foo # first call succeeds
<class '__main__.Child2'> # gets a Child2 instance (which doesn't have 'foo')
Traceback (most recent call last):
File "C:/Python34/so.py", line 25, in <module>
Child1.method(c2)
File "C:/Python34/so.py", line 11, in method
print(self.foo)
AttributeError: 'Child2' object has no attribute 'foo' # second call fails
However, as it is not clear what exactly the code is supposed to be doing, I can't suggest a fix. I don't know why you create but don't assign a ThndClass instance in MainWindow.initUI, for example.
Here is one possible fix; pass a Child1 instance to Child2.__init__, then use it either as an argument to Child2.method:
class Child2(Parent):
def __init__(self, c1): # provide Child1 instance as parameter
super().__init__()
self.bar = "bar"
self.method(c1) # pass instance to Child2.method
def method(self, c1):
c1.method() # call Child1.method with c1 as self parameter
(Note that c1.method() is equivalent to Child1.method(c1).)
or make it an instance attribute:
class Child2(Parent):
def __init__(self, c1): # provide Child1 instance as parameter
super().__init__()
self.bar = "bar"
self.c1 = c1 # make Child1 instance a Child2 instance attribute
self.method() # now no argument needed
def method(self):
self.c1.method() # call Child1.method with c1 as self parameter
(Note that self.c1.method() is equivalent to Child1.method(self.c1).)
In use (either way):
>>> c1 = Child1()
>>> c2 = Child2(c1)
<class '__main__.Child1'> # Child1.method gets a Child1 instance
foo # and is called successfully
Thank's to your help jonrsharpe here's my working code :)
import sys
from PIL import Image
from PyQt4 import QtCore, QtGui
class MainWindow(QtGui.QWidget):
def __init__(self):
super(MainWindow, self).__init__()
self.initUI()
def initUI(self):
self.TestVar = 0
self.TheCount = 2
if self.TheCount ==2:
self.TestVar = 2
Themain = self
ThndClass(Themain)
def Getit(self):
print("called correctly")
print(self.TestVar)
return self.TestVar
def main():
app = QtGui.QApplication([])
mw = MainWindow()
sys.exit(app.exec_())
class ThndClass(QtGui.QWidget):
def __init__(self, Themain):
super(ThndClass, self).__init__()
self.Themain = Themain
self.initUI2()
def initUI2(self):
print("Class Called")
print(self.Themain.Getit())
if __name__ == '__main__':
main()
All working good now : ) Thanks you very much !