This is the route from where I am calling the the celery task in fastapi project
#router.get("/")
def read_bhav(
db: Session = Depends(deps.get_db),
skip: int = 0,
limit: int = 100,
current_user: models.User = Depends(deps.get_current_active_user),
) -> Any:
"""
Retrieve bhavs.
"""
y, d, m = (datetime.now().year, datetime.now().day - 3, datetime.now().month)
if date(y, m, d).weekday() < 5:
# celery_app.send_task("app.worker.bhav_copy", (date(y, m, d), db))
bhav_copy.delay(date(y, m, d), db)
print(db)
return {'TEST': "Successfully added"}
else:
return {"FAIL": "No Stock data for Weekends"}
Here is my celery_app.py file were I am configuring my celery app
from celery import Celery
celery_app = Celery("worker", broker="amqp://guest#queue//")
celery_app.conf["CELERY_TASK_SERIALIZER"] = "pickle"
celery_app.conf["CELERY_ACCEPT_CONTENT"] = ["pickle"]
celery_app.conf["CELERY_RESULT_SERIALIZER"] = "pickle"
celery_app.conf['CELERY_ROUTES'] = {"app.worker.test_celery": "main-queue"}
celery_app.conf['CELERY_ROUTES'] = {"app.worker.bhav_copy": "main-queue"}
You can see I have tried multiple ways to serialize this into json or something of sort
#celery_app.task(acks_late=True)
def bhav_copy(date: date, db) -> str:
prices = get_price_list(dt=date)
prices.columns = map(str.lower, prices.columns)
price_in = []
for i in range(1, prices.shape[0] + 1):
price = prices[i - 1:i]
v = price.values.reshape(13)
price_in.append(dict(zip(price.columns, v)))
crud.bhav.insert_bulk(db=db, bhav_in=price_in)
return f"inserted stocks data"
Would really appreciate a solution which would work to solve this issue, currently with this code I am getting the following error from below
backend_1 | File "/usr/local/lib/python3.7/site-packages/kombu/serialization.py", line 350, in pickle_dumps
backend_1 | return dumper(obj, protocol=pickle_protocol)
backend_1 | kombu.exceptions.EncodeError: Can't pickle <class 'sqlalchemy.orm.session.Session'>: it's not the same object as sqlalchemy.orm.session.Session
Related
I am having trouble in making things work with a Custom ParallelEnv I wrote by using PettingZoo. I am using SuperSuit's ss.pettingzoo_env_to_vec_env_v1(env) as a wrapper to Vectorize the environment and make it work with Stable-Baseline3 and documented here.
You can find attached a summary of the most relevant part of the code:
from typing import Optional
from gym import spaces
import random
import numpy as np
from pettingzoo import ParallelEnv
from pettingzoo.utils.conversions import parallel_wrapper_fn
import supersuit as ss
from gym.utils import EzPickle, seeding
def env(**kwargs):
env_ = parallel_env(**kwargs)
env_ = ss.pettingzoo_env_to_vec_env_v1(env_)
#env_ = ss.concat_vec_envs_v1(env_, 1)
return env_
petting_zoo = env
class parallel_env(ParallelEnv, EzPickle):
metadata = {'render_modes': ['ansi'], "name": "PlayerEnv-Multi-v0"}
def __init__(self, n_agents: int = 20, new_step_api: bool = True) -> None:
EzPickle.__init__(
self,
n_agents,
new_step_api
)
self._episode_ended = False
self.n_agents = n_agents
self.possible_agents = [
f"player_{idx}" for idx in range(n_agents)]
self.agents = self.possible_agents[:]
self.agent_name_mapping = dict(
zip(self.possible_agents, list(range(len(self.possible_agents))))
)
self.observation_spaces = spaces.Dict(
{agent: spaces.Box(shape=(len(self.agents),),
dtype=np.float64, low=0.0, high=1.0) for agent in self.possible_agents}
)
self.action_spaces = spaces.Dict(
{agent: spaces.Discrete(4) for agent in self.possible_agents}
)
self.current_step = 0
def seed(self, seed=None):
self.np_random, seed = seeding.np_random(seed)
def observation_space(self, agent):
return self.observation_spaces[agent]
def action_space(self, agent):
return self.action_spaces[agent]
def __calculate_observation(self, agent_id: int) -> np.ndarray:
return self.observation_space(agent_id).sample()
def __calculate_observations(self) -> np.ndarray:
observations = {
agent: self.__calculate_observation(
agent_id=agent)
for agent in self.agents
}
return observations
def observe(self, agent):
return self.__calculate_observation(agent_id=agent)
def step(self, actions):
if self._episode_ended:
return self.reset()
observations = self.__calculate_observations()
rewards = random.sample(range(100), self.n_agents)
self.current_step += 1
self._episode_ended = self.current_step >= 100
infos = {agent: {} for agent in self.agents}
dones = {agent: self._episode_ended for agent in self.agents}
rewards = {
self.agents[i]: rewards[i]
for i in range(len(self.agents))
}
if self._episode_ended:
self.agents = {} # To satisfy `set(par_env.agents) == live_agents`
return observations, rewards, dones, infos
def reset(self,
seed: Optional[int] = None,
return_info: bool = False,
options: Optional[dict] = None,):
self.agents = self.possible_agents[:]
self._episode_ended = False
self.current_step = 0
observations = self.__calculate_observations()
return observations
def render(self, mode="human"):
# TODO: IMPLEMENT
print("TO BE IMPLEMENTED")
def close(self):
pass
Unfortunately when I try to test with the following main procedure:
from stable_baselines3 import DQN, PPO
from stable_baselines3.common.env_checker import check_env
from dummy_env import dummy
from pettingzoo.test import parallel_api_test
if __name__ == '__main__':
# Testing the parallel algorithm alone
env_parallel = dummy.parallel_env()
parallel_api_test(env_parallel) # This works!
# Testing the environment with the wrapper
env = dummy.petting_zoo()
# ERROR: AssertionError: The observation returned by the `reset()` method does not match the given observation space
check_env(env)
# Model initialization
model = PPO("MlpPolicy", env, verbose=1)
# ERROR: ValueError: could not broadcast input array from shape (20,20) into shape (20,)
model.learn(total_timesteps=10_000)
I get the following error:
AssertionError: The observation returned by the `reset()` method does not match the given observation space
If I skip check_env() I get the following one:
ValueError: could not broadcast input array from shape (20,20) into shape (20,)
It seems like that ss.pettingzoo_env_to_vec_env_v1(env) is capable of splitting the parallel environment in multiple vectorized ones, but not for the reset() function.
Does anyone know how to fix this problem?
Plese find the Github Repository to reproduce the problem.
You should double check the reset() function in PettingZoo. It will return None instead of an observation like GYM
Thanks to discussion I had in the issue section of the SuperSuit repository, I am able to post the solution to the problem. Thanks to jjshoots!
First of all it is necessary to have the latest SuperSuit version. In order to get that I needed to install Stable-Baseline3 using the instructions here to make it work with gym 0.24+.
After that, taking the code in the question as example, it is necessary to substitute
def env(**kwargs):
env_ = parallel_env(**kwargs)
env_ = ss.pettingzoo_env_to_vec_env_v1(env_)
#env_ = ss.concat_vec_envs_v1(env_, 1)
return env_
with
def env(**kwargs):
env_ = parallel_env(**kwargs)
env_ = ss.pettingzoo_env_to_vec_env_v1(env_)
env_ = ss.concat_vec_envs_v1(env_, 1, base_class="stable_baselines3")
return env_
The outcomes are:
Outcome 1: leaving the line with check_env(env) I got an error AssertionError: Your environment must inherit from the gym.Env class cf https://github.com/openai/gym/blob/master/gym/core.py
Outcome 2: removing the line with check_env(env), the agent starts training successfully!
In the end, I think that the argument base_class="stable_baselines3" made the difference.
Only the small problem on check_env remains to be reported, but I think it can be considered as trivial if the training works.
While using the Routing solver of the google or-tools,a runtime error is thrown. There was a no changes made in the code segment,before and after getting this error. Previously, it was working. But recently after a DB connection modification was made, I am getting this error.
(Although, I doubt how a dB connection modification could affect the routing solver)
I am using the Azure Databricks notebook. As I am new to operations research, I have taken the example given in the https://developers.google.com/optimization/routing/pickup_delivery#complete_programs page, as my reference.
This is Vehicle Routing with Pick and Delivery problem.
from __future__ import print_function
from ortools.constraint_solver import routing_enums_pb2
from ortools.constraint_solver import pywrapcp
def create_data_model():
"""Stores the data for the problem."""
data = {}
data['distance_matrix'] = dist
data['pickups_deliveries'] = nodes_pickup_delivery
data['num_vehicles'] = 2
data['depot'] = 0
return data
solution_list = []
def print_solution(data, manager, routing, assignment):
"""Prints assignment on console."""
total_distance = 0
for vehicle_id in range(data['num_vehicles']):
index = routing.Start(vehicle_id)
plan_output = 'Route for vehicle {}:\n'.format(vehicle_id)
route_distance = 0
i = []
while not routing.IsEnd(index):
i.append(manager.IndexToNode(index))
plan_output += ' {} -> '.format(str(cityList[manager.IndexToNode(index)]))
previous_index = index
index = assignment.Value(routing.NextVar(index))
route_distance += routing.GetArcCostForVehicle(previous_index, index, vehicle_id)
solution_list.append(i)
plan_output += '{}\n'.format(str(cityList[manager.IndexToNode(index)]))
plan_output += 'Distance of the route: {} miles\n'.format(route_distance)
#print(plan_output)
total_distance += route_distance
#print('Total Distance of all routes: {} miles'.format(total_distance))
def main():
"""Entry point of the program."""
# Instantiate the data problem.
data = create_data_model()
# Create the routing index manager.
manager = pywrapcp.RoutingIndexManager(len(data['distance_matrix']), data['num_vehicles'], data['depot'])
# Create Routing Model.
routing = pywrapcp.RoutingModel(manager)
# Define cost of each arc.
def distance_callback(from_index, to_index):
"""Returns the manhattan distance between the two nodes."""
# Convert from routing variable Index to distance matrix NodeIndex.
from_node = manager.IndexToNode(from_index)
to_node = manager.IndexToNode(to_index)
return data['distance_matrix'][from_node][to_node]
transit_callback_index = routing.RegisterTransitCallback(distance_callback)
routing.SetArcCostEvaluatorOfAllVehicles(transit_callback_index)
# Add Distance constraint.
dimension_name = 'Distance'
routing.AddDimension(
transit_callback_index,
0, # no slack
40, # vehicle maximum travel distance
True, # start cumul to zero
dimension_name)
distance_dimension = routing.GetDimensionOrDie(dimension_name)
distance_dimension.SetGlobalSpanCostCoefficient(100)
# Define Transportation Requests.
for request in data['pickups_deliveries']:
pickup_index = manager.NodeToIndex(request[0])
delivery_index = manager.NodeToIndex(request[1])
routing.AddPickupAndDelivery(pickup_index, delivery_index)
routing.solver().Add(routing.VehicleVar(pickup_index) == routing.VehicleVar(delivery_index))
routing.solver().Add(distance_dimension.CumulVar(pickup_index) <= distance_dimension.CumulVar(delivery_index))
# Setting first solution heuristic.
search_parameters = pywrapcp.DefaultRoutingSearchParameters()
#search_parameters.time_limit.seconds = 90
search_parameters.first_solution_strategy = (routing_enums_pb2.FirstSolutionStrategy.PARALLEL_CHEAPEST_INSERTION)
# Solve the problem.
assignment = routing.SolveWithParameters(search_parameters)
# Print solution on console.
if assignment:
print_solution(data, manager, routing, assignment)
if __name__ == '__main__':
main()
The error I am getting is pointing to the following code segment: 'plan_output = 'Route for vehicle {}:\n'.format(vehicle_id)'
The error thrown is:
RuntimeError: SWIG std::function invocation failed.
RuntimeErrorTraceback (most recent call last)
<command-2714173895177597> in <module>()
89
90 if __name__ == '__main__':
---> 91 main()
<command-2714173895177597> in main()
85 # Print solution on console.
86 if assignment:
---> 87 print_solution(data, manager, routing, assignment)
88
89
<command-2714173895177597> in print_solution(data, manager, routing, assignment)
18 for vehicle_id in range(data['num_vehicles']):
19 index = routing.Start(vehicle_id)
---> 20 plan_output = 'Route for vehicle {}:\n'.format(vehicle_id)
21 route_distance = 0
22 i = []
RuntimeError: SWIG std::function invocation failed.
Kindly help.
def create_data_model():
"""Stores the data for the problem."""
data = {}
data['distance_matrix'] = dist
data['pickups_deliveries'] = nodes_pickup_delivery
data['num_vehicles'] = 2
data['depot'] = 0 #Dummy location
return data
solution_list = []
def print_solution(data, manager, routing, assignment):
"""Prints assignment on console."""
total_distance = 0
for vehicle_id in range(data['num_vehicles']):
index = routing.Start(vehicle_id)
plan_output = 'Route for vehicle {}:\n'.format(vehicle_id)
route_distance = 0
i = []
while not routing.IsEnd(index):
i.append(manager.IndexToNode(index))
plan_output += ' {} -> '.format(manager.IndexToNode(index))
previous_index = index
index = assignment.Value(routing.NextVar(index))
route_distance += routing.GetArcCostForVehicle(previous_index, index, vehicle_id)
solution_list.append(i)
plan_output += '{}({})\n'.format(str(cityList[manager.IndexToNode(index)]))
plan_output += 'Distance of the route: {} miles\n'.format(route_distance)
print(plan_output)
total_distance += route_distance
print('Total Distance of all routes: {} miles'.format(total_distance))
So i have a program which will encrypt a string using AES and generate cipher which in bytes[].
I wish to store this cipher as it is in mysql database.
I found we could use VARBINARY data type in mysql to do so.
In what ways we could achieve so.
Here is my try to do so :
import ast
import mysql.connector
from Crypto.Cipher import AES
from Crypto.Random import get_random_bytes
def encrypt(key, msg):
iv = get_random_bytes(16)
cipher = AES.new(key, AES.MODE_CFB, iv)
ciphertext = cipher.encrypt(msg) # Use the right method here
db = iv + ciphertext
print(db)
cursor.executemany(sql_para_query,db)
print(cursor.fetchone())
connection.commit()
return iv + ciphertext
def decrypt(key, ciphertext):
iv = ciphertext[:16]
ciphertext = ciphertext[16:]
cipher = AES.new(key, AES.MODE_CFB, iv)
msg = cipher.decrypt(ciphertext)
return msg.decode("utf-8")
if __name__ == "__main__":
connection = mysql.connector.connect(host = "localhost", database = "test_db", user = "sann", password = "userpass",use_pure=True)
cursor = connection.cursor(prepared = True)
sql_para_query = """insert into test1 values(UNHEX(%s)) """
ed = input("(e)ncrypt or (d)ecrypt: ")
key = str(1234567899876543)
if ed == "e":
msg = input("message: ")
s= encrypt(key, msg)
print("Encrypted message: ", s)
file = open("e_tmp","wb+")
file.write(s)
print(type(s))
elif ed == "d":
#smsg = input("encrypted message: ")
#file = open("e_tmp","rb")
#smsg = file.read()
#print(type(smsg))
sql_para_query = """select * from test1"""
cursor.execute(sql_para_query)
row = cursor.fetchone()
print(row)
#smsg = str(smsg)
#msg = ast.literal_eval(smsg)
#print(msg)
#print(type(msg))
#s=decrypt(key, msg)
#print("Decrypted message: ", s)
#print(type(s))
Error I'm getting :
Traceback (most recent call last): File
"/home/mr_pool/.local/lib/python3.6/site-packages/mysql/connector/cursor.py",
line 1233, in executemany
self.execute(operation, params) File "/home/mr_pool/.local/lib/python3.6/site-packages/mysql/connector/cursor.py",
line 1207, in execute
elif len(self._prepared['parameters']) != len(params): TypeError: object of type 'int' has no len()
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File "tmp1.py", line 36, in
s= encrypt(key, msg) File "tmp1.py", line 14, in encrypt
cursor.executemany(sql_para_query,db) File "/home/mr_pool/.local/lib/python3.6/site-packages/mysql/connector/cursor.py",
line 1239, in executemany
"Failed executing the operation; {error}".format(error=err)) mysql.connector.errors.InterfaceError: Failed executing the operation;
object of type 'int' has no len()
Any other alternatives are also welcome.
My ultimate goal is to store the encrypted text in database.
I reproduced your error, but it seems there are more errors in your code.
The key as well as the message are strings, therefore I got this error:
TypeError: Object type <class 'str'> cannot be passed to C code
Which I fixed by encoding them in utf-8:
# line 38:
key = str(1234567899876543).encode("utf8")
# .... line 41:
s= encrypt(key, msg.encode("utf8"))
The UNHEX function in your SQL Query is not needed because we are entering the data as VARBINARY. You can change your statement to:
"""insert into test1 values(%s) """
The function executemany() can be replaced by execute() because you are only entering one statement. However I will write the solution for using both, execute or executemany.
insert with execute():
From the documentation:
cursor.execute(operation, params=None, multi=False)
iterator = cursor.execute(operation, params=None, multi=True)
This method executes the given database operation (query or command). The parameters found in the tuple or dictionary params are bound to the variables in the operation. Specify variables using %s or %(name)s parameter style (that is, using format or pyformat style). execute() returns an iterator if multi is True.
https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlcursor-execute.html
So we need just to build a tuple with your parameters by changing the cursor.execute line to:
cursor.execute(sql_para_query, (db, ))
insert with executemany():
From the documentation:
cursor.executemany(operation, seq_of_params)
This method prepares a database operation (query or command) and executes it against all parameter sequences or mappings found in the sequence seq_of_params.
https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlcursor-executemany.html
Therefore we need to build a sequence with values you'd like to insert. In your case just one value:
cursor.executemany(sql_para_query, [(db, )])
To insert multiple values, you can add as many tuples into your sequence as you want.
full code:
import ast
import mysql.connector
from Crypto.Cipher import AES
from Crypto.Random import get_random_bytes
def encrypt(key, msg):
iv = get_random_bytes(16)
cipher = AES.new(key, AES.MODE_CFB, iv)
ciphertext = cipher.encrypt(msg) # Use the right method here
db = iv + ciphertext
cursor.execute(sql_para_query, (db, ))
connection.commit()
return iv + ciphertext
def decrypt(key, ciphertext):
iv = ciphertext[:16]
ciphertext = ciphertext[16:]
cipher = AES.new(key, AES.MODE_CFB, iv)
msg = cipher.decrypt(ciphertext)
return msg.decode("utf-8")
if __name__ == "__main__":
connection = mysql.connector.connect(host = "localhost", database = "test_db", user = "sann", password = "userpass",use_pure=True)
cursor = connection.cursor(prepared = True)
sql_para_query = """insert into test1 values(%s) """
ed = input("(e)ncrypt or (d)ecrypt: ")
key = str(1234567899876543).encode("utf8")
if ed == "e":
msg = input("message: ")
s= encrypt(key, msg.encode("utf8"))
print("Encrypted message: ", s)
file = open("e_tmp","wb+")
file.write(s)
print(type(s))
elif ed == "d":
sql_para_query = """select * from test1"""
cursor.execute(sql_para_query)
row = cursor.fetchone()
msg = row[0] # row is a tuple, therefore get first element of it
print("Unencrypted message: ", msg)
s=decrypt(key, msg)
print("Decrypted message: ", s)
output:
#encrypt:
(e)ncrypt or (d)ecrypt: e
message: this is my test message !!
Encrypted message: b"\x8f\xdd\xe6f\xb1\x8e\xb51\xc1'\x9d\xbf\xb5\xe1\xc7\x87\x99\x0e\xd4\xb2\x06;g\x85\xc4\xc1\xd2\x07\xb5\xc53x\xb9\xbc\x03+\xa2\x95\r4\xd1*"
<class 'bytes'>
#decrypt:
(e)ncrypt or (d)ecrypt: d
Unencrypted message: bytearray(b"\x8f\xdd\xe6f\xb1\x8e\xb51\xc1\'\x9d\xbf\xb5\xe1\xc7\x87\x99\x0e\xd4\xb2\x06;g\x85\xc4\xc1\xd2\x07\xb5\xc53x\xb9\xbc\x03+\xa2\x95\r4\xd1*")
Decrypted message: this is my test message !!
I have a table called "KLIJENTI" in MySQL database on ampps localhost server. Table consists of 4 columns: ID_KLIJENTA - int, IME_KLIJENTA - string, PREZIME_KLIJENTA - string and ADRESA_KLIJENTA - string.
I followed tutorials on how to use slick to insert into and read from database, but it does not do what I want.
Here is my application.conf:
scalaTest = {
url = "jdbc:mysql://localhost/Scala1"
user = "root"
password = "mysql"
driver = com.mysql.jdbc.Driver
connectionPool = disabled
keepAliveConnection = true
logStatements = true
}
build.sbt:
name := """project4"""
mainClass in Compile := Some("HelloSlick")
libraryDependencies ++= List(
"com.typesafe.slick" %% "slick" % "3.1.0-RC2",
"org.slf4j" % "slf4j-nop" % "1.7.10",
"com.h2database" % "h2" % "1.4.187",
"org.scalatest" %% "scalatest" % "2.2.4" % "test" ,
"mysql" % "mysql-connector-java" % "5.1.28"
)
fork in run := true
And here is my helloSlick.scala:
import scala.concurrent.{Future, Await}
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.Duration
import slick.backend.DatabasePublisher
import slick.driver.MySQLDriver.api._
case class Klijent(ID_KLIJENTA: Option[Int] = None, IME_KLIJENTA: String, PREZIME_KLIJENTA: String, ADRESA_KLIJENTA: String)
class Klijenti(tag: Tag) extends Table[Klijent](tag, "KLIJENTI") {
def ID_KLIJENTA = column[Option[Int]]("ID_KLIJENTA", O.PrimaryKey, O.AutoInc)
def IME_KLIJENTA = column[String]("IME_KLIJENTA")
def PREZIME_KLIJENTA = column[String]("PREZIME_KLIJENTA")
def ADRESA_KLIJENTA = column[String]("ADRESA_KLIJENTA")
def * = (ID_KLIJENTA, IME_KLIJENTA, PREZIME_KLIJENTA , ADRESA_KLIJENTA) <>((Klijent.apply _).tupled, Klijent.unapply)
}
// The main application
object HelloSlick extends App {
val db = Database.forConfig("scalaTest")
val mojiKlijenti = TableQuery[Klijenti]
val q1 = sql"select IME_KLIJENTA from KLIJENTI WHERE ID==1 ".as[String]
val q2 = mojiKlijenti.filter(_.ID_KLIJENTA === 0)
println(q1)
println(q2)
Before I run this code, I already inserted one row into the database through phpmyadmin, so both values q1 and q2 should print out real values but instead of that, I get this:
background log: info: Running HelloSlick
background log: info: slick.jdbc.SQLActionBuilder$$anon$1#56300388
background log: info: Rep(Filter #1636634026)
My apache webserver and mysql are turned on, just in case someone asks..
Why am I not getting the real values?
EDIT:> Later on, I tried to execute following command mojiKlijenti += Klijent(Some(5),"Name","Surname","Address") and again, it compiler successfully but when I check my database in phpmyadmin, no record is added..
You are not getting the real values because you need to convert the Query into an Action by calling its result method and execute it.
val action = q2.result
val results = db.run( action)
results.foreach( println )
i have written a code to read registers of a modbus communication protocol. i have attached the code below as well.. am able to overcome the i/o errors by exception handling method where as the value error that i get , am not able throw that error and move on.
Basically what i am doing is am reading the data in the registers and sending up to the server. but my requirement is i have to read the values every second and for 24 hours. so i need to build a robust system that will overcome these value errors and continue executing the threads i have created.
the code to read registers is given below :
import minimalmodbus
import serial
from ctypes import *
import struct
import time
minimalmodbus.BAUDRATE = 9600
minimalmodbus.PARITY = serial.PARITY_NONE
minimalmodbus.BYTESIZE = 8
minimalmodbus.TIMEOUT=5
minimalmodbus.CLOSE_PORT_AFTER_EACH_CALL = True
energy_meter = serial.Serial("/dev/ttyUSB0", baudrate=9600,
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_ONE, bytesize=serial.EIGHTBITS, timeout=5)
energy_meter = minimalmodbus.Instrument('/dev/ttyUSB0', 2, mode='rtu')
#energy_meter.debug = True
def convert_in_float(value1, value2):
raw = struct.pack('>HH',value1,value2)
ans = struct.unpack('>f', raw)[0]
return ans
def sdm630():
parameter_list1 = [ 0 ] * 0x12
parameter_list2 = [ 0 ] * 3
parameter_list3 = [ 0 ] * 6
#print energy_meter
error = 0
try:
index = 0
read_values1 = energy_meter.read_registers( 0 , 0x24, 4)
for i in range ( 0, 0x24, 2):
parameter_list1[index] = convert_in_float( read_values1[i], read_values1[i+1])
#print "Parameter read from register : ", hex(index), "is : ", parameter_list1[index] ,"\n"
index = index + 1
#read parameter list 2 & 3 in a similar way
error = 0
return error, parameter_list1, parameter_list2, parameter_list3, int(time.time())
except IOError or ValueError:
print "got error"
error = 1
return error, parameter_list1, parameter_list2, parameter_list3, int(time.time())
also, i have written a separate code to dump all data to server and is shown below :
import time
from pymongo import MongoClient
client = MongoClient('mongodb://10.32.36.40:27017')
db = client.clytics
collection = db['raspberry_pi']
def pushData(error, value1, value2, value3, value4):
if error == 0 :
temp_js = {
#variable assignment
}
temp_js_id = collection.insert(temp_js)
using the above two codes i have created threads for each function. and i only execute this code and after 20 minutes of execution , i get value errors and the program doesnt execute anymore. the main program is given below :
import time
from threading import Thread
from threading import Timer
from Queue import Queue
from modbus import sdm630
from dumpInDB import pushData
from processData import process_the_data
DELAY_SEC = 1
DELAY_MIN = 60
LOOP_LIMIT = 60
def getData(q):
error, parameter_list1, parameter_list2 , parameter_list3, parameter_list4= sdm630()
print "In getData - data:", parameter_list1, parameter_list2
q.put([error, parameter_list1, parameter_list2, parameter_list3, parameter_list4])
def processData(q1,q2):
sec_data = q1.get()
min_data = process_the_data(sec_data)
print "In processData - data:", sec_data, min_data
q2.put(min_data)
print "queue:", q2.qsize()
def putData(q):
#print "In putData - data:", value[0], value[1], value[2]
for i in range(0, q.qsize()):
value = q.get()
print "In putData - data:", value[0], value[1], value[2], value[3]
pushData( value[0], value[1] , value[2], value[3] , value[4])
def thread1(threadName, q):
i = 0
while 1:
t = Timer( DELAY_SEC, getData, args = (q,))
t.start()
time.sleep(DELAY_SEC)
def thread2( threadName, q1,q2):
i = 0
print "in thread2"
while 1:
t = Timer( DELAY_SEC, processData, args = (q1,q2,))
t.start()
time.sleep(DELAY_SEC)
def thread3( threadName, q):
i = 0
print "in thread3"
while 1:
t = Timer( DELAY_MIN, putData, args = (q,))
t.start()
print "schedule time - min"
time.sleep(DELAY_MIN)
queue_second = Queue()
queue_minute = Queue()
thread1 = Thread( target=thread1, args=("Thread-1", queue_second) )
thread2 = Thread( target=thread2, args=("Thread-2", queue_second, queue_minute) )
thread3 = Thread( target=thread3, args=("Thread-3", queue_minute) )
thread1.start()
thread2.start()
thread3.start()
thread1.join()
thread2.join()
thread3.join()
am stuck with this error. shown below :
minimalmodbus.Instrument<id=0xb6b2d9b8, address=2, mode=rtu, close_port_after_each_call=True, precalculate_read_size=True, debug=False, serial=Serial<id=0xb6b482f0, open=False>(port='/dev/ttyUSB0', baudrate=9600, bytesize=8, parity='N', stopbits=1, timeout=5, xonxoff=False, rtscts=False, dsrdtr=False)>
Traceback (most recent call last):
File "topScript.py", line 7, in <module>
from modbus import sdm630
File "/home/pi/scripts/modbus.py", line 60, in <module>
sdm630()
File "/home/pi/scripts/modbus.py", line 32, in sdm630
read_values1 = energy_meter.read_registers( 0 , 0x24, 4)
File "/usr/local/lib/python2.7/dist-packages/minimalmodbus.py", line 498, in read_registers
numberOfRegisters=numberOfRegisters, payloadformat='registers')
File "/usr/local/lib/python2.7/dist-packages/minimalmodbus.py", line 697, in _genericCommand
payloadFromSlave = self._performCommand(functioncode, payloadToSlave)
File "/usr/local/lib/python2.7/dist-packages/minimalmodbus.py", line 798, in _performCommand
payloadFromSlave = _extractPayload(response, self.address, self.mode, functioncode)
File "/usr/local/lib/python2.7/dist-packages/minimalmodbus.py", line 1075, in _extractPayload
raise ValueError(text)
ValueError: Checksum error in rtu mode: '\xa6\xe6' instead of '\xf7[' . The response is: '\xff\xf7HCeN\xce\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00?\x80\x00\x00?\x80\x00\x00?\x80\x00\x00\xa6\xe6' (plain response: '\xff\xf7HCeN\xce\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
sometimes value error keeps popping up for sometime and finally fails and gives a message saying no more threads can be created.(reached to maximum level)
Your syntax to handle multiple exceptions is wrong. Use something like:
except (ValueError, IOError):
For more details see the Python tutorial https://docs.python.org/2/tutorial/errors.html