I'm trying to set-up a TCP echo client and server that can exchange messages using the JSON format.
I took the code from the documentation and modified it as follows:
Edit: include fix and have both server and client send JSON style messages.
import asyncio
# https://docs.python.org/3/library/asyncio-stream.html
import json
async def handle_echo(reader, writer):
data = await reader.read(100)
message = json.loads(data.decode())
addr = writer.get_extra_info('peername')
print("Received %r from %r" % (message, addr))
print("Send: %r" % json.dumps(message)) # message
json_mess_en = json.dumps(message).encode()
writer.write(json_mess_en)
#writer.write(json_mess) # not wokring
#writer.write(json.dumps(json_mess)) # not working
# Yielding from drain() gives the opportunity for the loop to schedule the write operation
# and flush the buffer. It should especially be used when a possibly large amount of data
# is written to the transport, and the coroutine does not yield-from between calls to write().
#await writer.drain()
#print("Close the client socket")
writer.close()
loop = asyncio.get_event_loop()
coro = asyncio.start_server(handle_echo, '0.0.0.0', 9090, loop=loop)
server = loop.run_until_complete(coro)
# Serve requests until Ctrl+C is pressed
print('Serving on {}'.format(server.sockets[0].getsockname()))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
# Close the server
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
and the client code:
import asyncio
import json
async def tcp_echo_client(message, loop):
reader, writer = await asyncio.open_connection('0.0.0.0', 9090,
loop=loop)
print('Send: %r' % message)
writer.write(json.dumps(message).encode())
data = await reader.read(100)
data_json = json.loads(data.decode())
print('Received: %r' % data_json)
print(data_json['welcome'])
print('Close the socket')
writer.close()
message = {'welcome': 'Hello World!'}
loop = asyncio.get_event_loop()
loop.run_until_complete(tcp_echo_client(message, loop))
loop.close()
Error
TypeError: data argument must be a bytes-like object, not 'str'
Should I use another function than writer.write to encode for JSON? Or any suggestions?
Found the solution, replace:
writer.write(json.dumps(json_mess))
for
# encode as 'UTF8'
json_mess_en = json.dumps(json_mess).encode()
writer.write(json_mess_en)
Related
I am getting data using a TCP and trying it to publish it in std_msgs/Float64MultiArray format, however when I am trying to convert the json data to numpy array I am getting the following error
File "/usr/lib/python3.8/json/encoder.py", line 179, in default
raise TypeError(f'Object of type {o.__class__.__name__} '
TypeError: Object of type Int32MultiArray is not JSON serializable
Code for receiving data from tcp and sending it to rosbridge (topic chatter)
import roslibpy
import socket
import time
import struct
import numpy as np
import json
from rospy.numpy_msg import numpy_msg
from rospy_tutorials.msg import Floats
from std_msgs.msg import String,Int32,Int32MultiArray,MultiArrayLayout,MultiArrayDimension,Float64MultiArray
# ROS Python Bridge
client = roslibpy.Ros(host='localhost', port=9090) #same as rosbridge port
client.run()
print("Is ROS connected? ",client.is_connected)
talker = roslibpy.Topic(client, '/chatter', 'std_msgs/Float64MultiArray')
data_to_send = Float64MultiArray() # the data to be sent, initialise the array
HOST = "0.0.0.0" # Standard loopback interface address (localhost)
PORT = 8081 # Port to listen on (non-privileged ports are > 1023)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind((HOST, PORT))
while True:
s.listen()
conn, addr = s.accept()
with conn:
print(f"Connected by {addr}")
while client.is_connected:
data = conn.recv(1024)
if not data:
break
conn.sendall(data)
data = json.loads(data)
data_list = data[0]
x_val = data_list['x']
y_val = data_list['y']
z_val = data_list['z']
pos_arry = np.unique([x_val,y_val,z_val])
pos_arry = pos_arry.tolist()
my_array_for_publishing = Int32MultiArray(data=pos_arry)
print(type(pos_arry))
talker.publish(roslibpy.Message({'data': my_array_for_publishing}))
print('Sending message...')
talker.unadvertise()
client.terminate()
This error can be resolve by sending the data in list format.
pos_arry = np.unique([x_val,y_val,z_val])
pos_arry = pos_arry.tolist()
and retrieving the using .at
example
std_msgs::Float64MultiArray val = listener.data;
std::cout << val.data.at(0)<< std::endl;
Basic Information
I am creating a python script that can encrypt and decrypt a file with previous session data.
The Problem
I am able to decrypt my file and read it using a key. This returns a bytes string which I can in turn convert to a string. However, this string needs to be converted to a dictionary, which I cannot do. Using ast, json and eval I have run into errors.
Bytes string
decrypted = fernet.decrypt(encrypted)
String
string = decrypted.decode("UTF-8").replace("'", '"')
If I use eval() or ast.literal_eval() I get the following error:
Then I tried using json.loads() and I get the following error:
The information blocked out on both images is to protect my SSH connections. In the first image it is giving me a SyntaxError at the last digit of my ip address.
The Function
The function that is responsible for this when called looks like this:
def FileDecryption():
with open('enc_key.key', 'rb') as filekey:
key = filekey.read()
filekey.close()
fernet = Fernet(key)
with open('saved_data.txt', 'rb') as enc_file:
encrypted = enc_file.read()
enc_file.close()
decrypted = fernet.decrypt(encrypted)
print(decrypted)
string = decrypted.decode("UTF-8").replace("'", '"')
data = f'{string}'
print(data)
#data = eval(data)
data = json.loads(data)
print(type(data))
for key in data:
#command_string = ["load", data[key][1], data[key][2], data[key][3], data[key][4]]
#SSH.CreateSSH(command_string)
print(key)
Any help would be appreciated. Thanks!
Your data seems like it was written incorrectly in the first place, but without a complete example hard to say.
Here's a complete example that round-trips a JSON-able data object.
# requirement:
# pip install cryptography
from cryptography.fernet import Fernet
import json
def encrypt(data, data_filename, key_filename):
key = Fernet.generate_key()
with open(key_filename, 'wb') as file:
file.write(key)
fernet = Fernet(key)
encrypted = fernet.encrypt(json.dumps(data).encode())
with open(data_filename, 'wb') as file:
file.write(encrypted)
def decrypt(data_filename, key_filename):
with open(key_filename, 'rb') as file:
key = file.read()
fernet = Fernet(key)
with open(data_filename, 'rb') as file:
return json.loads(fernet.decrypt(file.read()))
data = {'key1': 'value1', 'key2': 'value2'}
encrypt(data, 'saved_data.txt', 'enc_key.key')
decrypted = decrypt('saved_data.txt', 'enc_key.key')
print(decrypted)
Output:
{'key1': 'value1', 'key2': 'value2'}
Is the EHLO message required after the TLS connection has been established? I'm using an acorn ltl-6511M wildlife camera that doesn't seem to send an EHLO message after establishing the TLS connection, causing a 503 error in my aiosmtpd-based SMTP server. It works with gmail SMTP though. Is the camera following the protocol or is my server not robust enough?
The code I'm using is:
import email
from email.header import decode_header
from email import message_from_bytes
from email.policy import default
from aiosmtpd.controller import Controller
from aiosmtpd.smtp import LoginPassword, AuthResult
import os
import sys
import time
import signal
import logging
import ssl
##setting timezone
os.environ['TZ'] = "Europe/London"
time.tzset()
def onExit( sig, func=None):
print("*************Stopping program*****************")
controller.stop()
exit()
signal.signal(signal.SIGTERM, onExit)
# removes the spaces and replaces with _ so they're valid folder names
def clean(text):
return "".join(c if c.isalnum() else "_" for c in text)
log = logging.getLogger('mail.log')
auth_db = {
b"TestCamera1#gmail.com": b"password1",
b"user2": b"password2",
b"TestCamera1": b"password1",
}
def authenticator_func(server, session, envelope, mechanism, auth_data):
#this deliberately lets everything through
assert isinstance(auth_data, LoginPassword)
username = auth_data.login
password = auth_data.password
return AuthResult(success=True)
def configure_logging():
file_handler = logging.FileHandler("aiosmtpd.log", "a")
stderr_handler = logging.StreamHandler(sys.stderr)
logger = logging.getLogger("mail.log")
fmt = "[%(asctime)s %(levelname)s] %(message)s"
datefmt = None
formatter = logging.Formatter(fmt, datefmt, "%")
stderr_handler.setFormatter(formatter)
logger.addHandler(stderr_handler)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel(logging.DEBUG)
class CustomHandler:
def handle_exception(self, error):
print("exception occured")
print(error)
return '542 Internal Server Error'
async def handle_DATA(self, server, session, envelope):
peer = session.peer
data = envelope.content # type: bytes
msg = message_from_bytes(envelope.content, policy=default)
# decode the email subject
print("Msg:{}".format(msg))
print("Data:{}".format(data))
print("All of the relevant data has been extracted from the email")
return '250 OK'
if __name__ == '__main__':
configure_logging()
handler = CustomHandler()
#update hostname to your IP
context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
context.load_cert_chain('cert.pem', 'key.pem')
controller = Controller(handler, hostname='0.0.0.0', port=587, authenticator=authenticator_func, auth_required=True,auth_require_tls=True,tls_context=context)
# Run the event loop in a separate thread.
controller.start()
while True:
time.sleep(10)
The code after trying to integrate is:
import email
from email.header import decode_header
from email import message_from_bytes
from email.policy import default
from aiosmtpd.controller import Controller
from aiosmtpd.smtp import LoginPassword, AuthResult, SMTP
import os
import json
import re
import sys
import time
import signal
import logging
import ssl
from datetime import datetime
import configparser
##setting timezone
os.environ['TZ'] = "Europe/London"
time.tzset()
spacer = "*"*100
def onExit( sig, func=None):
print("*************Stopping program*****************",3)
controller.stop()
exit()
signal.signal(signal.SIGTERM, onExit)
# removes the spaces and replaces with _ so they're valid folder names
def clean(text):
return "".join(c if c.isalnum() else "_" for c in text)
log = logging.getLogger('mail.log')
auth_db = {
b"TestCamera1#gmail.com": b"password1",
b"user2": b"password2",
b"TestCamera1": b"password1",
}
def authenticator_func(server, session, envelope, mechanism, auth_data):
# Simple auth - is only being used because of the reolink cam
assert isinstance(auth_data, LoginPassword)
username = auth_data.login
password = auth_data.password
log.warning("Authenticator is being used")
return AuthResult(success=True)
def configure_logging():
file_handler = logging.FileHandler("aiosmtpd.log", "a")
stderr_handler = logging.StreamHandler(sys.stderr)
logger = logging.getLogger("mail.log")
fmt = "[%(asctime)s %(levelname)s] %(message)s"
datefmt = None
formatter = logging.Formatter(fmt, datefmt, "%")
stderr_handler.setFormatter(formatter)
logger.addHandler(stderr_handler)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel(logging.DEBUG)
class SMTPNoEhloAfterStarttls(SMTP):
async def smtp_STARTTLS(self, arg: str):
print(spacer)
print("using starttls")
host_name = self.session.host_name
extended_smtp = self.session.extended_smtp
await super().smtp_STARTTLS(arg)
if host_name and extended_smtp and not self.session.host_name:
# There was an EHLO before the STARTTLS.
# RFC3207 says that we MUST reset the state
# and forget the EHLO, but unfortunately
# the client doesn't re-send the EHLO after STARTTLS,
# so we need to pretend as if an EHLO has been sent.
self.session.host_name = host_name
self.session.extended_smtp = True
class ControllerNoEhloAfterStarttls(Controller):
def factory(self):
print(spacer)
print("updating default settings")
return SMTPNoEhloAfterStarttls(self.handler, **self.SMTP_kwargs)
class CustomHandler:
def handle_exception(self, error):
print("exception occured",3)
print(error)
return '542 Internal Server Error'
async def handle_DATA(self, server, session, envelope):
peer = session.peer
data = envelope.content # type: bytes
msg = message_from_bytes(envelope.content, policy=default)
# decode the email subject
print("Msg:{}".format(msg),3)
print("Data:{}".format(data),3)
print("All of the relevant data has been extracted from the email",3)
print(spacer,3)
return '250 OK'
if __name__ == '__main__':
configure_logging()
handler = CustomHandler()
# controller = Controller(handler, hostname='10.200.68.132', port=587)
context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
context.load_cert_chain('cert.pem', 'key.pem')
controller = Controller(handler, hostname='10.200.68.133', port=587, authenticator=authenticator_func, auth_required=True,auth_require_tls=True,tls_context=context)
# Run the event loop in a separate thread.
controller.start()
#Confirmed that this is needed to keep the SMTP server running constantly
while True:
time.sleep(10)
However, this hasn't made any difference to the error logs.
Yes, EHLO is required after STARTTLS, see RFC3207 Section 4.2 (which specifically mentions forgetting the EHLO line - emphasis mine):
Upon completion of the TLS handshake, the SMTP protocol is reset to
the initial state (the state in SMTP after a server issues a 220
service ready greeting). The server MUST discard any knowledge
obtained from the client, such as the argument to the EHLO command,
which was not obtained from the TLS negotiation itself.
This means that unfortunately your camera is not following the SMTP protocol. It is also unfortunate that GMail SMTP does not follow the protocol (it doesn't require EHLO in-between STARTTLS and AUTH LOGIN).
aiosmtpd is quite insistent on following the SMTP protocol and duly forgets the EHLO data before the STARTTLS; the EHLO hostname is stored in self.session.host_name on the aiosmtpd.smtp.SMTP object, and self.session is reset in SMTP.connection_made(), which is invoked after STARTTLS.
It is possible to make aiosmtpd break the SMTP specification and act in a highly non-conforming way. Obviously this is something you MUST NOT do in production. Use the ControllerNoEhloAfterStarttls defined below instead of the standard aiosmtpd Controller and then it should work.
from aiosmtpd.smtp import SMTP
from aiosmtpd.controller import Controller
class SMTPNoEhloAfterStarttls(SMTP):
async def smtp_STARTTLS(self, arg: str):
host_name = self.session.host_name
extended_smtp = self.session.extended_smtp
await super().smtp_STARTTLS(arg)
if host_name and extended_smtp and not self.session.host_name:
# There was an EHLO before the STARTTLS.
# RFC3207 says that we MUST reset the state
# and forget the EHLO, but unfortunately
# the client doesn't re-send the EHLO after STARTTLS,
# so we need to pretend as if an EHLO has been sent.
self.session.host_name = host_name
self.session.extended_smtp = True
class ControllerNoEhloAfterStarttls(Controller):
def factory(self):
return SMTPNoEhloAfterStarttls(self.handler, **self.SMTP_kwargs)
...and then down in if __name__ == "__main__":, instantiate the custom controller class instead of the default Controller:
controller = ControllerNoEhloAfterStarttls(handler, hostname='10.200.68.133', port=587, ......)
I'm trying to use native-messaging in a firefox extension. I tried to build the example from this https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Native_messaging
I copy/pasted all the code and set it up as described on the page exactly and it sends a "Ping" but doesn't receive a "Pong" back. The browser console says TypeError: a bytes-like object is required, not 'str'on line 17 in the python application. What can I do?
I use windows 7 and python 3.x
The web-extension is sending a json object to the application, which then tests the length and struct.unpacks the message. If the message is "ping" it tries to struct.pack and json.dumps a response "pong" which the web-extension receives as a response. Both messages and any errors get written into the console.log.
it says in the example:
Note that running python with the -u flag is required on Windows,
in order to ensure that stdin and stdout are opened in binary, rather
than text, mode.
And I did set up the .bat starter script to include the -u flag, but it appears that python still reads the stdin as a string which then gives the TypeError while trying to struct.unpack.
web-extension background.js for sending ping:
/*
On startup, connect to the "ping_pong" app.
*/
var port = browser.runtime.connectNative("ping_pong");
/*
Listen for messages from the app.
*/
port.onMessage.addListener((response) => {
console.log("Received: " + response);
});
/*
On a click on the browser action, send the app a message.
*/
browser.browserAction.onClicked.addListener(() => {
console.log("Sending: ping");
port.postMessage("ping");
});
python application to receive ping and send pong:
#!/usr/bin/python -u
# Note that running python with the `-u` flag is required on Windows,
# in order to ensure that stdin and stdout are opened in binary, rather
# than text, mode.
import json
import sys
import struct
# Read a message from stdin and decode it.
def get_message():
raw_length = sys.stdin.read(4)
if not raw_length:
sys.exit(0)
message_length = struct.unpack('=I', raw_length)[0]
message = sys.stdin.read(message_length)
return json.loads(message)
# Encode a message for transmission, given its content.
def encode_message(message_content):
encoded_content = json.dumps(message_content)
encoded_length = struct.pack('=I', len(encoded_content))
return {'length': encoded_length, 'content': encoded_content}
# Send an encoded message to stdout.
def send_message(encoded_message):
sys.stdout.write(encoded_message['length'])
sys.stdout.write(encoded_message['content'])
sys.stdout.flush()
while True:
message = get_message()
if message == "ping":
send_message(encode_message("pong"))
This is the line which gives the TypeError:
message_length = struct.unpack('=I', raw_length)[0]
The log should say:
Sending: ping
Received: pong
The log actually says:
Sending: ping
stderr output from native app ping_pong: Traceback (most recent call last):
stderr output from native app ping_pong: File "C:\\Users\\ping_pong\\ping_pong.py", line 37, in <module>
stderr output from native app ping_pong: message = get_message()
stderr output from native app ping_pong: File "C:\\Users\\ping_pong\\ping_pong.py", line 17, in get_message
stderr output from native app ping_pong: message_length = struct.unpack('=I', raw_length)[0]
stderr output from native app ping_pong: TypeError: a bytes-like object is required, not 'str'
Using Python 2 it works fine for me (on Linux).
#!/usr/bin/python2 -u
or
path\to\python2
in the windows batch file, respectively.
To make it work in Python 3 you need to pack the encoded string into a struct (I don't know why - but hey it works):
#!/usr/bin/python -u
# Note that running python with the `-u` flag is required on Windows,
# in order to ensure that stdin and stdout are opened in binary, rather
# than text, mode.
import json
import sys
import struct
# Read a message from stdin and decode it.
def get_message():
# use buffer to get bytes
raw_length = sys.stdin.buffer.read(4)
#raise ValueError(raw_length)
if not raw_length:
sys.exit(0)
message_length = struct.unpack('=I', raw_length)[0]
message = sys.stdin.buffer.read(message_length).decode("utf-8")
return json.loads(message)
# Encode a message for transmission, given its content.
def encode_message(message_content):
encoded_content = json.dumps(message_content).encode("utf-8")
encoded_length = struct.pack('=I', len(encoded_content))
return {'length': encoded_length, 'content': struct.pack(str(len(encoded_content))+"s",encoded_content)}
# Send an encoded message to stdout.
def send_message(encoded_message):
#raise ValueError(encoded_message)
sys.stdout.buffer.write(encoded_message['length'])
sys.stdout.buffer.write(encoded_message['content'])
sys.stdout.buffer.flush()
while True:
message = get_message()
if message == "ping":
send_message(encode_message("pong"))
I am a newbie to python and am trying to create a script to login to crucible and use the token to pass to other services.
1) I am able to make a xml request and get a response but as soon as I pass the headers to my conn.request it says HTTP Error 415, unsupported Media Type.
I have done quiet a bit of research on this topic and found out that the rest API might not be supporting the json reques, but Crucible says that there API supports json so seems to be some other issue,
2) when trying to pass the args generated using feauth the auth token is not getting used , for now I have appended it to url and it works.
Please help me with the same , below is my script
import httplib
import urllib
import json
from xml.etree.ElementTree import XML
import xml.dom.minidom
conn = httplib.HTTPSConnection("fisheye")
args=urllib.urlencode({'userName':'UNAME', 'password':'PWD'})
headers={'content-type':'application/json', 'accept':'application/json'}
#headers={'Authorization' : 'Basic %s' % base64.b64encode("username:password")}
r1 = conn.request("post", "/rest-service/auth-v1/login", args)
#status = r1[u'headers']['status']
#conn.connect()
r2 = conn.getresponse()
print r1,r2.status,r2.reason,r2
r3=r2.read()
print(r3)
r4=str(r3)
print r4
data = XML(r4).find("token").text
print data
# data1=urllib.quote_plus(data, safe=":")
# print data1
args=urllib.urlencode({'FEAUTH':data}).replace("%3A", ":")
print "args is", args
#args={}
req = conn.request("get","/rest-service/reviews-v1")
r3 = conn.getresponse()
status = r3.status
print "the url is"#, r3.getheader('Location')
url=r3.getheader('location', '')
print url
url1=r3.msg#.dict['location']
print url1
#print req.url
#print req.get_method()
print dir(req) # list lots of other stuff in Request
print "after sending open review request"
print r3
print req,r3.status,r3.reason,r3
r4=r3.read()
print(r4)
r5=str(r4)
print r5
# json_ob=json.loads(r3.read())
# print json_ob
I was able to resolve the issue by
1) removing the Content-Type from the headers and changed the accept to Accept(sentence cased).
2) The login request was a get request and hence it supports data transfer by URL append, it is only for post request that we can pass an argument.
In the header of the request, try to specify the media type:
headers = { 'Content-Type' : 'application/json' }
req = urllib2.Request(url, headers=headers)