Trying to handle a Py4JJavaError that is raised when entering invalid sql code to spark.sql(). My function is as follows:
import py4j
def sql_to_df_and_create_view(sql_string: str, view_name: str):
''' Takes in a SQL command in string format and creates a spark df from the command.
Secondly, creates a temporary view with name specified under view_name parameter. Ensures that
the SQL code is valid and that the view is created. '''
try:
df = spark.sql(sql_string)
except py4j.protocol.Py4JJavaError:
raise Exception(f'Invalid SQL code passed in by {sql_string}.')
return
df.createOrReplaceTempView(view_name)
return df
With the error code being:
org.apache.spark.sql.catalyst.parser.ParseException:
---------------------------------------------------------------------------
Py4JJavaError Traceback (most recent call last)
/databricks/spark/python/pyspark/sql/utils.py in deco(*a, **kw)
62 try:
---> 63 return f(*a, **kw)
64 except py4j.protocol.Py4JJavaError as e:
/databricks/spark/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py in get_return_value(answer,
gateway_client, target_id, name)
327 "An error occurred while calling {0}{1}{2}.\n".
--> 328 format(target_id, ".", name), value)
329 else:
Py4JJavaError: An error occurred while calling o213.sql.
: org.apache.spark.sql.catalyst.parser.ParseException:
mismatched input 'sd' expecting {'(', 'SELECT', 'FROM', 'ADD', 'DESC', 'WITH', 'VALUES',
'CREATE', 'TABLE', 'INSERT', 'DELETE', 'DESCRIBE', 'EXPLAIN', 'SHOW', 'USE', 'DROP', 'ALTER',
'MAP', 'SET', 'RESET', 'START', 'COMMIT', 'ROLLBACK', 'MERGE', 'UPDATE', 'CONVERT', 'REDUCE',
'REFRESH', 'CLEAR', 'CACHE', 'UNCACHE', 'DFS', 'TRUNCATE', 'ANALYZE', 'LIST', 'REVOKE',
'GRANT', 'LOCK', 'UNLOCK', 'MSCK', 'EXPORT', 'IMPORT', 'LOAD', 'OPTIMIZE', 'COPY'}(line 1, pos
0)
== SQL ==
sd
^^^
When I run the function with invalid sql in the sql_string argument, the error is not handled and it still raises the same error code rather than 'Exception(f'Invalid SQL code passed in by {sql_string}.')'. If anyone can figure out why this is not being handled correctly I would be v grateful :)
Related
I am using boto3 with Python 3.6 to start a Step Function execution. The Step Function is designed to share my base AMI across all my accounts. I have 4 variables I need to pass to the input parameter to kick off the execution. These are the AMI ID, the account list of accounts I own, the source account, and the KMS key. The AMI ID and the account list are constructed in my code and are the variables that need to get passed dynamically. According to the documentation, input is a string that contains the JSON input data for the execution and gives the following example: "input": "{\"ami_id\" : \"ami_id\"}". My question is how do I pass the variables in question to this parameter as the value? My code is below with the traceback:
CODE:
import boto3
import json
# Get an STS token to assume roles into AWS accounts
def get_sts_token(**kwargs):
role_arn = kwargs['RoleArn']
region_name = kwargs['RegionName']
sts = boto3.client(
'sts',
region_name=region_name,
)
token = sts.assume_role(
RoleArn=role_arn,
RoleSessionName='GetInstances',
DurationSeconds=900,
)
return token["Credentials"]
def get_accounts():
role_arn = "arn:aws:iam::xxxxxxxxxx:role/list-accounts-role"
region_name = "us-east-1"
token = get_sts_token(RoleArn=role_arn, RegionName=region_name)
access_key = token['AccessKeyId']
secret_key = token['SecretAccessKey']
session_token = token['SessionToken']
client = boto3.client('organizations',
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
aws_session_token=session_token)
moreAccounts=True
nexttoken=''
global accountList
accountList =[]
while moreAccounts:
if (len(nexttoken)>0):
accounts=client.list_accounts(NextToken=nexttoken)
else:
accounts=client.list_accounts()
if 'NextToken' in accounts:
nexttoken=accounts['NextToken']
else:
moreAccounts=False
for account in accounts['Accounts']:
if account['Status'] != 'SUSPENDED' and account['Status'] != 'CLOSED' :
account_id = account['Id']
accountList.append(account_id)
print(accountList)
def trigger_sfn():
ssm = boto3.client('ssm')
role_arn = "arn:aws:iam::xxxxxxxx:role/execute-sfn"
region_name = "us-east-1"
ami_id = ssm.get_parameter(Name='/BaseAMI/newest')['Parameter']['Value']
print(ami_id)
token = get_sts_token(RoleArn=role_arn, RegionName=region_name)
print(token)
access_key = token['AccessKeyId']
secret_key = token['SecretAccessKey']
session_token = token['SessionToken']
sfn = boto3.client('stepfunctions',
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
aws_session_token=session_token)
response = sfn.start_execution(
stateMachineArn='arn:aws:states:us-east-1:xxxxxxxx:stateMachine:ami-share',
input="{\"ami_id\": ami_id, \"source_account_id\": \"112233445566\", \"accountList\": accountList, \"kms_key_arn\": \"alias/aws/ebs\"}"
)
print(response)
TRACE:
An error occurred (InvalidExecutionInput) when calling the
StartExecution operation: Invalid State Machine Execution Input:
'com.fasterxml.jackson.core.JsonParseException: Unrecognized token
'ami_id': was expecting ('true', 'false' or 'null')
at [Source: (String)"{"ami_id": ami_id, "source_account_id":
"112233445566", "accountList": accountList, "kms_key_arn":
"alias/aws/ebs"}"; line: 1, column: 18]': InvalidExecutionInput
Traceback (most recent call last):
File "/var/task/lambda_function.py", line 6, in lambda_handler
trigger_sfn()
File "/var/task/lambda_function.py", line 96, in trigger_sfn
input="{\"ami_id\": ami_id, \"source_account_id\": \"112233445566\",
\"accountList\": accountList, \"kms_key_arn\": \"alias/aws/ebs\"}"
File "/var/runtime/botocore/client.py", line 314, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/var/runtime/botocore/client.py", line 612, in _make_api_call
raise error_class(parsed_response, operation_name)
botocore.errorfactory.InvalidExecutionInput: An error occurred
(InvalidExecutionInput) when calling the StartExecution operation:
Invalid State Machine Execution Input:
'com.fasterxml.jackson.core.JsonParseException: Unrecognized token
'ami_id': was expecting ('true', 'false' or 'null')
at [Source: (String)"{"ami_id": ami_id, "source_account_id":
"112233445566", "accountList": accountList, "kms_key_arn":
"alias/aws/ebs"}"; line: 1, column: 18]'
You can add the value of the variable to the string like this:
input="{\"ami_id\": \"" + ami_id + "\", \"source_account_id\": \"112233445566\", \"accountList\": accountList, \"kms_key_arn\": \"alias/aws/ebs\"}"
Your code currently sends the literal string "ami_id", instead of the variable value of ami_id
I'm really trying hard to execute an INSERT query by passing named placeholders and it doesn't work. This is the error I get: execute() got an unexpected keyword argument 'vid'
Here is my code:
query = """INSERT INTO `index`
(`video_id`, `label`, `img`, `context`, `genre`, `kind`) VALUES
({vid}, {label}, {img}, {context}, {genre}, {kind}) ON DUPLICATE KEY UPDATE
label={label}, img={img}, context={context}, genre={genre}, kind={kind}
"""
self.db.insert(query, {
'vid': video_id,
'label': label,
'img': img_source,
'context': json.dumps(context),
'genre': slugify(self.genre),
'kind': self.kind
})
in my db handler:
def insert(self, query, params):
print(query.format(**params))
try:
self.cursor.execute(query, **params)
self.connection.commit()
except Exception as e:
print(e)
self.connection.rollback()
quit()
I have a python script which everytime a wait_for_page call is made it writes the time it took to wait for the page to a database. The query is below:
conn = mysql.connector.connect(**config)
connect = conn.cursor()
params = {'build': self.tc.tag, 'page': unicode(self), 'object_id': self.object_id, 'page_header':
self.page_header, 'interval': t.interval, 'timestamp': timestamp}
query = u'INSERT INTO page_load_times (build, page, object_id, page_header, elapsed_time, date_run) ' \
'VALUES (%(build)s, %(page)s, %(object_id)s, %(page_header)s, %(interval)s, %(timestamp)s)'
connect.execute(query, params)
conn.commit()
conn.close()
Occasionally, when this runs, I get an error which says:
"Failed processing pyformat-parameters; %s" % err)
ProgrammingError: Failed processing pyformat-parameters; 'MySQLConverter'
object has no attribute '_list_to_mysql'
I know what is causing this, just uncertain how to go about fixing it. The 'page': unicode(self) param occasionally gets a list as a result.
In an attempt to fix this, I tweaked the above script to extract the list into a string, with the following:
page_list = u'{}'.format(self)
page_results = "('%s')" % "','".join(page_list)
params = {'build': self.tc.tag, 'page': page_results, 'object_id': self.object_id, 'page_header':
self.page_header, 'interval': t.interval, 'timestamp': timestamp}
When I run this, the error I am getting now is that the data is too long for the field. I debug it, to find that my page results has each character parsed out individually looking like so:
u'(\\'A\\',\\'p\\',\\'p\\',\\'M\\',\\'a\\',\\'i\\',\\'n\\',\\'M\\',\\'e\\',\\'n\\',\\'u\\',\\':\\',\\' \\',\\'N\\',\\'o\\',\\'n\\',\\'e\\')'
So the solution was to do the following, which takes the page_header and if it is in the instance of list to make that list a string:
conn = mysql.connector.connect(**config)
connect = conn.cursor()
page_list = u'{}'.format(self)
page_header_list = u'{}'.format(self.page_header)
if isinstance(page_header_list, list):
page_header_list = ', '.join(page_header_list)[0:100]
params = {'build': self.tc.tag, 'page': page_list, 'object_id': self.object_id,
'page_header': page_header_list, 'interval': t.interval, 'timestamp': timestamp}
query = u'INSERT INTO page_load_times (build, page, object_id, page_header, elapsed_time, date_run) ' \
'VALUES (%(build)s, %(page)s, %(object_id)s, %(page_header)s, %(interval)s, %(timestamp)s)'
connect.execute(query, params)
conn.commit()
conn.close()
Thank you #DarthOpto, you gave me lots of light. I solved it by putting srt around the variable:
params = {'build': self.tc.tag, 'page': srt(page_list),'object_id': self.object_id,
'page_header': srt(page_header_list),'interval': t.interval,'timestamp': timestamp}
Here below is my code to find a document by ObjectID:
def find(selector: JsValue, projection: Option[JsValue], sort: Option[JsValue],
page: Int, perPage: Int): Future[Seq[JsValue]] = {
var query = collection.genericQueryBuilder.query(selector).options(
QueryOpts(skipN = page * perPage)
)
projection.map(value => query = query.projection(value))
sort.map(value => query = query.sort(value.as[JsObject]))
// this is the line where the call crashes
query.cursor[JsValue].collect[Vector](perPage).transform(
success => success,
failure => failure match {
case e: LastError => DaoException(e.message, Some(DATABASE_ERROR))
}
)
}
Now let's suppose we invoke this method with an invalid ObjectID:
// ObjectId 53125e9c2004006d04b605abK is invalid (ends with a K)
find(Json.obj("_id" -> Json.obj("$oid" -> "53125e9c2004006d04b605abK")), None, None, 0, 25)
The call above causes the following exception when executing query.cursor[JsValue].collect[Vector](perPage) in the find method:
Caused by: java.util.NoSuchElementException: JsError.get
at play.api.libs.json.JsError.get(JsResult.scala:11) ~[play-json_2.10.jar:2.2.1]
at play.api.libs.json.JsError.get(JsResult.scala:10) ~[play-json_2.10.jar:2.2.1]
at play.modules.reactivemongo.json.collection.JSONGenericHandlers$StructureBufferWriter$.write(jsoncollection.scala:44) ~[play2-reactivemongo_2.10-0.10.2.jar:0.10.2]
at play.modules.reactivemongo.json.collection.JSONGenericHandlers$StructureBufferWriter$.write(jsoncollection.scala:42) ~[play2-reactivemongo_2.10-0.10.2.jar:0.10.2]
at reactivemongo.api.collections.GenericQueryBuilder$class.reactivemongo$api$collections$GenericQueryBuilder$$write(genericcollection.scala:323) ~[reactivemongo_2.10-0.10.0.jar:0.10.0]
at reactivemongo.api.collections.GenericQueryBuilder$class.cursor(genericcollection.scala:342) ~[reactivemongo_2.10-0.10.0.jar:0.10.0]
at play.modules.reactivemongo.json.collection.JSONQueryBuilder.cursor(jsoncollection.scala:110) ~[play2-reactivemongo_2.10-0.10.2.jar:0.10.2]
at reactivemongo.api.collections.GenericQueryBuilder$class.cursor(genericcollection.scala:331) ~[reactivemongo_2.10-0.10.0.jar:0.10.0]
at play.modules.reactivemongo.json.collection.JSONQueryBuilder.cursor(jsoncollection.scala:110) ~[play2-reactivemongo_2.10-0.10.2.jar:0.10.2]
at services.common.mongo.MongoDaoComponent$MongoDao$$anon$1.find(MongoDaoComponent.scala:249) ~[classes/:na]
... 25 common frames omitted
Any idea? Thanks.
I am creating a GUI where a user inputs a value and when he presses a pushbutton it runs an external function and displays error messages. I am having trouble with inserting the variable successfully in the GUI coding. I am confused as to where to insert my variable. I have tried handles but unfortunately its not working.
% --- Executes just before Stallfunction is made visible.
function Stallfunction_OpeningFcn(hObject, ~, handles, varargin)
% This function has no output args, see OutputFcn.
% hObject handle to figure
% eventdata reserved - to be defined in a future version of MATLAB
% handles structure with handles and user data (see GUIDATA)
% varargin command line arguments to Stallfunction (see VARARGIN)
% Choose default command line output for Stallfunction
handles.user_entry = user_entry;
% Update handles structure
guidata(hObject, handles);
% UIWAIT makes Stallfunction wait for user response (see UIRESUME)
% uiwait(handles.figure1);
I have inserted the variable in the above code, which is 'user_entry' is that correct?
user_entry is not assigned a value in your function. If you launch your GUI by passing a value for user_entry like this:
Stallfunction(user_entry)
then the first lines of your code in the openingFcn should be:
if ~isempty(varargin)
user_entry = varargin{1};
else
error('please start the GUI with an input value')
end
After this, you can assign user_entry to the handles structure as you're doing already.
Try this:
function num = get_num()
fig = figure('Units', 'characters', ...
'Position', [70 20 30 5], ...
'CloseRequestFcn', #close_Callback);
edit_num = uicontrol(...
'Parent', fig, ...
'Style', 'edit', ...
'Units', 'characters', ...
'Position', [1 1 10 3], ...
'HorizontalAlignment', 'left', ...
'String', 'init', ...
'Callback', #edit_num_Callback);
button_finish = uicontrol( ...
'Parent', fig, ...
'Tag', 'button_finish', ...
'Style', 'pushbutton', ...
'Units', 'characters', ...
'Position', [15 1 10 3], ...
'String', 'Finish', ...
'Callback', #button_finish_Callback);
% Nested functions
function edit_num_Callback(hObject,eventdata)
disp('this is a callback for edit box');
end
function button_finish_Callback(hObject,eventdata)
% Exit
close(fig);
end
function close_Callback(hObject,eventdata)
num_prelim = str2num(get(edit_num,'string'));
if(isempty(num_prelim))
errordlg('Must be a number.','Error','modal');
return;
end
num = num_prelim;
delete(fig);
end
waitfor(fig);
end
See if you can mess with this and get what you want. Also, learn to use nested functions and how callbacks work in matlab. Save this as a function file and then call "num = getnum;"