Error in MySQL Syntax run on remote server using PyCharm - mysql
I am fairly new to programming and I've been trying to implement this code in Python, using PyCharm. I'm running the code via a remote server, using PyCharm on my local computer. It was written by a former colleague, and has been giving a lot of encoding issues since we updated the packages like MySQL and the Python interpreter to 3.8. The MySQL version is 8.0, but this is an update. That was not the version installed originally when the code was written.
This is the full error that I am getting:
findBestMatch
*** WARNING: FoundCity[i] = {'Country': 'Austria', 'Page': 'Contact', 'Confidence': 10, 'Mentions': 1}
WriteToDB
Problem <class 'MySQLdb._exceptions.ProgrammingError'>
Traceback (most recent call last):
File "/remotepath/TextMining/NER/FindLocationStoreSQL.py", line 399, in
WriteToDB(c.title(), cn, idProject, 10, "Contact", "v1", cursor, db, database_country)
File "/remotepath/TextMining/NER/FindLocationStoreSQL.py", line 286, in WriteToDB
cursor.execute(sql,values)
File "/home/localhost/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 206, in execute
res = self._query(query)
File "/home/localhost/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 319, in _query
db.query(q)
File "/home/localhost/.local/lib/python3.8/site-packages/MySQLdb/connections.py", line 259, in query
_mysql.connection.query(self, query)
MySQLdb._exceptions.ProgrammingError: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'City text mined, Country from datasource'',Confidence='10',FoundWhere=''Contact'' at line 1")
Process finished with exit code -1
The code sample of the function it is trying to run is below:
def findBestMatch(FoundCity,FoundCountry,database_country):
pair_candidates = []
for i in range(0,5):
for j in range(0,5):
if len(FoundCity)>i and len(FoundCountry)>j:
city_i = FoundCity[i].get('City')
country_j = FoundCountry[j].get('Country')
if city_i != None and country_j != None:
#sql = "SELECT City,Country_CountryName,Longitude,Latitude FROM Semanticon.City where city like '{0}' and Country_CountryName like '{1}' and Population>0 order by Population desc".format(FoundCity[i]['City'].encode('utf-8'),FoundCountry[j]['Country'].encode('utf-8'))
sql = "SELECT City,Country_CountryName,Longitude,Latitude FROM Semanticon.City where city like '{0}' and Country_CountryName like '{1}' and Population>0 order by Population desc".format(
FoundCity[i]['City'], FoundCountry[j]['Country'])
try:
cursor.execute(sql)
except:
db = MySQLdb.connect(host, username, password, database, charset='utf8')
db.set_character_set("utf8")
cursor = db.cursor()
cursor.execute(sql)
resul = cursor.fetchall()
if len(resul)>0:
pair_candidates.append({"City":FoundCity[i]['City'],"Country":FoundCountry[j]['Country'],"Score":(FoundCity[i]["Confidence"]+FoundCountry[j]["Confidence"]+0.5*(FoundCity[i]["Mentions"]+FoundCountry[j]["Mentions"]))})
#return FoundCity[i]['City'],FoundCountry[j]['Country'],FoundCity[i]['Confidence']
else:
if city_i == None:
print("*** WARNING: FoundCity[i] = ", FoundCity[i])
else:
print("*** WARNING: FoundCountry[j] = ", FoundCountry[j])
I had to take the encoding out, hence the commented out 'sql' line. The encoding was causing problems and adding an extra 'b' to the string to be read from the database.
The 'WriteToDB' function that it's complaining about is below:
def WriteToDB(City,Country,ProjectId,Confidence,Location,Version,cursor,db,database_country):
sql = None
if database_country!="":
if database_country == Country:
if City != "":
#sql = "UPDATE ProjectLocation SET City='{0}',DataTrace='{1}',Confidence={2},FoundWhere='{3}' WHERE Projects_idProjects={4} and Country='{5}';".format(City," City text mined, Country from datasource",Confidence,Location,ProjectId,original_database_cntry)
sql = "UPDATE ProjectLocation SET City='%s',DataTrace='%s',Confidence='%s',FoundWhere='%s' WHERE Projects_idProjects='%s' and Country='%s';"
values = (City, " City text mined, Country from datasource", Confidence, Location, ProjectId,
original_database_cntry)
if database_country!=Country:
if Country.encode('utf-8') in database_country:
#sql = "UPDATE ProjectLocation SET City='{0}',DataTrace='{1}',Confidence={2},FoundWhere='{3}' WHERE Projects_idProjects={4} and Country='{5}';".format(
# City, " City text mined, Country from datasource", Confidence, Location, ProjectId,
#original_database_cntry)
sql = "UPDATE ProjectLocation SET City='%s',DataTrace='%s',Confidence='%s',FoundWhere='%s' WHERE Projects_idProjects='%s' and Country='%s';"
values = (City, " City text mined, Country from datasource", Confidence, Location, ProjectId,
original_database_cntry)
else:
print("Country conflict in project:"+str(ProjectId))
else:
#sql = "Insert into ProjectLocation (Type,City,Country,Projects_idProjects,Original_idProjects,IsLocationFromDataset,Confidence,FoundWhere,Version,DataTrace)" \
# "Values ('{0}','{1}','{2}',{3},{4},0,{5},'{6}','{7}','{8}')".format("Main",City,Country,ProjectId,ProjectId,Confidence,Location,Version,"Both minded from text v0.1")
sql = "Insert into ProjectLocation (Type,City,Country,Projects_idProjects,Original_idProjects,IsLocationFromDataset,Confidence,FoundWhere,Version,DataTrace)" \
"Values ('%s','%s','%s','%s',0,'%s','%s','%s','%s','%s')"
values = ("Main", City, Country, ProjectId,ProjectId, Confidence, Location, Version, "Both minded from text v0.1")
if sql!=None:
cursor.execute(sql,values)
db.commit()
I commented out the SQL queries as shown and tried to bind them instead, because it was giving a lot of encoding errors. I'm not sure how to get rid of this error and not end up with the encoding errors yet again.
Can someone help?
UPDATE to the question.
I reverted back all the sql queries, and used the queries with the encoding now (previously commented out) and I am getting 'b's in the output.
Any suggestions on how to properly encode these SQL queries so the binary encoding does not come out as b's in the output?
Here is a sample of the output:
ProjectID,ProjectName,FoundCity,FoundCountry,DatabaseCity,DatabaseCountry,Confidence,FoundWhere,Website
2542, Migrantour Country,,b'',b'',b'',10,Contact,link
3938,GeoSmartCity,,,b'',b'',10,Contact,link
Related
Saving dbplyr query (tbl_sql object) to MySQL without saving data locally
This question expands on this question Here, I'm using the custom function created by #Simon.S.A. shown in the answer to this question. I'm attempting to save a tbl_sql object in R to MySQL as a new table without first saving it locally. Here, the database and schema in my MySQL are named "test." The tbl_sql object in R is my_data, and I want to save this is a new table in MySQL labeled "car_data". library(DBI) library(tidyverse) library(dbplyr) #establish connection and import data from MySQL con <- DBI::dbConnect(RMariaDB::MariaDB(), dbname = "test", host = "127.0.0.1", user = "user", password = "password") my_data <- tbl(con, "mtcars") my_data <- my_data %>% filter(mpg >= 22) # write function to save tbl_sql as a new table in SQL write_to_database <- function(input_tbl, db, schema, tbl_name){ # connection tbl_connection <- input_tbl$src$con # SQL query sql_query <- glue::glue( "SELECT *\n", "INTO {db}.{schema}.{tbl_name}\n", "FROM (\n", dbplyr::sql_render(input_tbl), "\n) AS sub_query" ) result <- dbExecute(tbl_connection, as.character(sql_query)) } # execute function write_to_database(my_data, "test", "test", "car_data") After running final line, I get the following error. I'm not sure how I can fix this. Error: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '.test.car_data FROM ( SELECT * FROM `mtcars` WHERE (`mpg` >= 22.0) ) AS sub_quer' at line 2 [1064] 12. stop(structure(list(message = "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '.test.car_data\nFROM (\nSELECT *\nFROM `mtcars`\nWHERE (`mpg` >= 22.0)\n) AS sub_quer' at line 2 [1064]", call = NULL, cppstack = NULL), class = c("Rcpp::exception", "C++Error", "error", "condition"))) 11. result_create(conn#ptr, statement, is_statement) 10. initialize(value, ...) 9. initialize(value, ...) 8. new("MariaDBResult", sql = statement, ptr = result_create(conn#ptr, statement, is_statement), bigint = conn#bigint, conn = conn) 7. dbSend(conn, statement, params, is_statement = TRUE) 6. .local(conn, statement, ...) 5. dbSendStatement(conn, statement, ...) 4. dbSendStatement(conn, statement, ...) 3. dbExecute(tbl_connection, as.character(sql_query)) 2. dbExecute(tbl_connection, as.character(sql_query)) 1. write_to_database(my_data, "test", "test", "car_data")
Creating a table with INTO command is an SQL Server (even MS Access) specific syntax and not supported in MySQL. Instead, consider the counterpart statement: CREATE TABLE...SELECT. Also, schema differs between RDBMS's. For MySQL, database is synonymous to schema. Therefore, consider adjusted version of SQL build: sql_query <- glue::glue( "CREATE TABLE {db}.{tbl_name}\n AS \n", "SELECT * \n", "FROM (\n", dbplyr::sql_render(input_tbl), "\n) AS sub_query" )
R package monitoR error on dbUploadTemplate [pkg-monitor]
I'm using the R package monitoR and getting an error message that I can't figure out. I'm trying to upload a correlation template list ("bithTemps") to a MySQL database ("noh") using the dbUploadTemplate command. dbUploadTemplate(templates = bithTemps, uid = "root", pwd = "****", db.name = "noh", analyst = 1, locationID = "2", date.recorded = "2017/09/07", recording.equip = "Unknown", species.code = "BITH", type = "COR") This returns: Error: $ operator is invalid for atomic vectors I have confirmed the ODBC connection is working, that the template list is functional (i.e., it works when called to other arguments in the package), and that the SQL database has the required entries for analyst, location, and species code.
It seems that this error was actually triggered by a non-functional ODBC connection. This part of the dbUploadTemplate function species <- RODBC::sqlQuery(dbCon, paste("SELECT `pkSpeciesID`, `fldSpeciesCode` FROM `tblSpecies` WHERE `fldSpeciesCode` = '", paste(species.code, sep = "", collapse = "' OR `fldSpeciesCode` = '"), "'", sep = "")) queries a table in the SQL database and returns an object called 'species'. If the query fails (e.g., because RODBC can't connect) than 'species' is empty, and the following operation speciesID <- NULL for (i in 1:length(species.code)) { speciesID[i] <- species$pkSpeciesID[species$fldSpeciesCode == species.code[i]] } triggers the error. Fixing the ODBC connection resolves the error.
SQL % statement in python returns error
I created 5 tables in mysql workbench 5.7 in which I will pull data from APIgraph queries for a given facebook page. However, when I run the code, it throws an error: ProgrammingError: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '%s, %s)' at line 1 Here is the part of the code which I think contains the error : #create connection to db connection = connect_db() cursor = connection.cursor() #SQL request for inserting the date of the page into the database insert_page = ("INSERT INTO page" "(fb_id, name)" "VALUES (%s, %s)") insert_posts = ("INSERT INTO posts " "(page_id, fb_post_id, message, time_created)" "VALUES (%s, %s, %s, %s)") And I finally put the data at the end of the code: cursor.execute(insert_page, json_pageiddata) Any ideas? Thanks for helping EDIT: here is my json_pageiddtata variable, obtained from a URL query with APIgraph: pageid_url = create_pageid_url(current_page, APP_ID, APP_SECRET) json_pageiddata = render_to_json(pageid_url) print json_pageiddata["name"], json_pageiddata["id"]
If you are using data as dictionary u need to specify index name, try to use %(name)s.
Python. I'm creating a phone database and linking it with a python code
I'm trying to make a Python code that searches my sql database. I can make it search the phone model and display the duration of the contract. The problem is that when I try to make it display the price of each tariff, it returns error #1064. I figured out that the column name: "Price/Month" causes error in python so I changed it to PriceperMonth; but this error occurs. Example of Price per Month: £ 25. It has currency symbol. Maybe that is the one that causes the error. My code: #!/Python27/python import mysql.connector def SearchPhoneByPhone(Model): conn = mysql.connector.connect(user='root', \ host='127.0.0.1', database ='phone') cur = conn.cursor() query = "SELECT phone.Model, tariff.Duration price.PriceperMonth FROM phone, tariff, price \ WHERE phone.Phone_ID = price.Phone_ID AND tariff.Tariff_ID = price.Tariff_ID \ AND phone.Model LIKE '%" + Model + "%'" cur.execute(query) print "This model " + Model +" has the following tariffs:" for (Model, Duration, PriceperMonth) in cur.fetchall(): print Model + " (Duration of the Contract: " + Duration + ")" + " (Price per Month: " + PriceperMonth + ")" print "" cur.close() conn.close() SearchPhoneByPhone('iPhone 6 Plus') The error message: Traceback (most recent call last): File "C:\Users\Ren25\Dropbox\SAT-0400 Project\Phone Database Connect.py", line 25, in <module> SearchPhoneByPhone('iPhone 6 Plus') File "C:\Users\Ren25\Dropbox\SAT-0400 Project\Phone Database Connect.py", line 15, in SearchPhoneByPhone cur.execute(query) File "C:\Python27\lib\site-packages\mysql\connector\cursor.py", line 515, in execute self._handle_result(self._connection.cmd_query(stmt)) File "C:\Python27\lib\site-packages\mysql\connector\connection.py", line 488, in cmd_query result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query)) File "C:\Python27\lib\site-packages\mysql\connector\connection.py", line 395, in _handle_result raise errors.get_exception(packet) ProgrammingError: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '.PriceperMonth FROM phone, tariff, price WHERE phone.Phone_ID = price.Phone_' at line 1
you forget comma between these 2 columns in select query: tariff.Duration, price.PriceperMonth so your query should be: query = "SELECT phone.Model, tariff.Duration, price.PriceperMonth FROM phone, tariff, price \ WHERE phone.Phone_ID = price.Phone_ID AND tariff.Tariff_ID = price.Tariff_ID \ AND phone.Model LIKE '%" + Model + "%'"
Failed processing pyformat-parameters; 'MySQLConverter' object has no attribute '_list_to_mysql'
I have a python script which everytime a wait_for_page call is made it writes the time it took to wait for the page to a database. The query is below: conn = mysql.connector.connect(**config) connect = conn.cursor() params = {'build': self.tc.tag, 'page': unicode(self), 'object_id': self.object_id, 'page_header': self.page_header, 'interval': t.interval, 'timestamp': timestamp} query = u'INSERT INTO page_load_times (build, page, object_id, page_header, elapsed_time, date_run) ' \ 'VALUES (%(build)s, %(page)s, %(object_id)s, %(page_header)s, %(interval)s, %(timestamp)s)' connect.execute(query, params) conn.commit() conn.close() Occasionally, when this runs, I get an error which says: "Failed processing pyformat-parameters; %s" % err) ProgrammingError: Failed processing pyformat-parameters; 'MySQLConverter' object has no attribute '_list_to_mysql' I know what is causing this, just uncertain how to go about fixing it. The 'page': unicode(self) param occasionally gets a list as a result. In an attempt to fix this, I tweaked the above script to extract the list into a string, with the following: page_list = u'{}'.format(self) page_results = "('%s')" % "','".join(page_list) params = {'build': self.tc.tag, 'page': page_results, 'object_id': self.object_id, 'page_header': self.page_header, 'interval': t.interval, 'timestamp': timestamp} When I run this, the error I am getting now is that the data is too long for the field. I debug it, to find that my page results has each character parsed out individually looking like so: u'(\\'A\\',\\'p\\',\\'p\\',\\'M\\',\\'a\\',\\'i\\',\\'n\\',\\'M\\',\\'e\\',\\'n\\',\\'u\\',\\':\\',\\' \\',\\'N\\',\\'o\\',\\'n\\',\\'e\\')'
So the solution was to do the following, which takes the page_header and if it is in the instance of list to make that list a string: conn = mysql.connector.connect(**config) connect = conn.cursor() page_list = u'{}'.format(self) page_header_list = u'{}'.format(self.page_header) if isinstance(page_header_list, list): page_header_list = ', '.join(page_header_list)[0:100] params = {'build': self.tc.tag, 'page': page_list, 'object_id': self.object_id, 'page_header': page_header_list, 'interval': t.interval, 'timestamp': timestamp} query = u'INSERT INTO page_load_times (build, page, object_id, page_header, elapsed_time, date_run) ' \ 'VALUES (%(build)s, %(page)s, %(object_id)s, %(page_header)s, %(interval)s, %(timestamp)s)' connect.execute(query, params) conn.commit() conn.close()
Thank you #DarthOpto, you gave me lots of light. I solved it by putting srt around the variable: params = {'build': self.tc.tag, 'page': srt(page_list),'object_id': self.object_id, 'page_header': srt(page_header_list),'interval': t.interval,'timestamp': timestamp}