I have a list of strings that change dynamically. I need to create a MySQL table where each string in the list is a name of a column.Note: I have found some examples using sqlalchemy but it didn't help me at all.
here is my try:
f = open(filepath,"r")
pluginoutput= f.read()
pluginoptojson = json.loads(pluginoutput)
columnsnames = (pluginoptojson["columns"])
countcolumns = len(pluginoptojson["columns"])
count = 0
lst = []
for name in columnsnames:
if count < countcolumns:
lst.append(str(name))
count +=1
lst.append("caseid")
createsqltable = """CREATE TABLE IF NOT EXISTS %s""" + for t in columnsnames: """ """ (test)
c.execute(createsqltable)
conn.commit()
c.close()
conn.close()
my head is about to explode of thinking. any help will be appreciated.
first thanks alot to Irnzcig.
I just tested his recommendation and it worked like charm. the code is:
pluginoutput= f.read()
pluginoptojson = json.loads(pluginoutput)
columnsnames = (pluginoptojson["columns"])
countcolumns = len(pluginoptojson["columns"])
count = 0
lst = []
for name in columnsnames:
if count < countcolumns:
lst.append(str(name))
count +=1
lst.append("caseid")
table_name = "test1"
createsqltable = """CREATE TABLE IF NOT EXISTS """ + table_name + " (" + " VARCHAR(50),".join(lst) + " VARCHAR(50))"
c.execute(createsqltable)
conn.commit()
c.close()
conn.close()
gc.collect()
Related
I need to create an sql login/ sign up system for my program, however I keep hitting this error no matter what I do to change it. I need to have a randomly generated UTID, the users first and surname, along with a password that is verified, then the UserID is generated by taking the first three letters of the first name and the whole surname. I cant figure out how to overcome this.
I have tried to give the values inside the sql statement when inserting some literal datatypes, like writing "
c.execute('insert INTO tbl_Teachers (str(UTID), str(FName), str(SName), str(userPass), str(userID))VALUES(?,?,?,?,?);', var_insert) " but nothing seems to work.
def signup():
name = []
surname = []
print("Please enter the following details.")
user_type = str(input("Are you a teacher or a student: "))
if user_type == ("teacher") or ("Teacher"):
var_FName = str(input("First Name: "))
var_LName = str(input("Last Name: "))
var_password1 = str(input("Choose a password: "))
var_password2 = str(input("Please confirm password: "))
UTID = str(random.randint(0,100000))
print ("Your UserID is "+UTID+"")
name.append(var_FName)
surname.append(var_LName)
userID = []
for x in range (0, 3):
userID.append(var_FName[x])
for x in range (0,len(var_LName)):
userID.append(var_LName[x])
print (userID)
if var_password1 != var_password2:
print("Please try again.")
else:
var_insert = []
var_insert.append(UTID)
var_insert.append(var_FName)
var_insert.append(var_LName)
var_insert.append(str(var_password1))
var_insert.append(userID)
conn = sqlite3.connect('Program.db')
c = conn.cursor()
c.execute('insert INTO tbl_Teachers (UTID, FName, SName, userPass, userID)VALUES(?,?,?,?,?);', var_insert)
conn.commit()
InterfaceError: Error binding parameter 4 - probably unsupported type.
userID is supposed to be a string, but you're creating a list. Use string concatenation, not the append method.
userID = var_FName[0:3] + var_LName
2017-05-12 is inserted in to MySQL as 2017-06-12 - couldn't figure out with debugging...pasted the key parts of code
Windows platform; Used PyMySQL; If the issue isnt resolved, i will probably bulk update the table to reduce the month by one...i guess that should be easy...
exceptions = []
flag = 'N'
def obtain_list_of_db_tickers():
"""
Obtains a list of the ticker symbols in the database.
"""
with con:
cur = con.cursor()
cur.execute("SELECT id, ticker FROM symbol")
data = cur.fetchall()
list1,list2 = map(list,zip(*data))
uniqtickers = list(set(list2))
rangelist = list(range(1,len(uniqtickers)+1))
newtickers = list(zip(rangelist,uniqtickers))
return [(d[0], d[1]) for d in newtickers]
def get_daily_historic_data_yahoo(
ticker, start_date=(2017,5,12),
end_date=datetime.date.today().timetuple()[0:3]
) :
""""
Obtains data from Yahoo Finance returns and a list of tuples
ticker: Yahoo Finance ticker symbol eg: GOOG
start_date: Start date in (YYYY,M,D) format
End_date: End date in (YYYY,M,D) format
"""
# Construct the Yahoo URL with the correct integer query parameters
# for start and end dates. Note that some parameters are zero-based!
ticker_tup = (
ticker, start_date[1]-1, start_date[2],
start_date[0], end_date[1]-1, end_date[2],
end_date[0]
)
yahoo_url = "http://ichart.finance.yahoo.com/table.csv"
yahoo_url += "?s=%s&a=%s&b=%s&c=%s&d=%s&e=%s&f=%s"
yahoo_url = yahoo_url % ticker_tup
# Try connecting to Yahoo Finance and obtaining the data
# On failure, print an error message.
global flag
try:
yf_data = pd.read_csv(yahoo_url)
yf_data['Date'] = yf_data['Date'].apply(lambda x:
datetime.datetime.strptime(x, '%Y-%m-%d'))
prices = yf_data.set_index(['Date'])[['Open', 'High', 'Low', 'Close',
'Volume', 'Adj Close']].to_records().tolist()
except Exception as e:
print("Could not download Yahoo data: %s" %e)
flag = 'Y'
exceptions.append(ticker)
return None
return prices
def insert_daily_data_into_db(
data_vendor_id, symbol_id, ticker, daily_data
):
"""
Takes a list of tuples of daily data and adds it to the
MySQL database. Appends the vendor ID and symbol ID to the data.
daily_data: List of tuples of the OHLC data(with adj_close and volume)
"""
# Create the time now
now = datetime.datetime.utcnow()
#Amend the data to include the vendor ID and symbol IDprices
daily_data = [
(data_vendor_id, symbol_id, ticker, d[0], now, now,
d[1], d[2], d[3], d[4], d[5], d[6])
for d in daily_data
]
# Create the insert strings
column_str = """data_vendor_id, symbol_id, ticker, price_date, created_date,
last_updated_date, open_price, high_price, low_price,
close_price, volume, adj_close_price"""
insert_str = ("%s, " *12)[:-2]
final_str = "INSERT INTO daily_price (%s) VALUES (%s)" %\
(column_str, insert_str)
# Using the MySQL connection, carry out an INSERT INTO for every symbol
with con:
cur = con.cursor()
cur.executemany(final_str, daily_data)
if __name__ == "__main__":
# This ignores the warnings regarding Data Truncation
# from the Yahoo precision to Decimal(19,4) datatypes
warnings.filterwarnings('ignore')
# Loop over the tickers and insert the daily historical
# data into the database
tickers = obtain_list_of_db_tickers()
lentickers = len(tickers)
for i, t in enumerate(tickers):
print(
"Adding data for %s: %s out of %s" %
(t[1], i+1, lentickers)
)
yf_data = get_daily_historic_data_yahoo(t[1])
if (flag == 'N'):
insert_daily_data_into_db('1', t[0], t[1], yf_data)
elif (flag == 'Y'):
flag = 'N'
20170325_225012,ctcc01,voice,639128342574,639464810386,cap_timeout,6004,639180007006,2,0,null
20170325_235012,ctcc01,voice,639128342554,639464520384,cap_timeout,6004,639180007006,2,0,null
20170325_245012,ctcc01,voice,639128342174,639464820327,cap_timeout,6004,639180007006,2,0,null
Sample text data.csv file above:
Steps needed to complete:
Process the csv file
Each line should be inserted to MySQL Column. Column1, Column2, Column3 ... Column11
This is my Code so far.
import csv
import re
f = open('data.csv')
csv_f = csv.reader(f)
writer = csv.writer(f)
cdr = []
for row in csv_f:
cdr.append("Some auto increment id")
cdr.append(re.sub(r'_.*$', "", row[0]))
cdr.append(row[1])
cdr.append(row[2])
cdr.append(row[3])
cdr.append(row[4])
cdr.append(row[5])
cdr.append(row[6])
cdr.append(row[7])
cdr.append(row[8])
cdr.append(row[9])
cdr.append(row[10])
print cdr
with open('output.csv', 'wb') as f:
writer = csv.writer(f)
writer.writerows(cdr)
I was able to output it on the terminal the way i want it but it did make it in one list :). Somehow i don't know how can i split and insert it on the mysql.
['Some auto increment id', '20170325', 'ctcc01', 'voice', '639128342574', '639464820387', 'cap_timeout', '6004', '639180007006', '2', '0', 'null', 'Some auto increment id', '20170325', 'ctcc01', 'voice', '639128342574', '639464820387', 'no_subs', '6004', '639180007006', '2', '0', 'null', 'Some auto increment id', '20170325', 'ctcc01', 'voice', '639128342574', '639464820387', 'cap_timeout', '6004', '639180007006', '2', '0', 'null']
No. You and you need to use MySql.db.connect, and insert and commit.
Basically, you'll find your answer in a similar question here
The code should be:
# open file, and define a csv reader and writer - you've done that correctly
import csv
import re
f = open('data.csv')
csv_f = csv.reader(f)
writer = csv.writer(f)
vals = []
# open and connect to database
dbname = 'mydb' # or whatever your database is named
tablename = 'mytable' # or whatever table you wish to insert into
hostname = 'localhost' # or whatever your mysql db hostname is
username = 'root' # or whatever your username for mysql db is
pw = '' # or whatever your password is for that user
mydb = MySQLdb.connect(host=hostname, user=username, passwd=pw, db=dbname)
cursor = mydb.cursor()
# for each row create an 'INSERT INTO' execution-string
auto = 0 # auto-incrementing
exec_string = ""
rowid = ""
for row in csv_f:
# INSERT INTO mytable(Column, Column1,Column2, ... Column12)
# VALUES(auto, rowid, row(1), row(2)...
# execstr header:
exec_string = "INSERT INTO " + tablename + "(Column, "
for i in range(1,11): # columns
exec_string += "Column" + i + (", " if (i<11))
# ...it may be a mistake in the question and you need Column0
# ...in which case the end of the exec_string line should read + "("
# ...and the for should be in range(0,11): ...
# execstr values:
exec_string += ") Values("
for _ in range(12):
exec_string += "%S"
exec_string += ")" # close values function
vals = []
auto += 1
rowid = re.sub(r'_.*$', "", row[0])
vals.append(auto)
vals.append(rowid)
for i in range(2,12) # count to 12 starting with 2
vals.append(row[i])
# and execute it!
cursor.execute(exec_string, vals)
# commit and close the connection to the database.
mydb.commit()
cursor.close()
def compute_invigilated_mark():
"""Prints given name and family name with overall score"""
test = 0.15
exam = 0.60
first_name = input("Given names(s)? ")
last_name = input("Family name? ")
both_names = last_name.upper() + "," + first_name.title()
new_test_percent = float(input("Test percent? ")) * test
new_exam_percent = float(input("Exam percent? ")) * exam
overall_percent = new_test_percent + new_exam_percent
end_result = overall_percent / (exam + test)
print(both_names + end_result)
compute_invigilated_mark()
I want to get the end result of, For example: Bourne, Jason: 66.0
The rrror message:
builtins.TypeError: Can't convert 'float' object to str implicitly.
NB: I spaced it out like this so you could read it easier :).
both_names is a string and end_result is a float, yet you are trying to add\concat them together (print(both_names + end_result)).
You should convert end_result to a string:
print(both_names + str(end_result))
I have a table named passive than contains a list of timestamped events per user. I want to fill the attribute duration, which correspond to the time between the current row's event and the next event done by this user.
I tried the following query:
UPDATE passive as passive1
SET passive1.duration = (
SELECT min(UNIX_TIMESTAMP(passive2.event_time) - UNIX_TIMESTAMP(passive1.event_time) )
FROM passive as passive2
WHERE passive1.user_id = passive2.user_id
AND UNIX_TIMESTAMP(passive2.event_time) - UNIX_TIMESTAMP(passive1.event_time) > 0
);
This returns the error message Error 1093 - You can't specify target table for update in FROM.
In order to circumvent this limitation, I tried to follow the structure given in https://stackoverflow.com/a/45498/395857, which uses a nested subquery in the FROM clause to create an implicit temporary table, so that it doesn't count as the same table we're updating:
UPDATE passive
SET passive.duration = (
SELECT *
FROM (SELECT min(UNIX_TIMESTAMP(passive2.event_time) - UNIX_TIMESTAMP(passive.event_time))
FROM passive, passive as passive2
WHERE passive.user_id = passive2.user_id
AND UNIX_TIMESTAMP(passive2.event_time) - UNIX_TIMESTAMP(passive1.event_time) > 0
)
AS X
);
However, the passive table in the nested subquery doesn't refer to the same passive as in the main query. Because of that, all rows have the same passive.duration value. How can I refer to the main query's passive in the nested subquery? (or maybe are there some alternative ways to structure such a query?)
Try Like this....
UPDATE passive as passive1
SET passive1.duration = (
SELECT min(UNIX_TIMESTAMP(passive2.event_time) - UNIX_TIMESTAMP(passive1.event_time) )
FROM (SELECT * from passive) Passive2
WHERE passive1.user_id = passive2.user_id
AND UNIX_TIMESTAMP(passive2.event_time) - UNIX_TIMESTAMP(passive1.event_time) > 0
)
;
We can use a Python script to circumvent the issue:
'''
We need an index on user_id, timestamp to speed up
'''
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Download it at http://sourceforge.net/projects/mysql-python/?source=dlp
# Tutorials: http://mysql-python.sourceforge.net/MySQLdb.html
# http://zetcode.com/db/mysqlpython/
import MySQLdb as mdb
import datetime, random
def main():
start = datetime.datetime.now()
db=MySQLdb.connect(user="root",passwd="password",db="db_name")
db2=MySQLdb.connect(user="root",passwd="password",db="db_name")
cursor = db.cursor()
cursor2 = db2.cursor()
cursor.execute("SELECT observed_event_id, user_id, observed_event_timestamp FROM observed_events ORDER BY observed_event_timestamp ASC")
count = 0
for row in cursor:
count += 1
timestamp = row[2]
user_id = row[1]
primary_key = row[0]
sql = 'SELECT observed_event_timestamp FROM observed_events WHERE observed_event_timestamp > "%s" AND user_id = "%s" ORDER BY observed_event_timestamp ASC LIMIT 1' % (timestamp, user_id)
cursor2.execute(sql)
duration = 0
for row2 in cursor2:
duration = (row2[0] - timestamp).total_seconds()
if (duration > (60*60)):
duration = 0
break
cursor2.execute("UPDATE observed_events SET observed_event_duration=%s WHERE observed_event_id = %s" % (duration, primary_key))
if count % 1000 == 0:
db2.commit()
print "Percent done: " + str(float(count) / cursor.rowcount * 100) + "%" + " in " + str((datetime.datetime.now() - start).total_seconds()) + " seconds."
db.close()
db2.close()
diff = (datetime.datetime.now() - start).total_seconds()
print 'finished in %s seconds' % diff
if __name__ == "__main__":
main()