I have been able to get data from database and populate into the tableWidget, but the image column is not shown. I tried a code I found online and still, it didn't work. The image column in the database has the BLOB data type. Kindly assist in correcting my following code. Or you may want to advise and recommend another method other than the tableWidget
def getPersData(self):
con = MySQLdb.connect(host="localhost", user="root", password="", database="db")
with con:
cur = con.cursor()
query = cur.execute("SELECT * FROM persons")
rows = cur.fetchall()
for row_number, row_data in enumerate(rows):
self.ui.tableWidget.insertRow(row_number)
for column_number, column_data in enumerate(row_data):
if column_number == 1:
item = self.getImg(column_data)
self.ui.tableWidget.setCellWidget(row_number, column_number, item)
else:
self.ui.tableWidget.setItem(row_number, column_number, QTableWidgetItem(str(column_data)))
self.ui.tableWidget.verticalHeader().setDefaultSectionSize(100)
self.ui.tableWidget.show()
def getImg(self, img):
img_label = self.ui.label
img_label.setText("")
img_label.setScaledContents(True)
pixmap = QPixmap()
pixmap.loadFromData(img, "PNG")
img_label.setPixmap(pixmap)
return img_label
The logic to use the bytes (in my previous answer I proposed to use base64 so I use it in this case as well) to build a QPixmap that can be converted into a QIcon that can be displayed in the QTableWidget:
for row_number, row_data in enumerate(rows):
self.ui.tableWidget.insertRow(row_number)
for column_number, column_data in enumerate(row_data):
it = QTableWidgetItem()
if column_number == 1:
pixmap = QPixmap()
pixmap.loadFromData(QByteArray.fromBase64(row_data))
icon = QIcon(pixmap)
it.setIcon(icon)
else:
it.setText(row_data)
self.ui.tableWidget.setItem(row_number, column_number, it)
Related
I am having a problem with reading my csv file into the MySQL database. I have tried a number of solutions, but the errors just keep changing and the code isn't working. This same code had worked with another csv file, so I'm thinking I might be doing something wrong with this one?
Here is my code
from database_access import *
from builtins import bytes, int, str
import codecs
import csv
import requests
from urllib.parse import urlparse, urljoin
from bs4 import BeautifulSoup
import re
import cgi
import MySQLdb
import chardet
# from database_access import *
import MySQLdb
import simplejson
if __name__ == '__main__':
with open("SIMRA.csv",'r') as file:
reader = csv.reader(file)
#reader = csv.reader(text)
next(reader, None)
print ("project running")
#print (row[7])
#rowlist = []
all_links = []
all_project_ids = []
for row in reader:
if row[7] != "" and row[16] != "":
country = row[2]
city = row[8]
description = row[11] + '' + row[12]
title = row[7].replace("'", "''")
link = row[16]
#date_start = row[9]
#print a check here
print(title,description,country, city, link)
db = MySQLdb.connect(host, username, password, database, charset='utf8')
cursor = db.cursor()
new_project = True
proj_check = "SELECT * from Projects where ProjectName like '%" + title + "%'"
#proj_check = "SELECT * from Projects where ProjectName like %s",(title,)
#cur.execute("SELECT * FROM records WHERE email LIKE %s", (search,))
cursor.execute(proj_check)
num_rows = cursor.rowcount
if num_rows != 0:
new_project = False
url_compare = "SELECT * from Projects where ProjectWebpage like '" + link + "'"
#url_compare = "SELECT * from Projects where ProjectWebpage like %s",(link,)
cursor.execute(url_compare)
num_rows = cursor.rowcount
if num_rows != 0:
new_project = False
if new_project:
project_insert = "Insert into Projects (ProjectName,ProjectWebpage,FirstDataSource,DataSources_idDataSources) VALUES (%s,%s,%s,%s)"
cursor.execute(project_insert, (title, link,'SIMRA', 5))
projectid = cursor.lastrowid
print(projectid)
#ashoka_projectids.append(projectid)
db.commit()
ins_desc = "Insert into AdditionalProjectData (FieldName,Value,Projects_idProjects,DateObtained) VALUES (%s,%s,%s,NOW())"
cursor.executemany(ins_desc, ("Description", description, str(projectid)))
db.commit()
ins_location = "Insert into ProjectLocation (Type,Country,City,Projects_idProjects) VALUES (%s,%s,%s,%s)"
cursor.execute(ins_location, ("Main", country,city, str(projectid)))
db.commit()
else:
print('Project already exists!')
print(title)
all_links.append(link)
#print out SIMRA's links to a file for crawling later
with open('simra_links', 'w', newline='') as f:
write = csv.writer(f)
for row in all_links:
columns = [c.strip() for c in row.strip(', ').split(',')]
write.writerow(columns)
When I ran this, I got the following error:
File "/usr/lib/python3.8/codecs.py", line 322, in decode
(result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa3 in position 898: invalid start byte
I did some research and tried handling the encoding error by adding different forms of encoding, as seen here - UnicodeDecodeError: ‘utf8’ codec can’t decode byte 0xa5 in position 0: invalid start byte, and Python MySQLdb TypeError: not all arguments converted during string formatting. Added this in this in the csv open parameter -
with open("SIMRA.csv", 'r', encoding="cp437", errors='ignore') as file:
Running the code with these different encoding options came up with a different error:
MySQLdb._exceptions.ProgrammingError: not all arguments converted during bytes formatting
Further research suggested using tuples or lists in order to address this problem, so I added these in the 'select' function in the code, as suggested here - Python MySQLdb TypeError: not all arguments converted during string formatting and in the Python SQL documentation here - PythonMySqldb
So the select query became:
proj_check = "SELECT * from Projects where ProjectName like %s",(title,)
cursor.execute(proj_check)
num_rows = cursor.rowcount
if num_rows != 0:
new_project = False
url_compare = "SELECT * from Projects where ProjectWebpage like %s",(link,)
cursor.execute(url_compare)
num_rows = cursor.rowcount
if num_rows != 0:
new_project = False
When I ran the code, I came up with this Assertion Error and I have no idea what to do anymore.
File "/home/ros/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 205, in execute
assert isinstance(query, (bytes, bytearray))
AssertionError
I have run out of ideas. It might be that I'm missing something small, but I can't figure this out now as I've been battling with this for two days now.
Can anyone help point out what I'm missing? It will be greatly appreciated. This code ran perfectly with another csv file. I am running this with Python 3.8 btw.
Have solved this now. I had to use a different encoding with the original code and this solved the problem. So, I changed the csv open parameter to:
with open("SIMRA.csv",'r', encoding="ISO-8859-1") as file:
reader = csv.reader(file)
Were you expecting £? You need to specify what the encoding of the file is. It may be "latin1". See the syntax of LOAD DATA for how to specify CHARACTER SET latin1.
I’m working on a search engine project for my country. I have the country’s domains list of the sites to crawl. So I have built a bot (the bot was written in python) to crawl some of the sites at the moment. When crawling is successful, the crawler will commit the crawled content to MySQL database. So I have data that people can search for in the MySQL remote server as I speak.
Now, I want to implement NLP in the search such that when a user enters a keyword in the search box, relevant results from MySQL database will show to the user based on the keyword used. I’m using python 3.8 and NLTK for this project. I haven’t done anything about NLP before. This is my first time. But I have read about it though. I also want to ask if using MySQL database is the right option for the search engine. If not, why can’t I use it and what should I use? I’m currently using MySQL because I’m more familiar with it a lot and I enjoy when using it for data storage. I’ve been struggling with this thing since last December. What I really need is the right NLP algorithm to use for selecting relevant results from MySQL database. I know that NLP is difficult to implement but I will appreciate if you can at least try to help out.
Here’s the code:
What I have done so far, I copied some of the code from Kaggle.com . Here is the link https://www.kaggle.com/amitkumarjaiswal/nlp-search-engine/notebook but I still haven’t been able to make it work for my own project.
import pandas as pd
import numpy as np
import string
import random
import nltk
import os
import re
#import nltk.corpus
import csv
#nltk.download('all')
#print(os.listdir(nltk.data.find("corpora")))
#pip install --upgrade nltk
from nltk.corpus import brown
from nltk.corpus import reuters
from nltk.tokenize import word_tokenize
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
from nltk.stem import SnowballStemmer
#load 10k reuters news documents
len(reuters.fileids())
#view text from one document
reuters.raw(fileids=['test/14826'])[0:201]
exclude = set(string.punctuation)
alldocslist = []
for index, i in enumerate(reuters.fileids()):
text = reuters.raw(fileids=[i])
text = ''.join(ch for ch in text if ch not in exclude)
alldocslist.append(text)
print(alldocslist[1])
#tokenize words in all DOCS
plot_data = [[]] * len(alldocslist)
for doc in alldocslist:
text = doc
tokentext = word_tokenize(text)
plot_data[index].append(tokentext)
print(plot_data[0][1])
# Navigation: first index gives all documents, second index gives specific document, third index gives words of that doc
plot_data[0][1][0:10]
#make all words lower case for all docs
for x in range(len(reuters.fileids())):
lowers = [word.lower() for word in plot_data[0][x]]
plot_data[0][x] = lowers
plot_data[0][1][0:10]
# remove stop words from all docs
stop_words = set(stopwords.words('english'))
for x in range(len(reuters.fileids())):
filtered_sentence = [w for w in plot_data[0][x] if not w in stop_words]
plot_data[0][x] = filtered_sentence
plot_data[0][1][0:10]
#stem words EXAMPLE (could try others/lemmers )
snowball_stemmer = SnowballStemmer("english")
stemmed_sentence = [snowball_stemmer.stem(w) for w in filtered_sentence]
stemmed_sentence[0:10]
porter_stemmer = PorterStemmer()
snowball_stemmer = SnowballStemmer("english")
stemmed_sentence = [ porter_stemmer.stem(w) for w in filtered_sentence]
stemmed_sentence[0:10]
# Create inverse index which gives document number for each document and where word appears
#first we need to create a list of all words
l = plot_data[0]
flatten = [item for sublist in l for item in sublist]
words = flatten
wordsunique = set(words)
wordsunique = list(wordsunique)
import math
from textblob import TextBlob as tb
def tf(word, doc):
return doc.count(word) / len(doc)
def n_containing(word, doclist):
return sum(1 for doc in doclist if word in doc)
def idf(word, doclist):
return math.log(len(doclist) / (0.01 + n_containing(word, doclist)))
def tfidf(word, doc, doclist):
return (tf(word, doc) * idf(word, doclist))
# THIS ONE-TIME INDEXING IS THE MOST PROCESSOR-INTENSIVE STEP AND WILL TAKE TIME TO RUN (BUT ONLY NEEDS TO BE RUN ONCE)
plottest = plot_data[0][0:1000]
worddic = {}
for doc in plottest:
for word in wordsunique:
if word in doc:
word = str(word)
index = plottest.index(doc)
positions = list(np.where(np.array(plottest[index]) == word)[0])
idfs = tfidf(word,doc,plottest)
try:
worddic[word].append([index,positions,idfs])
except:
worddic[word] = []
worddic[word].append([index,positions,idfs])
# the index creates a dic with each word as a KEY and a list of doc indexs, word positions, and td-idf score as VALUES
worddic['china']
# pickel (save) the dictonary to avoid re-calculating
np.save('worddic_1000.npy', worddic)
# create word search which takes multiple words and finds documents that contain both along with metrics for ranking:
## (1) Number of occruances of search words
## (2) TD-IDF score for search words
## (3) Percentage of search terms
## (4) Word ordering score
## (5) Exact match bonus
from collections import Counter
def search(searchsentence):
try:
# split sentence into individual words
searchsentence = searchsentence.lower()
try:
words = searchsentence.split(' ')
except:
words = list(words)
enddic = {}
idfdic = {}
closedic = {}
# remove words if not in worddic
realwords = []
for word in words:
if word in list(worddic.keys()):
realwords.append(word)
words = realwords
numwords = len(words)
# make metric of number of occurances of all words in each doc & largest total IDF
for word in words:
for indpos in worddic[word]:
index = indpos[0]
amount = len(indpos[1])
idfscore = indpos[2]
enddic[index] = amount
idfdic[index] = idfscore
fullcount_order = sorted(enddic.items(), key=lambda x:x[1], reverse=True)
fullidf_order = sorted(idfdic.items(), key=lambda x:x[1], reverse=True)
# make metric of what percentage of words appear in each doc
combo = []
alloptions = {k: worddic.get(k, None) for k in (words)}
for worddex in list(alloptions.values()):
for indexpos in worddex:
for indexz in indexpos:
combo.append(indexz)
comboindex = combo[::3]
combocount = Counter(comboindex)
for key in combocount:
combocount[key] = combocount[key] / numwords
combocount_order = sorted(combocount.items(), key=lambda x:x[1], reverse=True)
# make metric for if words appear in same order as in search
if len(words) > 1:
x = []
y = []
for record in [worddic[z] for z in words]:
for index in record:
x.append(index[0])
for i in x:
if x.count(i) > 1:
y.append(i)
y = list(set(y))
closedic = {}
for wordbig in [worddic[x] for x in words]:
for record in wordbig:
if record[0] in y:
index = record[0]
positions = record[1]
try:
closedic[index].append(positions)
except:
closedic[index] = []
closedic[index].append(positions)
x = 0
fdic = {}
for index in y:
csum = []
for seqlist in closedic[index]:
while x > 0:
secondlist = seqlist
x = 0
sol = [1 for i in firstlist if i + 1 in secondlist]
csum.append(sol)
fsum = [item for sublist in csum for item in sublist]
fsum = sum(fsum)
fdic[index] = fsum
fdic_order = sorted(fdic.items(), key=lambda x:x[1], reverse=True)
while x == 0:
firstlist = seqlist
x = x + 1
else:
fdic_order = 0
# also the one above should be given a big boost if ALL found together
#could make another metric for if they are not next to each other but still close
return(searchsentence,words,fullcount_order,combocount_order,fullidf_order,fdic_order)
except:
return("")
search('indonesia crude palm oil')[1]
# 0 return will give back the search term, the rest will give back metrics (see above)
search('indonesia crude palm oil')[1][1:10]
# save metrics to dataframe for use in ranking and machine learning
result1 = search('china daily says what')
result2 = search('indonesia crude palm oil')
result3 = search('price of nickel')
result4 = search('north yemen sugar')
result5 = search('nippon steel')
result6 = search('China')
result7 = search('Gold')
result8 = search('trade')
df = pd.DataFrame([result1,result2,result3,result4,result5,result6,result7,result8])
df.columns = ['search term', 'actual_words_searched','num_occur','percentage_of_terms','td-idf','word_order']
df
# look to see if the top documents seem to make sense
alldocslist[1]
# create a simple (non-machine learning) rank and return function
def rank(term):
results = search(term)
# get metrics
num_score = results[2]
per_score = results[3]
tfscore = results[4]
order_score = results[5]
final_candidates = []
# rule1: if high word order score & 100% percentage terms then put at top position
try:
first_candidates = []
for candidates in order_score:
if candidates[1] > 1:
first_candidates.append(candidates[0])
second_candidates = []
for match_candidates in per_score:
if match_candidates[1] == 1:
second_candidates.append(match_candidates[0])
if match_candidates[1] == 1 and match_candidates[0] in first_candidates:
final_candidates.append(match_candidates[0])
# rule2: next add other word order score which are greater than 1
t3_order = first_candidates[0:3]
for each in t3_order:
if each not in final_candidates:
final_candidates.insert(len(final_candidates),each)
# rule3: next add top td-idf results
final_candidates.insert(len(final_candidates),tfscore[0][0])
final_candidates.insert(len(final_candidates),tfscore[1][0])
# rule4: next add other high percentage score
t3_per = second_candidates[0:3]
for each in t3_per:
if each not in final_candidates:
final_candidates.insert(len(final_candidates),each)
#rule5: next add any other top results for metrics
othertops = [num_score[0][0],per_score[0][0],tfscore[0][0],order_score[0][0]]
for top in othertops:
if top not in final_candidates:
final_candidates.insert(len(final_candidates),top)
# unless single term searched, in which case just return
except:
othertops = [num_score[0][0],num_score[1][0],num_score[2][0],per_score[0][0],tfscore[0][0]]
for top in othertops:
if top not in final_candidates:
final_candidates.insert(len(final_candidates),top)
for index, results in enumerate(final_candidates):
if index < 5:
print("RESULT", index + 1, ":", alldocslist[results][0:100],"...")
# example of output
rank('indonesia palm oil')
# example of output
rank('china')
# Create pseudo-truth set using first 5 words
# Because I don't have a turth set I will generate a pseudo one by pulling terms from the documents - this is far from perfect
# as it may not approximate well peoples actual queries but it will serve well to build the ML architecture
df_truth = pd.DataFrame()
for doc in plottest:
first_five = doc[0:5]
test_sentence = ' '.join(first_five)
result = search(test_sentence)
df_temp = pd.DataFrame([result])
df_truth= pd.concat([df_truth, df_temp])
df_truth['truth'] = range(0,len(plottest))
df_truth1 = pd.DataFrame()
seqlen = 3
for doc in plottest:
try:
start = random.randint(0,(len(doc)-seqlen))
random_seq = doc[start:start+seqlen]
test_sentence = ' '.join(random_seq)
except:
test_sentence = doc[0]
result = search(test_sentence)
df_temp = pd.DataFrame([result])
df_truth1= pd.concat([df_truth1, df_temp])
df_truth1['truth'] = range(0,len(plottest))
# create another psuedo-truth set using different random 4 word sequence from docs
df_truth2 = pd.DataFrame()
seqlen = 4
for doc in plottest:
try:
start = random.randint(0,(len(doc)-seqlen))
random_seq = doc[start:start+seqlen]
test_sentence = ' '.join(random_seq)
except:
test_sentence = doc[0]
result = search(test_sentence)
df_temp = pd.DataFrame([result])
df_truth2= pd.concat([df_truth2, df_temp])
df_truth2['truth'] = range(0,len(plottest))
# create another psuedo-truth set using different random 2 word sequence from docs
df_truth3 = pd.DataFrame()
seqlen = 2
for doc in plottest:
try:
start = random.randint(0,(len(doc)-seqlen))
random_seq = doc[start:start+seqlen]
test_sentence = ' '.join(random_seq)
except:
test_sentence = doc[0]
result = search(test_sentence)
df_temp = pd.DataFrame([result])
df_truth3= pd.concat([df_truth3, df_temp])
df_truth3['truth'] = range(0,len(plottest))
# combine the truth sets and save to disk
truth_set = pd.concat([df_truth,df_truth1,df_truth2,df_truth3])
truth_set.columns = ['search term', 'actual_words_searched','num_occur','percentage_of_terms','td-idf','word_order','truth']
truth_set.to_csv("truth_set_final.csv")
truth_set[0:10]
truth_set
test_set = truth_set[0:3]
test_set
# convert to long format for ML
# WARNING AGAIN THIS IS A SLOW PROCESS DUE TO RAM ILOC - COULD BE OPTIMISED FOR FASTER PERFORMANCE
# BUG When min(maxnum, len(truth_set) <- is a int not a list because of very short variable length)
# row is row
# column is variable
# i is the result
final_set = pd.DataFrame()
test_set = truth_set[1:100]
maxnum = 5
for row in range(0,len(test_set.index)):
test_set = truth_set[1:100]
for col in range(2,6):
for i in range(0,min(maxnum,len(truth_set.iloc[row][col]))):
x = pd.DataFrame([truth_set.iloc[row][col][i]])
x['truth'] = truth_set.iloc[row]['truth']
x.columns = [(str(truth_set.columns[col]),"index",i),(str(truth_set.columns[col]),"score",i),'truth']
test_set = test_set.merge(x,on='truth')
final_set = pd.concat([final_set,test_set])
final_set.head()
final_set.to_csv("ML_set_100.csv")
final_set2 = final_set.drop(['actual_words_searched','num_occur','percentage_of_terms','search term','td-idf','word_order'], 1)
final_set2.to_csv("ML_set_100_3.csv")
final_set2.head()
final_set3 = final_set2
final_set3[0:10]
Obviously, the code above isn't returning searched keywords from MySQL database. I believe you understand me? Thank you very much!
I would like to set up a trading bot via Google Cloud to run around the clock.
In Google Cloud Functions I use the Inline editor with runtime Python 3.7.
I have two questions:
1) Main.py section: Here I copied the full code of my Python script (Trading Bot) - see code below for reference (which works well when run as a script in my IDE Spyder).
However, below Google asks to provide a function to execute. However, my code is just a script with no main function. Can I just put at the top of the code e.g.: "def trading_bot(self):" and indent the remaining part below?
While the code as a script copied below works well, if I add the "def trading_bot(self):" at the top in my IDE (Spyder), the code doesnt seem to work properly...How can I make sure the code within the function runs properly, when I call the function from Google Cloud (or from my IDE).
2) Requirements.txt section: Can you provide guidance what exactly I need to put there, i.e. can I look up the dependencies used in my code somewhere? I use Anaconda for distribution, the classes imported for the script are at the top of the script provided below.
Thanks for any help. Glad also for your advice if you think Google Cloud Functions is not the best approach to run a trading bot but it seemed to me to be the simplest solution.
import bitmex
import json
from time import sleep
from bitmex_websocket import BitMEXWebsocket
import logging, time, requests
import numpy as np
import pandas as pd
import matplotlib.dates as mdates
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings("ignore")
from datetime import datetime
import math
from statistics import mean
#-------------------------
#variable
symbol = "XBTUSD"
#standard API connection
api_key = "XXX"
api_secret = "XXX"
#True for testnet
client = bitmex.bitmex(test=False, api_key=api_key, api_secret=api_secret)
#------------------
# Trading algorithm
symbol = "XBTUSD"
ordType = 'Stop'
#starting order quantity
orderQty = 1
leftBars = 6
rightBars = 2
#round to 0.5
def round_to_BTC(n):
result = round(n*2)/2
return result
t=1
while t < 1000000:
time_now = (time.strftime('%H:%M:%S', time.localtime(int(time.time()))))
t_now = time_now[6:8]
t1 = "00"
t2 = "59"
FMT = '%S'
def days_hours_minutes_seconds(td):
return td.days, td.seconds//3600, (td.seconds//60)%60, td.seconds
if t_now == str('00'):
#give 1 second to candlestick to properly close
sleep(1)
elif t_now > str('00') and t_now <= str('59'):
s1 = datetime.strptime(t2, FMT) - datetime.strptime(t_now, FMT)
s1_seconds = days_hours_minutes_seconds(s1)[3]+2
sleep(s1_seconds)
else:
pass
time_now = (time.strftime('%H:%M:%S', time.localtime(int(time.time()))))
print("The time is now: " + time_now)
#most recent swing candles, get highs and lows / #binsizes = {"1m": 1, "5m": 5, "1h": 60, "1d": 1440}
#+1 is the middle bar
totalBars = leftBars + rightBars + 1
swing_candles = client.Trade.Trade_getBucketed(symbol=symbol, binSize="1m", count=totalBars, reverse=True).result()[0]
last_highs = []
last_lows = []
i=0
while i <= (len(swing_candles)-1):
last_highs.append(swing_candles[i]["high"])
last_lows.append(swing_candles[i]["low"])
i += 1
#get the highest high and the lowest low
highest_high = max(last_highs)
lowest_low = min(last_lows)
#check if there are existing positions & orders
if client.Position.Position_get().result()[0] != []:
positions_quantity = client.Position.Position_get().result()[0][0]["currentQty"]
else:
positions_quantity = 0
#check existing orders
buy_orders_quantity = []
sell_orders_quantity = []
orders_quantity = client.Order.Order_getOrders(filter=json.dumps({"open": True})).result()[0]
h=0
while h <= len(orders_quantity)-1:
if orders_quantity[h]["side"] == "Sell":
sell_orders_quantity.append(orders_quantity[h])
elif orders_quantity[h]["side"] == "Buy":
buy_orders_quantity.append(orders_quantity[h])
h += 1
if highest_high == last_highs[rightBars] and positions_quantity == 0:
if buy_orders_quantity == []:
client.Order.Order_new(symbol = symbol, orderQty = orderQty*1, side = "Buy", ordType = 'Stop', stopPx = (highest_high+0.5), execInst ='LastPrice' ).result()
elif buy_orders_quantity != []:
orderID = buy_orders_quantity[0]["orderID"]
client.Order.Order_amend(orderID=orderID, orderQty=orderQty*1, stopPx = (highest_high+0.5)).result()
else:
pass
elif highest_high == last_highs[rightBars] and positions_quantity > 0:
#dont place any additional long
pass
elif highest_high == last_highs[rightBars] and positions_quantity < 0:
if buy_orders_quantity != []:
orderID = buy_orders_quantity[0]["orderID"]
client.Order.Order_amend(orderID=orderID, orderQty=orderQty*2, stopPx = (highest_high+0.5)).result()
else:
client.Order.Order_new(symbol = symbol, orderQty = (orderQty)*2, side = "Buy", ordType = 'Stop', stopPx = (highest_high+0.5), execInst ='LastPrice' ).result()
elif lowest_low == last_lows[rightBars] and positions_quantity == 0:
if sell_orders_quantity == []:
client.Order.Order_new(symbol = symbol, orderQty = (orderQty)*-1, side = "Sell", ordType = 'Stop', stopPx = (lowest_low-0.5), execInst ='LastPrice' ).result()
elif sell_orders_quantity != []:
orderID = sell_orders_quantity[0]["orderID"]
client.Order.Order_amend(orderID=orderID, orderQty=orderQty*-1, stopPx = (lowest_low-0.5)).result()
else:
pass
elif lowest_low == last_lows[rightBars] and positions_quantity < 0:
#dont place any additional shorts
pass
elif lowest_low == last_lows[rightBars] and positions_quantity > 0:
if sell_orders_quantity != []:
orderID = sell_orders_quantity[0]["orderID"]
client.Order.Order_amend(orderID=orderID, orderQty=orderQty*-2, stopPx = (lowest_low-0.5)).result()
else:
client.Order.Order_new(symbol = symbol, orderQty = (orderQty)*-2, side = "Sell", ordType = 'Stop', stopPx = (lowest_low-0.5), execInst ='LastPrice' ).result()
positions_quantity = client.Position.Position_get().result()[0][0]["currentQty"]
buy_orders_quantity = []
sell_orders_quantity = []
orders_quantity = client.Order.Order_getOrders(filter=json.dumps({"open": True})).result()[0]
h=0
while h <= len(orders_quantity)-1:
if orders_quantity[h]["side"] == "Sell":
sell_orders_quantity.append(orders_quantity[h])
elif orders_quantity[h]["side"] == "Buy":
buy_orders_quantity.append(orders_quantity[h])
h += 1
if positions_quantity > 0:
if sell_orders_quantity != []:
orderID = sell_orders_quantity[0]["orderID"]
client.Order.Order_amend(orderID=orderID, orderQty=orderQty*-2).result()
elif positions_quantity < 0:
if buy_orders_quantity != []:
orderID = buy_orders_quantity[0]["orderID"]
client.Order.Order_amend(orderID=orderID, orderQty=orderQty*2).result()
print("Your current position is " + str(positions_quantity))
print("This is iteration: " + str(t))
t += 1
As concerns my second question, I solved it in the following way:
In the command terminal, type: pip freeze > requirements.txt
The file contains all dependencies.
As concerns question 1 I still dont understand what code exactly needs to be put in the section main.py.
Thanks!
Cloud Functions is not an adequate product for your use case. They are mostly used for lightweight calculations or not high resource consuming methods.
The magic of CF consists in that they execute your code whenever you hit the URL that belongs to it. This is important to understand for your question number 1. If you want your function to work, you need to always create a method that accepts the "request" parameter. As it is the information from the HTTP request made when the URL is hit.
You can take a look at this document for reference.
You function should always start like this
from flask #import your dependencies
def my_awesome_function(request):
#Your logic
In this case you should write "my_awesome_function" on the Function to Execute textbox.
You also have to be careful with your resources, as CF have 5 presentations. They differ in CPU and Memory you can read more about this here.
This, among many reasons, you should not use Cloud Functions for your bot. I could recommend you to use a virtual machine, but activities related to use of the Services for cryptocurrency mining without Google's prior written approval are frowned upon and may result in the deactivation of your product as stated in the terms of service.
I am using Django to display rows in mysql.
The table in mysql has a primary key which it bigint, and one of them is 871195445245063168, 18 digits.
But on my page, I see 871195445245063200 displayed, the least 3 digits are rounded. I am wondering where I make it wrong.
1, I define a class with a function named data_query to query mysql.
class MyQuery:
self.conn = MySQLdb.connect(host = self.DBHOST, user = self.DBUSER,
passwd = self.DBPWD,port = self.DBPORT,charset = self.CHARSET,connect_timeout=3)
def data_query(self,sql):
cursor = self.conn.cursor(MySQLdb.cursors.DictCursor)
start = time.time()
cursor.execute(sql)
end = time.time()
sql_time = end - start
column_description = cursor.description
column_name = [ column[0] for column in column_description ]
res = cursor.fetchall()
cursor.close()
self.conn.close()
return res,column_name,sql_time
2, I defined a json encoder as follows
class CJsonEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, datetime.datetime):
try:
return obj.strftime('%Y-%m-%d %H:%M:%S')
except ValueError:
return str(obj)
elif isinstance(obj, datetime.date):
try:
return obj.strftime('%Y-%m-%d')
except ValueError:
return str(obj)
elif isinstance(obj,datetime.timedelta):
return str(obj)
elif isinstance(obj, decimal.Decimal):
return float(obj)
elif isinstance(obj,ObjectId):
return str(obj)
else:
return json.JSONEncoder.default(self, obj)
3, I get my display like this, with sensitive info replaced.
db = MyQuery(host, user, pwd, port)
sql_statement = 'select * from mytable where Findex=871195445245063168 limit 10'
sql_result, table_column_name, sql_time = db.data_query(sql_statement)
query_result = {}
column_name = column_format(table_column_name)
query_result['column'] = column_name
query_result['data'] = list(sql_result)
return HttpResponse(json.dumps(query_result, cls=CJsonEncoder), content_type='application/json')
So, what I go wrong here? Thanks.
This is a JavaScript issue. Your number is bigger than the largest safe integer in JavaScript (Number.MAX_SAFE_INTEGER), so it is rounded.
You can verify this in your browser console or in node.js
$ node
> x = 871195445245063168
871195445245063200
I assume you are either using your response in some kind of JavaScript frontend or you have some Browser extension to render the JSON, which is written in JavaScript.
If you request that URL with a client like curl, you will see that it is returned correctly from the server.
I have a large csv file that I can not load into memory. I need to find which variables are constant. How can I do that?
I am reading the csv as
d = pd.read_csv(load_path, header=None, chunksize=10)
Is there an elegant way to solve the problem?
The data contains string and numerical variables
This is my current slow solution that does not use pandas
constant_variables = [True for i in range(number_of_columns)]
with open(load_path) as f:
line0 = next(f).split(',')
for num, line in enumerate(f):
line = line.split(',')
for i in range(n_col):
if line[i] != line0[i]:
constant_variables[i] = False
if num % 10000 == 0:
print(num)
You have 2 methods I can think of iterate over each column and check for uniqueness:
col_list = pd.read_csv(path, nrows=1).columns
for col in range(len(col_list)):
df = pd.read_csv(path, usecols=col)
if len(df.drop_duplicates()) == len(df):
print("all values are constant for: ", df.column[0])
or iterate over the csv in chunks and check again the lengths:
for df in pd.read_csv(path, chunksize=1000):
t = dict(zip(df, [len(df[col].value_counts()) for col in df]))
print(t)
The latter will read in chunks and tell you how unique each columns data is, this is just rough code which you can modify for your needs