Generate n-gram for a specific column present in mysql db - mysql

Question

I'm writing a code to generate n-grams for every record in the table by reading a specific column.
def extract_from_db(inp_cust_id):
sql_db = TatDBHelper()
t_sql = "select notes from raw_data where customer_id = {0}"
db_data = sql_db.execute_read(t_sql.format(inp_cust_id))
for row in db_data:
text = row.values()
bi_grams = generate_ngrams(text[0].encode("utf-8"), 2)
print bi_grams
def generate_ngrams(sentence, n):
sentence = sentence.lower()
sentence = re.sub(r'[^a-zA-Z0-9\s]', ' ', sentence)
tokens = [token for token in sentence.split(" ") if token != ""]
ngrams = zip(*[tokens[i:] for i in range(n)])
return [" ".join(ngram) for ngram in ngrams]
I'm getting the output like:
['i highly', 'highly recommend', 'recommend it']
['the penguin', 'penguin encounter', 'encounter was', 'was awesome']
I want the output to look like below, can anybody help me to get this.
['i highly',
'highly recommend',
'recommend it',
...
]

Answer 1

creat another list all_ngrams, and keep appending the values to it , using .extend(), and finally you will have all the ngrams in one list.
Try this :
def extract_from_db(inp_cust_id):
sql_db = TatDBHelper()
t_sql = "select notes from raw_data where customer_id = {0}"
db_data = sql_db.execute_read(t_sql.format(inp_cust_id))
all_ngrams = []
for row in db_data:
text = row.values()
bi_grams = generate_ngrams(text[0].encode("utf-8"), 2)
all_ngrams.extend(bi_grams)
print all_ngrams