How to parse russian text properly using Python 2.7 and BeautifulSoup? - html

Question

I am trying to parse all posts from a russian website(http://games4you.ucoz.ua/news/). I am using Python 2.7.9 and BeautifulSoup 4. I am working in PyCharm. I've tried a lot of things to make it work, but still I get this instead of russian text: '\u0421\u0442\u0440\u0430\u0442\u0435\u0433\u0456\u0457'
This is my code:
# Parsing information from games4you.ucoz.ua
# -*- coding: utf-8 -*-
import re
import csv
import urllib
from bs4 import BeautifulSoup
BASE_URL = "http://games4you.ucoz.ua/news/"
def get_html(url):
response = urllib.urlopen(url)
return response.read()
def get_page_count(html):
soup = BeautifulSoup(html)
paggination = soup.find('div', class_='catPages1')
return int(paggination.find_all('a')[-2].text)
def save(games, path):
# with open(path, 'w') as csvfile:
# writer = csv.writer(csvfile)
#
# writer.writerow(('Title', 'Category', 'Date', 'Time'))
#
# writer.writerows(
# (game['title'], ', '.join(game['category']), game['date'], game['time']) for game in games
# )
with open(path,'w+') as f:
f.write(str(games).encode("UTF-8"))
def parse(html):
soup = BeautifulSoup(html)
# Getting the <div> that contains all posts on page
all_entries = soup.find('div',id='allEntries')
# Getting all of the posts (every table represents one post)
tables = all_entries.find_all('table',class_='eBlock')
# Creating a list o dictionaries for games information
games = []
for table in tables:
try:
# Getting the game title
game_title = table.tr.td.a.text
game_post_body = table.find('div',class_='eMessage')
# Getting the game description
game_description = game_post_body.p.text.split('....')[0] + '.'
game_details = table.find('div',class_='eDetails')
# Getting the game category
game_category = game_details.a.text
game_post_details = game_details.text
except:
print 'Some error'
continue
# Getting the post views count
post_views = game_post_details[:game_post_details.find('function')].split()[-2]
# Getting the post date
post_date = game_details.span.text
# Getting the post time
post_time = game_details.span['title']
# print 'Game title: ',game_title,'\n'
# print 'Views: ',post_views,'\n'
# print 'Game category: ',game_category,'\n'
# print 'Game description: ','\n',game_description,'\n'
# print 'Post date: ',post_date,'\n'
# print 'Post time: ',post_time,'\n'
games.append({
'title': game_title,
'category' : game_category,
'description' : game_description,
'date' : post_date,
'time' : post_time
})
return games
def main():
total_pages = get_page_count(get_html(BASE_URL))
print('Total found %d pages...' % total_pages)
games = []
for page in range(1, total_pages + 1):
print('Parsing %d%% (%d/%d)' % (page / total_pages * 100, page, total_pages))
games.extend(parse(get_html(BASE_URL + "?page%d" % page)))
print('Saving...')
save(games, 'games.txt')
main()

Answer 1

in Python2
>>> import HTMLParser
>>> s = 'Ell és la víctima que expia els nostres pecats, i no tan sols els nostres, sinó els del món sencer.'
>>> print HTMLParser.HTMLParser().unescape(s)
Ell és la víctima que expia els nostres pecats, i no tan sols els nostres, sinó els del món sencer.
in Python3
>>> import html
>>> html.unescape(s)
your example
'Стратегії'
For "normal" utf-8 file writing(reading) use
import codecs
f = codecs.open(filename, 'w', 'utf-8')
hope this helps

Answer 2

Yes, I did it! Guess I messed up with the decoding/coding text and using different charsets. Everything I had to do is simply convert the data I get from BeautifulSoup from Unicode to Utf-8, like this:
game_title = game_title.encode("utf-8")
game_category = game_category.encode("utf-8")
game_description = game_description.encode("utf-8")
post_date = post_date.encode("utf-8")
post_time = post_time.encode("utf-8")
Nothing else was needed. This is the result code that worked for me:
# Parsing information from games4you.ucoz.ua
import csv
import urllib
from bs4 import BeautifulSoup
BASE_URL = "http://games4you.ucoz.ua/news/"
def get_html(url):
response = urllib.urlopen(url)
return response.read()
def get_page_count(html):
soup = BeautifulSoup(html)
paggination = soup.find('div', class_='catPages1')
return int(paggination.find_all('a')[-2].text)
def save(games, path):
with open(path, 'w+') as csvfile:
writer = csv.writer(csvfile)
writer.writerow(('Title', 'Category', 'Date', 'Time'))
writer.writerows(
(game['title'],game['category'], game['date'], game['time']) for game in games
)
def parse(html):
soup = BeautifulSoup(html)
# Getting the <div> that contains all posts on page
all_entries = soup.find('div',id='allEntries')
# Getting all of the posts (every table represents one post)
tables = all_entries.find_all('table',class_='eBlock')
# Creating a list o dictionaries for games information
games = []
for table in tables:
try:
# Getting the game title
game_title = table.tr.td.a.text
game_post_body = table.find('div',class_='eMessage')
# Getting the game description
game_description = game_post_body.p.text.split('....')[0] + '.'
game_details = table.find('div',class_='eDetails')
# Getting the game category
game_category = game_details.a.text
game_post_details = game_details.text
except:
print 'Some error'
continue
# Getting the post views count
post_views = game_post_details[:game_post_details.find('function')].split()[-2]
# Getting the post date
post_date = game_details.span.text
# Getting the post time
post_time = game_details.span['title']
# Converting all data from Unicode to Utf-8
game_title = game_title.encode("utf-8")
game_category = game_category.encode("utf-8")
game_description = game_description.encode("utf-8")
post_date = post_date.encode("utf-8")
post_time = post_time.encode("utf-8")
# Writing data to the list
games.append({
'title': game_title,
'category' : game_category,
'description' : game_description,
'date' : post_date,
'time' : post_time
})
return games
def main():
total_pages = get_page_count(get_html(BASE_URL))
print('Total found %d pages...' % total_pages)
games = []
for page in range(1, total_pages + 1):
print('Parsing %d%% (%d/%d)' % (page / total_pages * 100, page, total_pages))
games.extend(parse(get_html(BASE_URL + "?page%d" % page)))
print('Saving...')
save(games, 'games.csv')
main()