Web Scraping: How to get info from dynamic pages? - html

Question

I'm newbie in web scraping. I know how to get data from an HTML or from a JSON but there is a place where I can't know how to do it. I would like to get the positions of points and X's that you can see in the short chart of this page.
http://www.fiba.basketball/euroleaguewomen/18-19/game/2410/Nadezhda-ZVVZ-USK-Praha#|tab=shot_chart
How can I do that?

Answer 1

I'm fairly new as well, but learning as I go. It looks like this page is dynamic, so you'd need to use Selenium to load the page first, before grabbing the html with beautifulsoup to get the x and y coordinates from the Made Shots and Missed shots. So I gave it a shot and was able to get a dataframe with the x, y coords along with if it was 'made' or 'miss'.
I plotted it afterwards just to check to see if it matched, and it appears to be flipped about the x-axis. I believe this is because when you plot on a chart like this graphically, the top, left corner is your (0,0). So your y coordinates are going to be opposite when you want to plot it. I could be wrong though.
None the less, here's the code I used.
import pandas as pd
import bs4
from selenium import webdriver
driver = webdriver.Chrome('C:\chromedriver_win32\chromedriver.exe')
driver.get('http://www.fiba.basketball/euroleaguewomen/18-19/game/2410/Nadezhda-ZVVZ-USK-Praha#|tab=shot_chart')
html = driver.page_source
soup = bs4.BeautifulSoup(html,'html.parser')
made_shots = soup.findAll("svg", {"class": "shot-hit icon icon-point clickable"})
missed_shots = soup.findAll("svg", {"class": "shot-miss icon icon-miss clickable"})
def get_coordiantes(element, label):
results = pd.DataFrame()
for point in element:
x_point = float(point.get('x'))
y_point = float(point.get('y'))
marker = label
temp_df = pd.DataFrame([[x_point, y_point, marker]], columns=['x','y','marker'])
results = results.append(temp_df)
return results
made_results = get_coordiantes(made_shots, 'made')
missed_results = get_coordiantes(missed_shots, 'missed')
results = made_results.append(missed_results)
results = results.reset_index(drop=True)
results['y'] = results['y'] * -1
driver.close()
gives this output:
In [6]:results.head(5)
Out[6]:
x y marker
0 33.0 -107.0 made
1 159.0 -160.0 made
2 143.0 -197.0 made
3 38.0 -113.0 made
4 65.0 -130.0 made
and when I plot it:
import seaborn as sns
import numpy as np
# Add a column: the color depends of x and y values, but you can use whatever function.
value=(results['marker'] == 'made')
results['color']= np.where( value==True , "green", "red")
# plot
sns.regplot(data=results, x="x", y="y", fit_reg=False, scatter_kws={'facecolors':results['color']})
ADDITIONAL: I'm sure there's a better, more efficient, cleaner way to code this up. But just doing it on the fly, came up with this. It should get you going. Feel free to dive into it and look at the html source code to start seeing how it's grabbing the different data. have fun.
import pandas as pd
import bs4
from selenium import webdriver
driver = webdriver.Chrome('C:\chromedriver_win32\chromedriver.exe')
driver.get('http://www.fiba.basketball/euroleaguewomen/18-19/game/2410/Nadezhda-ZVVZ-USK-Praha#|tab=shot_chart')
html = driver.page_source
soup = bs4.BeautifulSoup(html,'html.parser')
###############################################################################
shots = soup.findAll("g", {"class": "shot-item"})
results = pd.DataFrame()
for point in shots:
hit = point.get('data-play-by-play-action-hit')
action_id = point.get('data-play-by-play-action-id')
period = point.get('data-play-by-play-action-period')
player_id = point.get('data-play-by-play-action-player-id')
team_id = point.get('data-play-by-play-action-team-id')
x_point = float(point.find('svg').get('x'))
y_point = float(point.find('svg').get('y'))
temp_df = pd.DataFrame([[hit, action_id, period, player_id, team_id, x_point, y_point]],
columns=['hit','action_id','period','player_id','team_id','x','y'])
results = results.append(temp_df)
results['y'] = results['y'] * -1
results = results.reset_index(drop=True)
###############################################################################
player_ids = soup.findAll('label', {"class": "item-label"})
players = pd.DataFrame()
for player in player_ids:
player_id = player.find('input').get('data-play-by-play-action-player-id')
if player_id == None:
continue
player_name = player.find('span').text
temp_df = pd.DataFrame([[player_id, player_name]],
columns=['player_id','player_name'])
players = players.append(temp_df)
players = players.reset_index(drop=True)
###############################################################################
team_ids = soup.findAll('div', {"class": "header-scores_desktop"})
teams_A = team_ids[0].find('div', {"class": "team-A"})
team_id_A = teams_A.find('img').get('src').rsplit('/')[-1]
team_name_A = teams_A.find('span').text
teams_B = team_ids[0].find('div', {"class": "team-B"})
team_id_B = teams_B.find('img').get('src').rsplit('/')[-1]
team_name_B = teams_B.find('span').text
teams = pd.DataFrame([[team_id_A, team_name_A],[team_id_B,team_name_B]],
columns=['team_id','team_name'])
teams = teams.reset_index(drop=True)
###############################################################################
actions = pd.DataFrame()
action_ids = soup.findAll('div', {"class": "overlay-wrapper"})
for action in action_ids:
action_id = action.get('data-play-by-play-action-id')
time_remaining = action.find('div').find('span', {'class': 'time'}).text
full_name = action.find('div').find('span', {'class': 'athlete-name'}).text
if not action.find('div').find('span', {'class': 'action-code'}):
result_of_action = '+0'
else:
result_of_action = action.find('div').find('span', {'class': 'action-code'}).text
action_description = action.find('div').find('span', {'class': 'action-description'}).text
team_A_score = action.find('div').find('span', {'class': 'team-A'}).text
team_B_score = action.find('div').find('span', {'class': 'team-B'}).text
temp_df = pd.DataFrame([[action_id, time_remaining, full_name, result_of_action, team_A_score, team_B_score, action_description]],
columns=['action_id','time_remaining', 'full_name', 'result_of_action', team_name_A+'_score', team_name_B+' score', 'action-description'])
actions = actions.append(temp_df)
actions = actions.reset_index(drop=True)
###############################################################################
results = pd.merge(results, players, how='left', on='player_id')
results = pd.merge(results, teams, how='left', on='team_id')
results = pd.merge(results, actions, how='left', on='action_id')
driver.close()
And to clean it a bit, you can sort the rows so that they are in order, play-by-play from start to finish
results.sort_values(['period', 'time_remaining'], ascending=[True, False], inplace=True)
results = results.reset_index(drop=True)