The srcset= strings on the website are https links, but when I use selenium's driver.find('img')['srcset'], it gives me base64 strings - html

Question

I'm trying to scrape images from a website. In the website's html code, the srcset sections exist and are of the form
srcset="https://...."
For example,
srcset="https://secure.img1-fg.wfcdn.com/im/80458162/resize-h300-w300%5Ecompr-r85/1068/106844956/Derry+84%2522+Reversible+Sectional.jpg 300w,https://secure.img1-fg.wfcdn.com/im/19496430/resize-h400-w400%5Ecompr-r85/1068/106844956/Derry+84%2522+Reversible+Sectional.jpg 400w,https://secure.img1-fg.wfcdn.com/im/75516274/resize-h500-w500%5Ecompr-r85/1068/106844956/Derry+84%2522+Reversible+Sectional.jpg 500w"
However, when I try to get these srcset link using selenium and beautiful soup, I get the following:
"data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"
Moreover, every time the srcset fails to get a valid link, the string that it gets is always
"data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"
I tried a bunch of different lines of code, but haven't had success with any of it. Here is the full code I currently have:
def get_info_from_product_link(product_link): #get the price and correctly filtered image link
info = dict()
driver = webdriver.Chrome('C:/Users/Brian/Downloads/chromedriver_win32/chromedriver.exe')
driver.implicitly_wait(200)
try:
driver.get(product_link)
soup = BeautifulSoup(driver.page_source, 'html.parser')
time.sleep(60)
image_carousel = soup.find_all('li', {"class" : "ProductDetailImageCarouselVariantB-carouselItem"})
print("Number of images in gallery: ", len(image_carousel))
#deal with captcha
while len(image_carousel) <= 0:
print("CAPTCHA ENCOUNTERED. FIX")
soup = BeautifulSoup(driver.page_source, 'html.parser')
image_carousel = soup.find_all('li', {"class" : "ProductDetailImageCarouselVariantB-carouselItem"})
time.sleep(30)
valid_image_links = []
highest_resolution_images = []
#get correct image links
#i = 1
for image_block in image_carousel:
try:
#print("image_block:")
#print(image_block)
#print("Image: ", i)
#i += 1
images = image_block.find('div', {"class" : "ImageComponent ImageComponent--overlay"})
#image_links = images.find('img').get_attribute('srcset').split(',')
print(images)
#driver.implicitly_wait(60)
#wait = WebDriverWait(images, 30)
#image_links = wait.until(EC.visibility_of_element_located((By.tagName, "img"))).get_attribute("srcset").split(',')
#image_links = wait.until(EC.text_to_be_present_in_element_value((By.tagName, 'img'), "https")).get_attribute("srcset").split(',')
#image_links = wait.until(EC.text_to_be_present_in_element_value((By.CSS_SELECTOR, "img [srcset*='https']"), "https")).get_attribute("srcset").split(',')
#image_links = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "img[src*='https']"))).get_attribute("src").split(',')
images.implicitly_wait(30)
image_links = images.find_element_by_tag_name('img').get_attribute('srcset').split(',')
#"div[class='ajax_enabled'] [style='display:block']"
#image_links = images.find('img')['srcset'].split(',')
#print("Image links:")
#print(image_links)
#print("Number of links: ", len(image_links))
for link in image_links:
print(link)
for image_info in image_links:
image_link = image_info.split(" ")[0]
try:
if hasValidBackground(image_link) and hasValidSize(image_link):
valid_image_links.append(image_link)
else:
print("Invalid image size or background")
except:
print('ERROR when reading image: ' + image_link)
continue
if len(valid_image_links) > 0:
highest_resolution_images.append(valid_image_links[-1])
valid_image_links.clear()
except:
print("Error. Invalid image link.")
pass
#extract one link to a correctly filtered image
if len(highest_resolution_images) <= 0:
return -1
valid_image_link = highest_resolution_images[0];
info['img_url'] = valid_image_link
#get price information
standard_price_block = soup.find('div', {"class" : "StandardPriceBlock"})
base_price_block = standard_price_block.find('div', {"class" : "BasePriceBlock BasePriceBlock--highlight"})
if base_price_block is None:
base_price_block = standard_price_block.find('div', {"class" : "BasePriceBlock"})
base_price = base_price_block.find('span').text
#price_block = soup.find('span', {"class" : "notranslate"})
#base_price = standard_price_block.find('span').text
info['price'] = base_price
print(base_price)
#print(f"Image link: {image_link}\n")
#print(f"Link to product: {product_link}\n")
driver.close()
#browser.switch_to.window(browser.window_handles[0])
return info
except TimeoutException as e:
print("Page Load Timeout Occurred. Quitting...")
driver.close()
I was testing using this website:
https://www.wayfair.com/furniture/pdp/foundstone-derry-84-reversible-sectional-w001832490.html
My goal is to process each image in the image gallery/carousel and find one that has white background and has valid size of height >= 80 and width >= 80
I'm just starting to learn web scraping, so any help would be much appreciated!!