Selenium + Xvfb concurrency issues - google-chrome

Question

test.py
#!/usr/bin/env python2
import os
from pyvirtualdisplay import Display
from selenium import webdriver
import sys
display = Display(visible=0, size=(800, 600))
display.start()
try:
capabilities = webdriver.common.desired_capabilities.DesiredCapabilities.CHROME
browser = webdriver.Chrome('/opt/chrome-driver/chromedriver', desired_capabilities=capabilities)
try:
browser.get('http://example.com')
browser.execute_script('return 1 + 1;')
finally:
browser.quit()
finally:
display.stop()
print 'Done'
And then run
seq 1 20 | parallel -j 5 ./test.py
I get a random number of errors like this:
Traceback (most recent call last):
File "./test.py", line 15, in <module>
browser.execute_script('return 1 + 1;')
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webdriver.py", line 429, in execute_script
{'script': script, 'args':converted_args})['value']
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webdriver.py", line 201, in execute
self.error_handler.check_response(response)
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/errorhandler.py", line 194, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message: chrome not reachable
(Session info: chrome=50.0.2661.86)
(Driver info: chromedriver=2.21.371461 (633e689b520b25f3e264a2ede6b74ccc23cb636a),platform=Linux 3.19.0-59-generic x86_64)
If I remove the concurrency (parallel -j 1), it works.
If I remove Xvfb, it works.
What does this error mean, and how can I fix it (without giving up concurrency or virtual displays)?

Answer 1

Looks like starting xvfb has concurrency issues when starting. xvfb-run unreliable when multiple instances invoked in parallel
So I can fix this by requiring that xfvb sessions are started serially.
#!/usr/bin/env python2
import fcntl
import os
from pyvirtualdisplay import Display
from selenium import webdriver
import sys
import xvfbwrapper
display = None
browser = None
try:
with open('/tmp/xvfb.lock', 'w') as lock:
fcntl.lockf(lock, fcntl.LOCK_EX)
display = Display(visible=0, size=(800, 600))
display.start()
capabilities = webdriver.common.desired_capabilities.DesiredCapabilities.CHROME
browser = webdriver.Chrome('/opt/chrome-driver/chromedriver', desired_capabilities=capabilities)
browser.get('http://example.com')
browser.execute_script('return 1 + 1;')
finally:
if browser:
browser.quit()
if display:
display.stop()
print 'Done'
As long as everybody uses /tmp/xvfb.lock to control starting a Xvfb session starting, this works.
P.S. I don't think the selenium driver starting needed to be serialized too, but I did run into a transient problem that I think was fixed by this.