Apache HTTPClient DigestAuth doesn't forward "opaque" value from Challenge - apache-httpclient-4.x

Question

I'm trying to use Digest authentication with HTTP Client against a 3rd-party web service that I don't control.
I started out with the sample code from here:
http://hc.apache.org/httpcomponents-client-4.5.x/httpclient/examples/org/apache/http/examples/client/ClientPreemptiveDigestAuthentication.java
I got it working against httpbin.org, before attempting the next step described below.
It appears that the target 3rd-party service that I'm using requires the opaque value to be copied from the WWW-Authentication header on the initial response to the Authorization header on the next request, as described here:
https://security.stackexchange.com/questions/24425/what-is-the-opaque-field-in-http-digest-access-authentication-used-for
However, I have turned on wire-logging and stepped through the code (again this is really just the sample code linked above, no need to copy/paste it here) and I see that the opaque is NOT copied.
Any ideas what prevents it from being copied?
I even tried overriding the processChallenge method:
DigestScheme digestAuth = new DigestScheme() {
#Override
public void processChallenge(
Header header) throws MalformedChallengeException {
but it appears that any value introduced into the Parameters at this point is ignored in the next request.

Answer 1

Finally fixed by overriding the Authorize header explicitly, instead of relying on the internals of HttpClient to do it automatically:
package [...];
import java.util.Arrays;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
import org.apache.http.*;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.*;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.auth.DigestScheme;
import org.apache.http.impl.client.*;
import org.testng.Assert;
public class DigestTest {
private static final String URL
= "https://...";
private static final String PASSWORD = ...;
private static final String USER = ...;
public static void main(String[] args) throws Exception {
new DigestTest().run();
}
public void run() throws Exception {
HttpGet httpget = new HttpGet(URL);
HttpHost target
= new HttpHost(httpget.getURI().getHost(), 443, "https");
CredentialsProvider credsProvider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials
= new UsernamePasswordCredentials(USER, PASSWORD);
credsProvider.setCredentials(
new AuthScope(target.getHostName(), target.getPort()),
credentials);
CookieStore cookieStore = new BasicCookieStore();
CloseableHttpClient httpclient
= HttpClients.custom().setDefaultCookieStore(cookieStore)
.setDefaultCredentialsProvider(credsProvider).build();
try {
DigestScheme digestAuth = new DigestScheme();
digestAuth.overrideParamter("qop", "auth");
digestAuth.overrideParamter("nc", "0");
digestAuth.overrideParamter("cnonce", DigestScheme.createCnonce());
AuthCache authCache = new BasicAuthCache();
authCache.put(target, digestAuth);
HttpClientContext localContext = HttpClientContext.create();
localContext.setAuthCache(authCache);
CloseableHttpResponse response;
response = httpclient.execute(target, httpget, localContext);
Map<String, String> wwwAuth = Arrays
.stream(response.getHeaders("WWW-Authenticate")[0]
.getElements())
.collect(Collectors.toMap(HeaderElement::getName,
HeaderElement::getValue));
// the first call ALWAYS fails with a 401
Assert.assertEquals(response.getStatusLine().getStatusCode(), 401);
digestAuth.overrideParamter("opaque", wwwAuth.get("opaque"));
digestAuth.overrideParamter("nonce", wwwAuth.get("nonce"));
digestAuth.overrideParamter("realm", wwwAuth.get("Digest realm"));
Header authenticate = digestAuth.authenticate(credentials, httpget,
localContext);
httpget.addHeader(authenticate);
response = httpclient.execute(target, httpget, localContext);
// the 2nd call is the real deal
Assert.assertEquals(response.getStatusLine().getStatusCode(), 200);
System.out.println(IOUtils
.toString(response.getEntity().getContent(), "utf-8"));
} finally {
httpclient.close();
}
}
}