TestHttpClientRequest

package com.atlassian.sal.core.net;

import com.atlassian.sal.api.net.Request;
import com.atlassian.sal.api.net.ResponseException;
import com.google.common.base.Throwables;
import org.apache.commons.codec.binary.Base64;
import org.apache.http.Header;
import org.apache.http.HttpClientConnection;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpException;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.ProtocolVersion;
import org.apache.http.auth.AUTH;
import org.apache.http.client.RedirectException;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.conn.ConnectionRequest;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.entity.StringEntity;
import org.apache.http.message.BasicHttpResponse;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HttpContext;
import org.apache.http.protocol.HttpRequestExecutor;
import org.hamcrest.FeatureMatcher;
import org.hamcrest.Matcher;
import org.hamcrest.Matchers;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.contrib.java.lang.system.RestoreSystemProperties;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.junit.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;

import java.io.IOException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.text.MessageFormat;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;

import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.notNullValue;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.arrayContaining;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.typeCompatibleWith;
import static org.hamcrest.core.IsEqual.equalTo;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

@RunWith(MockitoJUnitRunner.class)
public class TestHttpClientRequest {

    @Rule
    public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties();

    private static final String DUMMY_HOST = "dummy.atlassian.test";
    private static final String DUMMY_HTTP_URL = MessageFormat.format("http://{0}:8090/", DUMMY_HOST);

    private static final String DUMMY_PROXY_HOST = "dummy.proxy.atlassian.test";
    private static final String DUMMY_PROXY_PORT = "12345";
    private static final String DUMMY_PROXY_USER = "dummyproxyuser";
    private static final String DUMMY_PROXY_PASSWORD = "dummyproxypassword";

    @Rule
    public ExpectedException thrown = ExpectedException.none();

    @Mock
    private HttpRequestExecutor mockRequestExecutor;

    @Mock
    private HttpClientConnectionManager mockConnectionManager;

    private HttpClientRequestFactory requestFactory;
    private ArgumentCaptor<HttpRequest> requestCaptor = ArgumentCaptor.forClass(HttpRequest.class);
    private ArgumentCaptor<HttpRoute> routeCaptor = ArgumentCaptor.forClass(HttpRoute.class);

    @Before
    public void setup() throws InterruptedException, ExecutionException, IOException, HttpException {
        // Clear all system proxy settings
        requestFactory = new HttpClientWithMockConnectionRequestFactory(mockConnectionManager, mockRequestExecutor);

        // Always respond with a 200/OK message
        when(mockRequestExecutor.execute(any(HttpRequest.class), any(HttpClientConnection.class),
                any(HttpContext.class))).thenReturn(createOkResponse());

        // This allows us to hook in to the connection details that HttpClient would have made
        final HttpClientConnection conn = mock(HttpClientConnection.class);
        final ConnectionRequest connRequest = mock(ConnectionRequest.class);
        when(connRequest.get(anyLong(), any(TimeUnit.class))).thenReturn(conn);
        when(mockConnectionManager.requestConnection(any(HttpRoute.class), any())).thenReturn(connRequest);
    }

    private static HttpResponse createOkResponse() {
        final BasicHttpResponse response = new BasicHttpResponse(new ProtocolVersion("HTTP", 1, 1), HttpStatus.SC_OK, "OK");
        response.setEntity(new StringEntity("test body", StandardCharsets.UTF_8));
        return response;
    }

    private static HttpResponse createRedirectResponse(String location) {
        final BasicHttpResponse response = new BasicHttpResponse(new ProtocolVersion("HTTP", 1, 1), HttpStatus.SC_MOVED_PERMANENTLY, "Redirect");
        response.setEntity(new StringEntity("Redirect", StandardCharsets.UTF_8));
        response.setHeader("Location", location);
        return response;
    }

    private static HttpResponse createNoContentResponse() {
        return new BasicHttpResponse(new ProtocolVersion("HTTP", 1, 1), HttpStatus.SC_NO_CONTENT, "No Content");
    }

    @Test
    public void assertThatHeaderIsAddedToRequest() throws ResponseException, IOException, HttpException {
        final HttpClientRequest request = requestFactory.createRequest(Request.MethodType.GET, DUMMY_HTTP_URL);

        final String testHeaderName = "foo";
        final String testHeaderValue = "bar";
        request.addHeader(testHeaderName, testHeaderValue);
        request.execute();

        verify(mockRequestExecutor).execute(requestCaptor.capture(), any(HttpClientConnection.class), any(HttpContext.class));
        final HttpRequest lastRequest = requestCaptor.getValue();
        assertThat(lastRequest, notNullValue());

        final Header[] headers = lastRequest.getHeaders(testHeaderName);
        //noinspection unchecked
        assertThat(headers, arrayContaining(headerWithValue(equalTo(testHeaderValue))));
    }

    @Test
    public void assertThatMultiValuedHeadersAreAddedToRequest() throws ResponseException, IOException, HttpException {
        final HttpClientRequest request = requestFactory.createRequest(Request.MethodType.GET, DUMMY_HTTP_URL);

        final String testHeaderName = "foo";
        final String testHeaderValue1 = "bar1";
        final String testHeaderValue2 = "bar2";
        request.addHeader(testHeaderName, testHeaderValue1);
        request.addHeader(testHeaderName, testHeaderValue2);
        request.execute();

        verify(mockRequestExecutor).execute(requestCaptor.capture(), any(HttpClientConnection.class), any(HttpContext.class));
        final HttpRequest lastRequest = requestCaptor.getValue();
        assertThat(lastRequest, notNullValue());

        final Header[] headers = lastRequest.getHeaders(testHeaderName);
        //noinspection unchecked
        assertThat(headers, arrayContaining(
                headerWithValue(equalTo(testHeaderValue1)),
                headerWithValue(equalTo(testHeaderValue2))));
    }

    @Test
    public void testThatRouteIsNotCached() throws Exception {
        System.setProperty(SystemPropertiesProxyConfig.PROXY_HOST_PROPERTY_NAME, DUMMY_PROXY_HOST);
        System.setProperty(SystemPropertiesProxyConfig.PROXY_PORT_PROPERTY_NAME, DUMMY_PROXY_PORT);
        final HttpClientRequestFactory factory = new HttpClientWithMockConnectionRequestFactory(mockConnectionManager, mockRequestExecutor);

        HttpClientRequest request = factory.createRequest(Request.MethodType.GET, DUMMY_HTTP_URL);
        request.execute();

        System.setProperty(SystemPropertiesProxyConfig.PROXY_HOST_PROPERTY_NAME, "hostName");
        System.setProperty(SystemPropertiesProxyConfig.PROXY_PORT_PROPERTY_NAME, "1234");

        request = requestFactory.createRequest(Request.MethodType.GET, DUMMY_HTTP_URL);
        request.execute();

        verify(mockConnectionManager, times(2)).connect(any(HttpClientConnection.class), routeCaptor.capture(), anyInt(), any(HttpContext.class
        ));
        List<HttpRoute> routes = routeCaptor.getAllValues();
        assertThat(routes.get(0), proxyHostIs(hostWithNameAndPort(DUMMY_PROXY_HOST, Integer.parseInt(DUMMY_PROXY_PORT))));
        assertThat(routes.get(1), proxyHostIs(hostWithNameAndPort("hostName", Integer.parseInt("1234"))));
    }

    @Test
    public void assertThatParameterIsAddedToRequest() throws IOException, ResponseException, HttpException {
        final HttpClientRequest request = requestFactory.createRequest(Request.MethodType.POST, DUMMY_HTTP_URL);

        final String testParameterName = "foo";
        final String testParameterValue = "bar";

        request.addRequestParameters(testParameterName, testParameterValue);
        request.execute();

        verify(mockRequestExecutor).execute(requestCaptor.capture(), any(HttpClientConnection.class), any(HttpContext.class));
        final HttpRequest lastRequest = requestCaptor.getValue();

        assertThat(lastRequest, notNullValue());
        assertThat(lastRequest.getClass(), is(typeCompatibleWith(HttpEntityEnclosingRequest.class)));
        assertThat(lastRequest, requestParameters(contains(parameterWithNameAndValue(
                testParameterName, testParameterValue))));
    }

    @Test
    public void assertThatMultiValuedParametersAreAddedToRequest() throws IOException, ResponseException, HttpException {
        final HttpClientRequest request = requestFactory.createRequest(Request.MethodType.POST, DUMMY_HTTP_URL);

        final String testParameterName = "foo";
        final String testParameterValue1 = "bar1";
        final String testParameterValue2 = "bar2";

        request.addRequestParameters(testParameterName, testParameterValue1, testParameterName, testParameterValue2);
        request.execute();

        verify(mockRequestExecutor).execute(requestCaptor.capture(), any(HttpClientConnection.class), any(HttpContext.class));
        final HttpRequest lastRequest = requestCaptor.getValue();

        assertThat(lastRequest, notNullValue());
        assertThat(lastRequest.getClass(), is(typeCompatibleWith(HttpEntityEnclosingRequest.class)));
        //noinspection unchecked
        assertThat(lastRequest, requestParameters(contains(
                parameterWithNameAndValue(testParameterName, testParameterValue1),
                parameterWithNameAndValue(testParameterName, testParameterValue2)
        )));
    }

    @Test
    public void assertThatParametersAreAddedWithMultipleCalls() throws IOException, ResponseException, HttpException {
        final HttpClientRequest request = requestFactory.createRequest(Request.MethodType.POST, DUMMY_HTTP_URL);

        final String testParameterName = "foo";
        final String testParameterValue1 = "bar1";
        final String testParameterValue2 = "bar2";

        request.addRequestParameters(testParameterName, testParameterValue1);
        request.addRequestParameters(testParameterName, testParameterValue2);
        request.execute();

        verify(mockRequestExecutor).execute(requestCaptor.capture(), any(HttpClientConnection.class), any(HttpContext.class));
        final HttpRequest lastRequest = requestCaptor.getValue();

        assertThat(lastRequest, notNullValue());
        assertThat(lastRequest.getClass(), is(typeCompatibleWith(HttpEntityEnclosingRequest.class)));
        //noinspection unchecked
        assertThat(lastRequest, requestParameters(contains(
                parameterWithNameAndValue(testParameterName, testParameterValue1),
                parameterWithNameAndValue(testParameterName, testParameterValue2)
        )));
    }

    @Test
    public void assertThatAddingParametersToGetRequestThrowsException() {
        final HttpClientRequest request = requestFactory.createRequest(Request.MethodType.GET, DUMMY_HTTP_URL);
        thrown.expect(IllegalStateException.class);
        request.addRequestParameters("foo", "bar");
    }

    @Test
    public void assertThatSystemProxyHostSettingHonoured() throws IOException, ResponseException, HttpException {
        System.setProperty(SystemPropertiesProxyConfig.PROXY_HOST_PROPERTY_NAME, DUMMY_PROXY_HOST);
        System.setProperty(SystemPropertiesProxyConfig.PROXY_PORT_PROPERTY_NAME, DUMMY_PROXY_PORT);

        final HttpClientRequestFactory factory = new HttpClientWithMockConnectionRequestFactory(mockConnectionManager, mockRequestExecutor);
        final HttpClientRequest request = factory.createRequest(Request.MethodType.GET, DUMMY_HTTP_URL);
        request.execute();

        verify(mockConnectionManager).connect(any(HttpClientConnection.class), routeCaptor.capture(), anyInt(), any(HttpContext.class));
        HttpRoute route = routeCaptor.getValue();
        assertThat(route, proxyHostIs(hostWithNameAndPort(DUMMY_PROXY_HOST, Integer.parseInt(DUMMY_PROXY_PORT))));
    }

    @Test
    public void assertThatProxyNotUsedByDefault() throws IOException, ResponseException, HttpException {
        final HttpClientRequestFactory factory = new HttpClientWithMockConnectionRequestFactory(mockConnectionManager, mockRequestExecutor);
        final HttpClientRequest request = factory.createRequest(Request.MethodType.GET, DUMMY_HTTP_URL);
        request.execute();

        verify(mockConnectionManager).connect(any(HttpClientConnection.class), routeCaptor.capture(), anyInt(), any(HttpContext.class));
        final HttpRoute route = routeCaptor.getValue();
        assertThat(route, proxyHostIs(Matchers.nullValue(HttpHost.class)));
    }

    @Test
    public void assertThatNonProxyHostsSystemPropertyHonoured() throws IOException, ResponseException, HttpException {
        System.setProperty(SystemPropertiesProxyConfig.PROXY_HOST_PROPERTY_NAME, DUMMY_PROXY_HOST);
        System.setProperty(SystemPropertiesProxyConfig.PROXY_PORT_PROPERTY_NAME, DUMMY_PROXY_PORT);
        System.setProperty(SystemPropertiesProxyConfig.PROXY_NON_HOSTS_PROPERTY_NAME, "localhost");

        final HttpClientRequestFactory factory = new HttpClientWithMockConnectionRequestFactory(mockConnectionManager, mockRequestExecutor);

        // deliberatly set after factory is created to match
        // how jira and confluence use it
        System.setProperty(SystemPropertiesProxyConfig.PROXY_NON_HOSTS_PROPERTY_NAME, "*.notproxied.test|localhost");

        final HttpClientRequest request = factory.createRequest(Request.MethodType.GET, DUMMY_HTTP_URL);

        request.execute();
        request.setUrl("http://www.notproxied.test").execute();

        verify(mockConnectionManager, times(2)).connect(any(HttpClientConnection.class), routeCaptor.capture(), anyInt(), any(HttpContext.class));

        final List<HttpRoute> routes = routeCaptor.getAllValues();

        //noinspection unchecked
        assertThat(routes, contains(
                proxyHostIs(hostWithNameAndPort(DUMMY_PROXY_HOST, Integer.parseInt(DUMMY_PROXY_PORT))),
                proxyHostIs(Matchers.nullValue(HttpHost.class))
        ));
    }

    @Test
    public void assertThatProxyAuthenticationSystemPropertyHonoured() throws IOException, ResponseException, HttpException {
        System.setProperty(SystemPropertiesProxyConfig.PROXY_HOST_PROPERTY_NAME, DUMMY_PROXY_HOST);
        System.setProperty(SystemPropertiesProxyConfig.PROXY_PORT_PROPERTY_NAME, DUMMY_PROXY_PORT);
        System.setProperty(SystemPropertiesProxyConfig.PROXY_USER_PROPERTY_NAME, DUMMY_PROXY_USER);
        System.setProperty(SystemPropertiesProxyConfig.PROXY_PASSWORD_PROPERTY_NAME, DUMMY_PROXY_PASSWORD);

        final HttpClientRequestFactory factory = new HttpClientWithMockConnectionRequestFactory(mockConnectionManager, mockRequestExecutor);
        final HttpClientRequest request = factory.createRequest(Request.MethodType.GET, DUMMY_HTTP_URL);
        final String username = "charlie";
        final String password = "password123";
        request.addBasicAuthentication(DUMMY_HOST, username, password);
        request.execute();
        verify(mockConnectionManager).connect(any(HttpClientConnection.class), routeCaptor.capture(), anyInt(), any(HttpContext.class));
        final HttpRoute route = routeCaptor.getValue();
        assertThat(route, proxyHostIs(hostWithNameAndPort(DUMMY_PROXY_HOST, Integer.parseInt(DUMMY_PROXY_PORT))));

        verify(mockRequestExecutor).execute(requestCaptor.capture(), any(HttpClientConnection.class), any(HttpContext.class));
        final HttpRequest lastRequest = requestCaptor.getValue();
        final Header[] headers = lastRequest.getHeaders(AUTH.PROXY_AUTH_RESP);

        //noinspection unchecked
        assertThat(headers, arrayContaining(headerWithValue(
                basicAuthWithUsernameAndPassword(DUMMY_PROXY_USER, DUMMY_PROXY_PASSWORD))));
    }

    private void assertThatBasicAuthenticationHeadersAdded(String url) throws Exception {
        final HttpClientRequest request = requestFactory.createRequest(Request.MethodType.GET, url);

        final String username = "charlie";
        final String password = "password123";
        final URI uri = new URI(url);
        request.addBasicAuthentication(uri.getHost(), username, password);
        request.execute();

        verify(mockRequestExecutor).execute(requestCaptor.capture(), any(HttpClientConnection.class), any(HttpContext.class));
        final HttpRequest lastRequest = requestCaptor.getValue();
        final Header[] headers = lastRequest.getHeaders(AUTH.WWW_AUTH_RESP);

        //noinspection unchecked
        assertThat(headers, arrayContaining(headerWithValue(basicAuthWithUsernameAndPassword(username, password))));
    }

    @Test
    public void assertThatBasicAuthenticationHeadersAddedDefaultPort() throws Exception {
        assertThatBasicAuthenticationHeadersAdded(MessageFormat.format("http://{0}/", DUMMY_HOST));
    }

    @Test
    public void assertThatBasicAuthenticationHeadersAddedNotDefaultPort() throws Exception {
        assertThatBasicAuthenticationHeadersAdded(MessageFormat.format("http://{0}:8090/", DUMMY_HOST));
    }

    @Test
    public void assertThatBasicAuthenticationHeadersAddedForHttpsNotDefaultPort() throws Exception {
        assertThatBasicAuthenticationHeadersAdded(MessageFormat.format("https://{0}:8090/", DUMMY_HOST));
    }

    @Test
    public void assertThatBasicAuthenticationHeadersAddedForHttpsDefaultPort() throws Exception {
        assertThatBasicAuthenticationHeadersAdded(MessageFormat.format("https://{0}/", DUMMY_HOST));
    }

    @Test
    public void assertThatExceptionThrownAfterDefaultMaximumRedirects() throws ResponseException, IOException, HttpException {

        when(mockRequestExecutor.execute(any(HttpRequest.class), any(HttpClientConnection.class),
                any(HttpContext.class))).thenAnswer(new Answer<HttpResponse>() {
            int redirectCount = 0;

            @Override
            public HttpResponse answer(final InvocationOnMock invocationOnMock) throws Throwable {
                // We just add a changing query string parameter here to avoid a circular redirect
                return createRedirectResponse(DUMMY_HTTP_URL + "?redirect_count=" + redirectCount++);
            }
        });

        final HttpClientRequest request = requestFactory.createRequest(Request.MethodType.GET, DUMMY_HTTP_URL);

        try {
            request.execute();
            fail("An exception should be thrown when maximum redirects is exceeded.");
        } catch (ResponseException expectedException) {
            // Although JUnit has an ExpectedException rule, we need to catch this both to ensure that a
            // RedirectException is the cause, and to subsequently verify the execution method call count.
            assertThat(Throwables.getCausalChain(expectedException), Matchers.<Throwable>hasItem(instanceOf(RedirectException.class)));
        }

        verify(mockRequestExecutor, times(SystemPropertiesConnectionConfig.DEFAULT_MAX_REDIRECTS + 1))
                .execute(any(HttpRequest.class), any(HttpClientConnection.class), any(HttpContext.class));
    }

    @Test
    public void assertThatNoContentDoesNotThrowNPE() throws ResponseException, IOException, HttpException {
        when(mockRequestExecutor.execute(any(HttpRequest.class), any(HttpClientConnection.class),
                any(HttpContext.class))).thenAnswer(new Answer<HttpResponse>() {
            @Override
            public HttpResponse answer(final InvocationOnMock invocationOnMock) throws Throwable {
                return createNoContentResponse();
            }
        });
        final HttpClientRequest request = requestFactory.createRequest(Request.MethodType.GET, DUMMY_HTTP_URL);
        request.execute();
    }

    @Test
    public void assertThatRequestParameterIsUTF8Encoded() throws ResponseException, IOException, HttpException
    {
        final HttpClientRequest request = requestFactory.createRequest(Request.MethodType.POST, DUMMY_HTTP_URL);
        final String testParameterName = "test";
        final String testParameterValue = "тест";
        request.addRequestParameters(testParameterName, testParameterValue);

        request.execute();

        verify(mockRequestExecutor).execute(requestCaptor.capture(), any(HttpClientConnection.class), any(HttpContext.class));
        final HttpRequest lastRequest = requestCaptor.getValue();

        //noinspection unchecked
        assertThat(lastRequest, requestParameters(contains(
                parameterWithNameAndValue(testParameterName, testParameterValue)
        )));
    }

    private static Matcher<Header> headerWithValue(final Matcher<String> valueMatcher) {
        return new FeatureMatcher<Header, String>(valueMatcher, "header with value", "header value") {
            @Override
            protected String featureValueOf(final Header header) {
                return header.getValue();
            }
        };
    }

    private static Matcher<HttpRequest> requestParameters(final Matcher<Iterable<? extends NameValuePair>> parametersMatcher) {
        return new FeatureMatcher<HttpRequest, Iterable<? extends NameValuePair>>(
                parametersMatcher, "parameters with value", "parameters value") {
            @Override
            protected Iterable<? extends NameValuePair> featureValueOf(final HttpRequest httpRequest) {
                try {
                    return URLEncodedUtils.parse(((HttpEntityEnclosingRequest) httpRequest).getEntity());
                } catch (IOException e) {
                    throw new RuntimeException(e);
                }
            }
        };
    }

    private static Matcher<NameValuePair> parameterWithNameAndValue(final String name, final String value) {

        final NameValuePair expectedParameter = new BasicNameValuePair(name, value);
        return Matchers.equalTo(expectedParameter);
    }

    private static Matcher<HttpRoute> proxyHostIs(final Matcher<HttpHost> routeMatcher) {
        return new FeatureMatcher<HttpRoute, HttpHost>(routeMatcher, "proxy host for route", "proxy host") {
            @Override
            protected HttpHost featureValueOf(final HttpRoute route) {
                return route.getProxyHost();
            }
        };
    }

    private static Matcher<HttpHost> hostWithNameAndPort(final String name, final int port) {

        final HttpHost expectedHost = new HttpHost(name, port);
        return Matchers.equalTo(expectedHost);
    }

    private static Matcher<String> basicAuthWithUsernameAndPassword(final String username, final String password) {

        final String basicAuthCreds = MessageFormat.format("{0}:{1}", username, password);
        final String encodedBasicAuthCreds = Base64.encodeBase64String(basicAuthCreds.getBytes());
        final String expectedBasicAuthHeader = MessageFormat.format("{0} {1}", AuthSchemes.BASIC, encodedBasicAuthCreds);

        return Matchers.equalTo(expectedBasicAuthHeader);
    }

}