HADOOP-18890. Remove use of okhttp in runtime code (#6057)

Contributed by PJ Fanning

LICENSE-binary

@@ -241,8 +241,6 @@ com.google.guava:guava:27.0-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.microsoft.azure:azure-storage:7.0.0
 com.nimbusds:nimbus-jose-jwt:9.31
-com.squareup.okhttp3:okhttp:4.10.0
-com.squareup.okio:okio:3.4.0
 com.zaxxer:HikariCP:4.0.3
 commons-beanutils:commons-beanutils:1.9.4
 commons-cli:commons-cli:1.5.0
@@ -319,8 +317,8 @@ org.apache.hbase:hbase-common:1.7.1
 org.apache.hbase:hbase-protocol:1.7.1
 org.apache.htrace:htrace-core:3.1.0-incubating
 org.apache.htrace:htrace-core4:4.1.0-incubating
-org.apache.httpcomponents:httpclient:4.5.6
+org.apache.httpcomponents:httpclient:4.5.13
-org.apache.httpcomponents:httpcore:4.4.10
+org.apache.httpcomponents:httpcore:4.4.13
 org.apache.kafka:kafka-clients:2.8.2
 org.apache.kerby:kerb-admin:2.0.3
 org.apache.kerby:kerb-client:2.0.3
@@ -357,8 +355,6 @@ org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.51.v20230217
 org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.51.v20230217
 org.ehcache:ehcache:3.3.1
 org.ini4j:ini4j:0.5.4
-org.jetbrains.kotlin:kotlin-stdlib:1.4.10
-org.jetbrains.kotlin:kotlin-stdlib-common:1.4.10
 org.lz4:lz4-java:1.7.1
 org.objenesis:objenesis:2.6
 org.xerial.snappy:snappy-java:1.1.10.1

NOTICE-binary

@@ -334,19 +334,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
------------------------------------------------------------------------
-
-This product contains a modified portion of 'OkHttp', an open source
-HTTP & SPDY client for Android and Java applications, which can be obtained
-at:
-
-  * LICENSE:
-    * okhttp/third_party/okhttp/LICENSE (Apache License 2.0)
-  * HOMEPAGE:
-    * https://github.com/square/okhttp
-  * LOCATION_IN_GRPC:
-    * okhttp/third_party/okhttp
-
 This product contains a modified portion of 'Netty', an open source
 networking library, which can be obtained at:
 

hadoop-client-modules/hadoop-client/pom.xml

@@ -118,18 +118,6 @@
           <groupId>org.eclipse.jetty</groupId>
           <artifactId>jetty-server</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.jetbrains.kotlin</groupId>
-          <artifactId>kotlin-stdlib</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.jetbrains.kotlin</groupId>
-          <artifactId>kotlin-stdlib-common</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.squareup.okhttp3</groupId>
-          <artifactId>okhttp</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-core</artifactId>

hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml

@@ -161,10 +161,6 @@
       <artifactId>esdk-obs-java</artifactId>
       <version>${esdk.version}</version>
       <exclusions>
-        <exclusion>
-          <artifactId>okio</artifactId>
-          <groupId>com.squareup.okio</groupId>
-        </exclusion>
         <exclusion>
           <artifactId>log4j-core</artifactId>
           <groupId>org.apache.logging.log4j</groupId>

hadoop-common-project/hadoop-common/pom.xml

@@ -394,8 +394,8 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>com.squareup.okio</groupId>
+      <groupId>org.jetbrains.kotlin</groupId>
-      <artifactId>okio-jvm</artifactId>
+      <artifactId>kotlin-stdlib-jdk8</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>

hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml

@@ -94,17 +94,4 @@
     <Bug pattern="EI_EXPOSE_REP" />
   </Match>
 
-  <!--okhttp classes from Kotlin are not analysed for NP check. -->
-  <Match>
-    <Class name="org.apache.hadoop.hdfs.web.oauth2.ConfRefreshTokenBasedAccessTokenProvider" />
-    <Method name="refresh" />
-    <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
-  </Match>
-
-  <Match>
-    <Class name="org.apache.hadoop.hdfs.web.oauth2.CredentialBasedAccessTokenProvider" />
-    <Method name="refresh" />
-    <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
-  </Match>
-
 </FindBugsFilter>

hadoop-hdfs-project/hadoop-hdfs-client/pom.xml

@@ -34,28 +34,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
   </properties>
 
   <dependencies>
-    <dependency>
-      <groupId>com.squareup.okhttp3</groupId>
-      <artifactId>okhttp</artifactId>
-      <exclusions>
-        <exclusion>
-          <groupId>com.squareup.okio</groupId>
-          <artifactId>okio-jvm</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-    <dependency>
-      <groupId>com.squareup.okio</groupId>
-      <artifactId>okio-jvm</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.jetbrains.kotlin</groupId>
-      <artifactId>kotlin-stdlib</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.jetbrains.kotlin</groupId>
-      <artifactId>kotlin-stdlib-common</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java

@@ -19,13 +19,10 @@
 package org.apache.hadoop.hdfs.web.oauth2;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import okhttp3.OkHttpClient;
-import okhttp3.Request;
-import okhttp3.RequestBody;
-import okhttp3.Response;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -33,7 +30,17 @@
 import org.apache.hadoop.hdfs.web.URLConnectionFactory;
 import org.apache.hadoop.util.JsonSerialization;
 import org.apache.hadoop.util.Timer;
+import org.apache.http.HttpHeaders;
 import org.apache.http.HttpStatus;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
 
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY;
@@ -103,34 +110,37 @@ public synchronized String getAccessToken() throws IOException {
   }
 
   void refresh() throws IOException {
-    OkHttpClient client =
+    final List<NameValuePair> pairs = new ArrayList<>();
-        new OkHttpClient.Builder().connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT,
+    pairs.add(new BasicNameValuePair(GRANT_TYPE, REFRESH_TOKEN));
-                TimeUnit.MILLISECONDS)
+    pairs.add(new BasicNameValuePair(REFRESH_TOKEN, refreshToken));
-            .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
+    pairs.add(new BasicNameValuePair(CLIENT_ID, clientId));
+    final RequestConfig config = RequestConfig.custom()
+        .setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
+        .setConnectionRequestTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
+        .setSocketTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
         .build();
-
+    try (CloseableHttpClient client =
-    String bodyString =
+             HttpClientBuilder.create().setDefaultRequestConfig(config).build()) {
-        Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, REFRESH_TOKEN, refreshToken, CLIENT_ID, clientId);
+      final HttpPost httpPost = new HttpPost(refreshURL);
-
+      httpPost.setEntity(new UrlEncodedFormEntity(pairs, StandardCharsets.UTF_8));
-    RequestBody body = RequestBody.create(bodyString, URLENCODED);
+      httpPost.setHeader(HttpHeaders.CONTENT_TYPE, URLENCODED);
-
+      try (CloseableHttpResponse response = client.execute(httpPost)) {
-    Request request = new Request.Builder().url(refreshURL).post(body).build();
+        final int statusCode = response.getStatusLine().getStatusCode();
-    try (Response response = client.newCall(request).execute()) {
+        if (statusCode != HttpStatus.SC_OK) {
-      if (!response.isSuccessful()) {
-        throw new IOException("Unexpected code " + response);
-      }
-      if (response.code() != HttpStatus.SC_OK) {
           throw new IllegalArgumentException(
-            "Received invalid http response: " + response.code() + ", text = "
+              "Received invalid http response: " + statusCode + ", text = " +
-                + response.toString());
+                  EntityUtils.toString(response.getEntity()));
         }
-
+        Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(
-      Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(response.body().string());
+            EntityUtils.toString(response.getEntity()));
 
         String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
         accessTokenTimer.setExpiresIn(newExpiresIn);
 
         accessToken = responseBody.get(ACCESS_TOKEN).toString();
+      }
+    } catch (RuntimeException e) {
+      throw new IOException("Exception while refreshing access token", e);
     } catch (Exception e) {
       throw new IOException("Exception while refreshing access token", e);
     }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java

@@ -19,13 +19,10 @@
 package org.apache.hadoop.hdfs.web.oauth2;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import okhttp3.OkHttpClient;
-import okhttp3.Request;
-import okhttp3.RequestBody;
-import okhttp3.Response;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -33,7 +30,17 @@
 import org.apache.hadoop.hdfs.web.URLConnectionFactory;
 import org.apache.hadoop.util.JsonSerialization;
 import org.apache.hadoop.util.Timer;
+import org.apache.http.HttpHeaders;
 import org.apache.http.HttpStatus;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
 
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY;
@@ -97,38 +104,37 @@ public synchronized String getAccessToken() throws IOException {
   }
 
   void refresh() throws IOException {
-    OkHttpClient client = new OkHttpClient.Builder()
+    final List<NameValuePair> pairs = new ArrayList<>();
-            .connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
+    pairs.add(new BasicNameValuePair(CLIENT_SECRET, getCredential()));
-            .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
+    pairs.add(new BasicNameValuePair(GRANT_TYPE, CLIENT_CREDENTIALS));
+    pairs.add(new BasicNameValuePair(CLIENT_ID, clientId));
+    final RequestConfig config = RequestConfig.custom()
+        .setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
+        .setConnectionRequestTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
+        .setSocketTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
         .build();
-
+    try (CloseableHttpClient client =
-    String bodyString = Utils.postBody(CLIENT_SECRET, getCredential(),
+             HttpClientBuilder.create().setDefaultRequestConfig(config).build()) {
-        GRANT_TYPE, CLIENT_CREDENTIALS,
+      final HttpPost httpPost = new HttpPost(refreshURL);
-        CLIENT_ID, clientId);
+      httpPost.setEntity(new UrlEncodedFormEntity(pairs, StandardCharsets.UTF_8));
-
+      httpPost.setHeader(HttpHeaders.CONTENT_TYPE, URLENCODED);
-    RequestBody body = RequestBody.create(bodyString, URLENCODED);
+      try (CloseableHttpResponse response = client.execute(httpPost)) {
-
+        final int statusCode = response.getStatusLine().getStatusCode();
-    Request request = new Request.Builder()
+        if (statusCode != HttpStatus.SC_OK) {
-        .url(refreshURL)
+          throw new IllegalArgumentException(
-        .post(body)
+              "Received invalid http response: " + statusCode + ", text = " +
-        .build();
+                  EntityUtils.toString(response.getEntity()));
-    try (Response response = client.newCall(request).execute()) {
-      if (!response.isSuccessful()) {
-        throw new IOException("Unexpected code " + response);
         }
-
-      if (response.code() != HttpStatus.SC_OK) {
-        throw new IllegalArgumentException("Received invalid http response: "
-            + response.code() + ", text = " + response.toString());
-      }
-
         Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(
-          response.body().string());
+            EntityUtils.toString(response.getEntity()));
 
         String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
         timer.setExpiresIn(newExpiresIn);
 
         accessToken = responseBody.get(ACCESS_TOKEN).toString();
+      }
+    } catch (RuntimeException e) {
+      throw new IOException("Unable to obtain access token from credential", e);
     } catch (Exception e) {
       throw new IOException("Unable to obtain access token from credential", e);
     }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java

@@ -18,7 +18,6 @@
  */
 package org.apache.hadoop.hdfs.web.oauth2;
 
-import okhttp3.MediaType;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
@@ -30,8 +29,8 @@
 public final class OAuth2Constants {
   private OAuth2Constants() { /** Private constructor. **/ }
 
-  public static final MediaType URLENCODED
+  public static final String URLENCODED
-      = MediaType.parse("application/x-www-form-urlencoded; charset=utf-8");
+      = "application/x-www-form-urlencoded; charset=utf-8";
 
   /* Constants for OAuth protocol */
   public static final String ACCESS_TOKEN = "access_token";

hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml

@@ -205,8 +205,8 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>com.squareup.okhttp3</groupId>
+      <groupId>org.jetbrains.kotlin</groupId>
-      <artifactId>okhttp</artifactId>
+      <artifactId>kotlin-stdlib-jdk8</artifactId>
       <scope>test</scope>
     </dependency>
   </dependencies>

hadoop-project/pom.xml

@@ -131,10 +131,8 @@
     <hikari.version>4.0.3</hikari.version>
     <derby.version>10.14.2.0</derby.version>
     <mssql.version>6.2.1.jre7</mssql.version>
-    <okhttp3.version>4.10.0</okhttp3.version>
+    <okhttp3.version>4.11.0</okhttp3.version>
-    <okio.version>3.4.0</okio.version>
+    <kotlin-stdlib.version>1.6.20</kotlin-stdlib.version>
-    <kotlin-stdlib.verion>1.6.20</kotlin-stdlib.verion>
-    <kotlin-stdlib-common.version>1.6.20</kotlin-stdlib-common.version>
     <jdom2.version>2.0.6.1</jdom2.version>
     <jna.version>5.2.0</jna.version>
     <gson.version>2.9.0</gson.version>
@@ -220,62 +218,24 @@
 
   <dependencyManagement>
     <dependencies>
-      <dependency>
-        <groupId>com.squareup.okhttp3</groupId>
-        <artifactId>okhttp</artifactId>
-        <version>${okhttp3.version}</version>
-        <exclusions>
-          <exclusion>
-            <groupId>org.jetbrains.kotlin</groupId>
-            <artifactId>kotlin-stdlib</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.jetbrains.kotlin</groupId>
-            <artifactId>kotlin-stdlib-common</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>com.squareup.okio</groupId>
-            <artifactId>okio-jvm</artifactId>
-          </exclusion>
-        </exclusions>
-      </dependency>
-      <dependency>
-        <groupId>com.squareup.okio</groupId>
-        <artifactId>okio-jvm</artifactId>
-        <version>${okio.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.jetbrains.kotlin</groupId>
-        <artifactId>kotlin-stdlib</artifactId>
-        <version>${kotlin-stdlib.verion}</version>
-        <exclusions>
-          <exclusion>
-            <groupId>org.jetbrains</groupId>
-            <artifactId>annotations</artifactId>
-          </exclusion>
-        </exclusions>
-      </dependency>
-      <dependency>
-        <groupId>org.jetbrains.kotlin</groupId>
-        <artifactId>kotlin-stdlib-common</artifactId>
-        <version>${kotlin-stdlib-common.version}</version>
-      </dependency>
       <dependency>
         <groupId>com.squareup.okhttp3</groupId>
         <artifactId>mockwebserver</artifactId>
         <version>${okhttp3.version}</version>
         <scope>test</scope>
         <exclusions>
-          <exclusion>
-            <groupId>com.squareup.okio</groupId>
-            <artifactId>okio-jvm</artifactId>
-          </exclusion>
           <exclusion>
             <groupId>org.jetbrains.kotlin</groupId>
             <artifactId>kotlin-stdlib-jdk8</artifactId>
           </exclusion>
         </exclusions>
       </dependency>
+      <dependency>
+        <groupId>org.jetbrains.kotlin</groupId>
+        <artifactId>kotlin-stdlib-jdk8</artifactId>
+        <version>${kotlin-stdlib.version}</version>
+        <scope>test</scope>
+      </dependency>
       <dependency>
         <groupId>jdiff</groupId>
         <artifactId>jdiff</artifactId>