From c16484ffb28549dc6883034cb500810f8d5a0fbe Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Tue, 19 Sep 2023 12:38:36 +0100 Subject: [PATCH] HADOOP-18890. Remove use of okhttp in runtime code (#6057) Contributed by PJ Fanning --- LICENSE-binary | 8 +- NOTICE-binary | 13 ---- hadoop-client-modules/hadoop-client/pom.xml | 12 --- .../hadoop-huaweicloud/pom.xml | 4 - hadoop-common-project/hadoop-common/pom.xml | 4 +- .../dev-support/findbugsExcludeFile.xml | 13 ---- .../hadoop-hdfs-client/pom.xml | 22 ------ ...fRefreshTokenBasedAccessTokenProvider.java | 72 +++++++++-------- .../CredentialBasedAccessTokenProvider.java | 78 ++++++++++--------- .../hdfs/web/oauth2/OAuth2Constants.java | 5 +- .../hadoop-hdfs-httpfs/pom.xml | 4 +- hadoop-project/pom.xml | 56 ++----------- 12 files changed, 99 insertions(+), 192 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index e0c5923d04..5a653da811 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -241,8 +241,6 @@ com.google.guava:guava:27.0-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.microsoft.azure:azure-storage:7.0.0 com.nimbusds:nimbus-jose-jwt:9.31 -com.squareup.okhttp3:okhttp:4.10.0 -com.squareup.okio:okio:3.4.0 com.zaxxer:HikariCP:4.0.3 commons-beanutils:commons-beanutils:1.9.4 commons-cli:commons-cli:1.5.0 @@ -319,8 +317,8 @@ org.apache.hbase:hbase-common:1.7.1 org.apache.hbase:hbase-protocol:1.7.1 org.apache.htrace:htrace-core:3.1.0-incubating org.apache.htrace:htrace-core4:4.1.0-incubating -org.apache.httpcomponents:httpclient:4.5.6 -org.apache.httpcomponents:httpcore:4.4.10 +org.apache.httpcomponents:httpclient:4.5.13 +org.apache.httpcomponents:httpcore:4.4.13 org.apache.kafka:kafka-clients:2.8.2 org.apache.kerby:kerb-admin:2.0.3 org.apache.kerby:kerb-client:2.0.3 @@ -357,8 +355,6 @@ org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.51.v20230217 org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.51.v20230217 org.ehcache:ehcache:3.3.1 org.ini4j:ini4j:0.5.4 -org.jetbrains.kotlin:kotlin-stdlib:1.4.10 -org.jetbrains.kotlin:kotlin-stdlib-common:1.4.10 org.lz4:lz4-java:1.7.1 org.objenesis:objenesis:2.6 org.xerial.snappy:snappy-java:1.1.10.1 diff --git a/NOTICE-binary b/NOTICE-binary index b96e052658..6db51d08b4 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -334,19 +334,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ------------------------------------------------------------------------ - -This product contains a modified portion of 'OkHttp', an open source -HTTP & SPDY client for Android and Java applications, which can be obtained -at: - - * LICENSE: - * okhttp/third_party/okhttp/LICENSE (Apache License 2.0) - * HOMEPAGE: - * https://github.com/square/okhttp - * LOCATION_IN_GRPC: - * okhttp/third_party/okhttp - This product contains a modified portion of 'Netty', an open source networking library, which can be obtained at: diff --git a/hadoop-client-modules/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml index 9170bf4b54..08452aa20e 100644 --- a/hadoop-client-modules/hadoop-client/pom.xml +++ b/hadoop-client-modules/hadoop-client/pom.xml @@ -118,18 +118,6 @@ org.eclipse.jetty jetty-server - - org.jetbrains.kotlin - kotlin-stdlib - - - org.jetbrains.kotlin - kotlin-stdlib-common - - - com.squareup.okhttp3 - okhttp - com.sun.jersey jersey-core diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml b/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml index b96883b9ac..4892a7ac86 100755 --- a/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml +++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml @@ -161,10 +161,6 @@ esdk-obs-java ${esdk.version} - - okio - com.squareup.okio - log4j-core org.apache.logging.log4j diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 207f1f5351..c453546695 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -394,8 +394,8 @@ test - com.squareup.okio - okio-jvm + org.jetbrains.kotlin + kotlin-stdlib-jdk8 test diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml index 508388aa48..c96b3a99bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml @@ -94,17 +94,4 @@ - - - - - - - - - - - - - diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml index 01ab5f4137..9e370788a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@ -34,28 +34,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> - - com.squareup.okhttp3 - okhttp - - - com.squareup.okio - okio-jvm - - - - - com.squareup.okio - okio-jvm - - - org.jetbrains.kotlin - kotlin-stdlib - - - org.jetbrains.kotlin - kotlin-stdlib-common - org.apache.hadoop hadoop-common diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java index e944e8c1c8..7b82cad215 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java @@ -19,13 +19,10 @@ package org.apache.hadoop.hdfs.web.oauth2; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; - -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -33,7 +30,17 @@ import org.apache.hadoop.hdfs.web.URLConnectionFactory; import org.apache.hadoop.util.JsonSerialization; import org.apache.hadoop.util.Timer; +import org.apache.http.HttpHeaders; import org.apache.http.HttpStatus; +import org.apache.http.NameValuePair; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY; @@ -103,34 +110,37 @@ public synchronized String getAccessToken() throws IOException { } void refresh() throws IOException { - OkHttpClient client = - new OkHttpClient.Builder().connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, - TimeUnit.MILLISECONDS) - .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) - .build(); + final List pairs = new ArrayList<>(); + pairs.add(new BasicNameValuePair(GRANT_TYPE, REFRESH_TOKEN)); + pairs.add(new BasicNameValuePair(REFRESH_TOKEN, refreshToken)); + pairs.add(new BasicNameValuePair(CLIENT_ID, clientId)); + final RequestConfig config = RequestConfig.custom() + .setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT) + .setConnectionRequestTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT) + .setSocketTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT) + .build(); + try (CloseableHttpClient client = + HttpClientBuilder.create().setDefaultRequestConfig(config).build()) { + final HttpPost httpPost = new HttpPost(refreshURL); + httpPost.setEntity(new UrlEncodedFormEntity(pairs, StandardCharsets.UTF_8)); + httpPost.setHeader(HttpHeaders.CONTENT_TYPE, URLENCODED); + try (CloseableHttpResponse response = client.execute(httpPost)) { + final int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode != HttpStatus.SC_OK) { + throw new IllegalArgumentException( + "Received invalid http response: " + statusCode + ", text = " + + EntityUtils.toString(response.getEntity())); + } + Map responseBody = JsonSerialization.mapReader().readValue( + EntityUtils.toString(response.getEntity())); - String bodyString = - Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, REFRESH_TOKEN, refreshToken, CLIENT_ID, clientId); + String newExpiresIn = responseBody.get(EXPIRES_IN).toString(); + accessTokenTimer.setExpiresIn(newExpiresIn); - RequestBody body = RequestBody.create(bodyString, URLENCODED); - - Request request = new Request.Builder().url(refreshURL).post(body).build(); - try (Response response = client.newCall(request).execute()) { - if (!response.isSuccessful()) { - throw new IOException("Unexpected code " + response); + accessToken = responseBody.get(ACCESS_TOKEN).toString(); } - if (response.code() != HttpStatus.SC_OK) { - throw new IllegalArgumentException( - "Received invalid http response: " + response.code() + ", text = " - + response.toString()); - } - - Map responseBody = JsonSerialization.mapReader().readValue(response.body().string()); - - String newExpiresIn = responseBody.get(EXPIRES_IN).toString(); - accessTokenTimer.setExpiresIn(newExpiresIn); - - accessToken = responseBody.get(ACCESS_TOKEN).toString(); + } catch (RuntimeException e) { + throw new IOException("Exception while refreshing access token", e); } catch (Exception e) { throw new IOException("Exception while refreshing access token", e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java index 25ceb88460..1803e997ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java @@ -19,13 +19,10 @@ package org.apache.hadoop.hdfs.web.oauth2; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; - -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -33,7 +30,17 @@ import org.apache.hadoop.hdfs.web.URLConnectionFactory; import org.apache.hadoop.util.JsonSerialization; import org.apache.hadoop.util.Timer; +import org.apache.http.HttpHeaders; import org.apache.http.HttpStatus; +import org.apache.http.NameValuePair; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY; @@ -97,38 +104,37 @@ public synchronized String getAccessToken() throws IOException { } void refresh() throws IOException { - OkHttpClient client = new OkHttpClient.Builder() - .connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) - .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) - .build(); - - String bodyString = Utils.postBody(CLIENT_SECRET, getCredential(), - GRANT_TYPE, CLIENT_CREDENTIALS, - CLIENT_ID, clientId); - - RequestBody body = RequestBody.create(bodyString, URLENCODED); - - Request request = new Request.Builder() - .url(refreshURL) - .post(body) + final List pairs = new ArrayList<>(); + pairs.add(new BasicNameValuePair(CLIENT_SECRET, getCredential())); + pairs.add(new BasicNameValuePair(GRANT_TYPE, CLIENT_CREDENTIALS)); + pairs.add(new BasicNameValuePair(CLIENT_ID, clientId)); + final RequestConfig config = RequestConfig.custom() + .setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT) + .setConnectionRequestTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT) + .setSocketTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT) .build(); - try (Response response = client.newCall(request).execute()) { - if (!response.isSuccessful()) { - throw new IOException("Unexpected code " + response); + try (CloseableHttpClient client = + HttpClientBuilder.create().setDefaultRequestConfig(config).build()) { + final HttpPost httpPost = new HttpPost(refreshURL); + httpPost.setEntity(new UrlEncodedFormEntity(pairs, StandardCharsets.UTF_8)); + httpPost.setHeader(HttpHeaders.CONTENT_TYPE, URLENCODED); + try (CloseableHttpResponse response = client.execute(httpPost)) { + final int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode != HttpStatus.SC_OK) { + throw new IllegalArgumentException( + "Received invalid http response: " + statusCode + ", text = " + + EntityUtils.toString(response.getEntity())); + } + Map responseBody = JsonSerialization.mapReader().readValue( + EntityUtils.toString(response.getEntity())); + + String newExpiresIn = responseBody.get(EXPIRES_IN).toString(); + timer.setExpiresIn(newExpiresIn); + + accessToken = responseBody.get(ACCESS_TOKEN).toString(); } - - if (response.code() != HttpStatus.SC_OK) { - throw new IllegalArgumentException("Received invalid http response: " - + response.code() + ", text = " + response.toString()); - } - - Map responseBody = JsonSerialization.mapReader().readValue( - response.body().string()); - - String newExpiresIn = responseBody.get(EXPIRES_IN).toString(); - timer.setExpiresIn(newExpiresIn); - - accessToken = responseBody.get(ACCESS_TOKEN).toString(); + } catch (RuntimeException e) { + throw new IOException("Unable to obtain access token from credential", e); } catch (Exception e) { throw new IOException("Unable to obtain access token from credential", e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java index 2f28b65e40..dbe95aca31 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java @@ -18,7 +18,6 @@ */ package org.apache.hadoop.hdfs.web.oauth2; -import okhttp3.MediaType; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -30,8 +29,8 @@ public final class OAuth2Constants { private OAuth2Constants() { /** Private constructor. **/ } - public static final MediaType URLENCODED - = MediaType.parse("application/x-www-form-urlencoded; charset=utf-8"); + public static final String URLENCODED + = "application/x-www-form-urlencoded; charset=utf-8"; /* Constants for OAuth protocol */ public static final String ACCESS_TOKEN = "access_token"; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index b471fd062d..b5b264ffa8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -205,8 +205,8 @@ test - com.squareup.okhttp3 - okhttp + org.jetbrains.kotlin + kotlin-stdlib-jdk8 test diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 479a2183ad..e95e7471c5 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -131,10 +131,8 @@ 4.0.3 10.14.2.0 6.2.1.jre7 - 4.10.0 - 3.4.0 - 1.6.20 - 1.6.20 + 4.11.0 + 1.6.20 2.0.6.1 5.2.0 2.9.0 @@ -220,62 +218,24 @@ - - com.squareup.okhttp3 - okhttp - ${okhttp3.version} - - - org.jetbrains.kotlin - kotlin-stdlib - - - org.jetbrains.kotlin - kotlin-stdlib-common - - - com.squareup.okio - okio-jvm - - - - - com.squareup.okio - okio-jvm - ${okio.version} - - - org.jetbrains.kotlin - kotlin-stdlib - ${kotlin-stdlib.verion} - - - org.jetbrains - annotations - - - - - org.jetbrains.kotlin - kotlin-stdlib-common - ${kotlin-stdlib-common.version} - com.squareup.okhttp3 mockwebserver ${okhttp3.version} test - - com.squareup.okio - okio-jvm - org.jetbrains.kotlin kotlin-stdlib-jdk8 + + org.jetbrains.kotlin + kotlin-stdlib-jdk8 + ${kotlin-stdlib.version} + test + jdiff jdiff