From e211f6f83d1a9868a273112176054fd05e6821e7 Mon Sep 17 00:00:00 2001 From: Lei313 <47049042+ThinkerLei@users.noreply.github.com> Date: Tue, 12 Mar 2024 18:36:43 +0800 Subject: [PATCH] HDFS-17391. Adjust the checkpoint io buffer size to the chunk size (#6594). Contributed by lei w. Signed-off-by: He Xiaoqiao --- .../hdfs/server/namenode/TransferFsImage.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java index 816155991e..ac31c16c91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java @@ -311,7 +311,7 @@ private static void uploadImage(URL url, Configuration conf, ImageServlet.setVerificationHeadersForPut(connection, imageFile); // Write the file to output stream. - writeFileToPutRequest(conf, connection, imageFile, canceler); + writeFileToPutRequest(conf, connection, imageFile, canceler, chunkSize); int responseCode = connection.getResponseCode(); if (responseCode != HttpURLConnection.HTTP_OK) { @@ -330,7 +330,8 @@ private static void uploadImage(URL url, Configuration conf, } private static void writeFileToPutRequest(Configuration conf, - HttpURLConnection connection, File imageFile, Canceler canceler) + HttpURLConnection connection, File imageFile, Canceler canceler, + int bufferSize) throws IOException { connection.setRequestProperty(Util.CONTENT_TYPE, "application/octet-stream"); connection.setRequestProperty(Util.CONTENT_TRANSFER_ENCODING, "binary"); @@ -338,7 +339,7 @@ private static void writeFileToPutRequest(Configuration conf, FileInputStream input = new FileInputStream(imageFile); try { copyFileToStream(output, imageFile, input, - ImageServlet.getThrottler(conf), canceler); + ImageServlet.getThrottler(conf), canceler, bufferSize); } finally { IOUtils.closeStream(input); IOUtils.closeStream(output); @@ -352,13 +353,14 @@ private static void writeFileToPutRequest(Configuration conf, public static void copyFileToStream(OutputStream out, File localfile, FileInputStream infile, DataTransferThrottler throttler) throws IOException { - copyFileToStream(out, localfile, infile, throttler, null); + copyFileToStream(out, localfile, infile, throttler, null, -1); } private static void copyFileToStream(OutputStream out, File localfile, FileInputStream infile, DataTransferThrottler throttler, - Canceler canceler) throws IOException { - byte buf[] = new byte[IO_FILE_BUFFER_SIZE]; + Canceler canceler, int bufferSize) throws IOException { + int bufSize = bufferSize > 0 ? bufferSize : IO_FILE_BUFFER_SIZE; + byte[] buf = new byte[bufSize]; long total = 0; int num = 1; IOException ioe = null; @@ -369,13 +371,13 @@ private static void copyFileToStream(OutputStream out, File localfile, .aboutToSendFile(localfile); if (CheckpointFaultInjector.getInstance(). - shouldSendShortFile(localfile)) { - // Test sending image shorter than localfile - long len = localfile.length(); - buf = new byte[(int)Math.min(len/2, IO_FILE_BUFFER_SIZE)]; - // This will read at most half of the image - // and the rest of the image will be sent over the wire - infile.read(buf); + shouldSendShortFile(localfile)) { + // Test sending image shorter than localfile + long len = localfile.length(); + buf = new byte[(int) Math.min(len / 2, bufSize)]; + // This will read at most half of the image + // and the rest of the image will be sent over the wire + infile.read(buf); } while (num > 0) { if (canceler != null && canceler.isCancelled()) {