+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om; + +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; +import static org.apache.hadoop.ozone.OzoneConfigKeys. + OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.UUID; + +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.ServletOutputStream; +import javax.servlet.WriteListener; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConsts; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; +import org.mockito.Matchers; + +import static org.apache.hadoop.ozone.OzoneConsts. + OZONE_DB_CHECKPOINT_REQUEST_FLUSH; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Class used for testing the OM DB Checkpoint provider servlet. + */ +public class TestOMDbCheckpointServlet { + private MiniOzoneCluster cluster = null; + private OMMetrics omMetrics; + private OzoneConfiguration conf; + private String clusterId; + private String scmId; + private String omId; + + @Rule + public Timeout timeout = new Timeout(60000); + + /** + * Create a MiniDFSCluster for testing. + *
+ * Ozone is made active by setting OZONE_ENABLED = true + * + * @throws IOException + */ + @Before + public void init() throws Exception { + conf = new OzoneConfiguration(); + clusterId = UUID.randomUUID().toString(); + scmId = UUID.randomUUID().toString(); + omId = UUID.randomUUID().toString(); + conf.setBoolean(OZONE_ACL_ENABLED, true); + conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2); + cluster = MiniOzoneCluster.newBuilder(conf) + .setClusterId(clusterId) + .setScmId(scmId) + .setOmId(omId) + .build(); + cluster.waitForClusterToBeReady(); + omMetrics = cluster.getOzoneManager().getMetrics(); + } + + /** + * Shutdown MiniDFSCluster. + */ + @After + public void shutdown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + public void testDoGet() throws ServletException, IOException { + + File tempFile = null; + try { + OMDBCheckpointServlet omDbCheckpointServletMock = + mock(OMDBCheckpointServlet.class); + + doCallRealMethod().when(omDbCheckpointServletMock).init(); + + HttpServletRequest requestMock = mock(HttpServletRequest.class); + HttpServletResponse responseMock = mock(HttpServletResponse.class); + + ServletContext servletContextMock = mock(ServletContext.class); + when(omDbCheckpointServletMock.getServletContext()) + .thenReturn(servletContextMock); + + when(servletContextMock.getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE)) + .thenReturn(cluster.getOzoneManager()); + when(requestMock.getParameter(OZONE_DB_CHECKPOINT_REQUEST_FLUSH)) + .thenReturn("true"); + doNothing().when(responseMock).setContentType("application/x-tgz"); + doNothing().when(responseMock).setHeader(Matchers.anyString(), + Matchers.anyString()); + + tempFile = File.createTempFile("testDoGet_" + System + .currentTimeMillis(), ".tar.gz"); + + FileOutputStream fileOutputStream = new FileOutputStream(tempFile); + when(responseMock.getOutputStream()).thenReturn( + new ServletOutputStream() { + @Override + public boolean isReady() { + return true; + } + + @Override + public void setWriteListener(WriteListener writeListener) { + } + + @Override + public void write(int b) throws IOException { + fileOutputStream.write(b); + } + }); + + doCallRealMethod().when(omDbCheckpointServletMock).doGet(requestMock, + responseMock); + + omDbCheckpointServletMock.init(); + + Assert.assertTrue( + omMetrics.getLastCheckpointCreationTimeTaken() == 0); + Assert.assertTrue( + omMetrics.getLastCheckpointTarOperationTimeTaken() == 0); + Assert.assertTrue( + omMetrics.getLastCheckpointStreamingTimeTaken() == 0); + + omDbCheckpointServletMock.doGet(requestMock, responseMock); + + Assert.assertTrue(tempFile.length() > 0); + Assert.assertTrue( + omMetrics.getLastCheckpointCreationTimeTaken() > 0); + Assert.assertTrue( + omMetrics.getLastCheckpointTarOperationTimeTaken() > 0); + Assert.assertTrue( + omMetrics.getLastCheckpointStreamingTimeTaken() > 0); + } finally { + FileUtils.deleteQuietly(tempFile); + } + + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index edcb7efd2c..96acfb3f06 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -24,6 +24,8 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.time.Duration; +import java.time.Instant; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; @@ -53,6 +55,7 @@ public class OMDBCheckpointServlet extends HttpServlet { private static final long serialVersionUID = 1L; private transient DBStore omDbStore; + private transient OMMetrics omMetrics; private transient DataTransferThrottler throttler = null; @Override @@ -67,6 +70,8 @@ public void init() throws ServletException { } omDbStore = om.getMetadataManager().getStore(); + omMetrics = om.getMetrics(); + OzoneConfiguration configuration = om.getConfiguration(); long transferBandwidth = configuration.getLongBytes( OMConfigKeys.OZONE_DB_CHECKPOINT_TRANSFER_RATE_KEY, @@ -112,19 +117,38 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) { response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); return; } + omMetrics.setLastCheckpointCreationTimeTaken( + checkpoint.checkpointCreationTimeTaken()); + + Instant start = Instant.now(); checkPointTarFile = OmUtils.createTarFile( checkpoint.getCheckpointLocation()); - LOG.info("Tar location = " + checkPointTarFile.getAbsolutePath()); + Instant end = Instant.now(); + + long duration = Duration.between(start, end).toMillis(); + LOG.debug("Time taken to archive the checkpoint : " + duration + + " milliseconds"); + LOG.info("Checkpoint Tar location = " + + checkPointTarFile.getAbsolutePath()); + omMetrics.setLastCheckpointTarOperationTimeTaken(duration); + response.setContentType("application/x-tgz"); response.setHeader("Content-Disposition", "attachment; filename=\"" + checkPointTarFile.getName() + "\""); checkpointFileInputStream = new FileInputStream(checkPointTarFile); + start = Instant.now(); TransferFsImage.copyFileToStream(response.getOutputStream(), checkPointTarFile, checkpointFileInputStream, throttler); + end = Instant.now(); + + duration = Duration.between(start, end).toMillis(); + LOG.debug("Time taken to write the checkpoint to response output " + + "stream: " + duration + " milliseconds"); + omMetrics.setLastCheckpointStreamingTimeTaken(duration); checkpoint.cleanupCheckpoint(); } catch (IOException e) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java index 2ef0aca56e..946b89867b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java @@ -25,6 +25,7 @@ import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableGaugeLong; /** * This class is for maintaining Ozone Manager statistics. @@ -105,6 +106,10 @@ public class OMMetrics { // few minutes before restart may not be included in this count. private @Metric MutableCounterLong numKeys; + // Metrics to track checkpointing statistics from last run. + private @Metric MutableGaugeLong lastCheckpointCreationTimeTaken; + private @Metric MutableGaugeLong lastCheckpointTarOperationTimeTaken; + private @Metric MutableGaugeLong lastCheckpointStreamingTimeTaken; public OMMetrics() { } @@ -390,6 +395,18 @@ public void incNumGetServiceListFails() { numGetServiceListFails.incr(); } + public void setLastCheckpointCreationTimeTaken(long val) { + this.lastCheckpointCreationTimeTaken.set(val); + } + + public void setLastCheckpointTarOperationTimeTaken(long val) { + this.lastCheckpointTarOperationTimeTaken.set(val); + } + + public void setLastCheckpointStreamingTimeTaken(long val) { + this.lastCheckpointStreamingTimeTaken.set(val); + } + @VisibleForTesting public long getNumVolumeCreates() { return numVolumeCreates.value(); @@ -606,6 +623,21 @@ public long getNumAbortMultipartUploadFails() { return numAbortMultipartUploadFails.value(); } + @VisibleForTesting + public long getLastCheckpointCreationTimeTaken() { + return lastCheckpointCreationTimeTaken.value(); + } + + @VisibleForTesting + public long getLastCheckpointTarOperationTimeTaken() { + return lastCheckpointTarOperationTimeTaken.value(); + } + + @VisibleForTesting + public long getLastCheckpointStreamingTimeTaken() { + return lastCheckpointStreamingTimeTaken.value(); + } + public void unRegister() { MetricsSystem ms = DefaultMetricsSystem.instance(); ms.unregisterSource(SOURCE_NAME);