001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.apache.hadoop.hbase.client.MetricsConnection.CLIENT_SIDE_METRICS_ENABLED_KEY;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertNotEquals;
023import static org.junit.Assert.assertNotNull;
024import static org.junit.Assert.assertTrue;
025
026import java.io.IOException;
027import java.util.ArrayList;
028import java.util.List;
029import java.util.concurrent.CountDownLatch;
030import java.util.concurrent.TimeUnit;
031import java.util.concurrent.atomic.AtomicLong;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.hbase.*;
034import org.apache.hadoop.hbase.HBaseClassTestRule;
035import org.apache.hadoop.hbase.client.backoff.ClientBackoffPolicy;
036import org.apache.hadoop.hbase.client.backoff.ExponentialClientBackoffPolicy;
037import org.apache.hadoop.hbase.client.backoff.ServerStatistics;
038import org.apache.hadoop.hbase.client.coprocessor.Batch;
039import org.apache.hadoop.hbase.regionserver.HRegionServer;
040import org.apache.hadoop.hbase.regionserver.Region;
041import org.apache.hadoop.hbase.testclassification.MediumTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
044import org.junit.AfterClass;
045import org.junit.BeforeClass;
046import org.junit.ClassRule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049import org.slf4j.Logger;
050import org.slf4j.LoggerFactory;
051
052/**
053 * Test that we can actually send and use region metrics to slowdown client writes
054 */
055@Category(MediumTests.class)
056public class TestClientPushback {
057
058  @ClassRule
059  public static final HBaseClassTestRule CLASS_RULE =
060      HBaseClassTestRule.forClass(TestClientPushback.class);
061
062  private static final Logger LOG = LoggerFactory.getLogger(TestClientPushback.class);
063  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
064
065  private static final TableName tableName = TableName.valueOf("client-pushback");
066  private static final byte[] family = Bytes.toBytes("f");
067  private static final byte[] qualifier = Bytes.toBytes("q");
068  private static final long flushSizeBytes = 512;
069
070  @BeforeClass
071  public static void setupCluster() throws Exception{
072    Configuration conf = UTIL.getConfiguration();
073    // enable backpressure
074    conf.setBoolean(HConstants.ENABLE_CLIENT_BACKPRESSURE, true);
075    // use the exponential backoff policy
076    conf.setClass(ClientBackoffPolicy.BACKOFF_POLICY_CLASS, ExponentialClientBackoffPolicy.class,
077      ClientBackoffPolicy.class);
078    // turn the memstore size way down so we don't need to write a lot to see changes in memstore
079    // load
080    conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSizeBytes);
081    // ensure we block the flushes when we are double that flushsize
082    conf.setLong(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER, HConstants.DEFAULT_HREGION_MEMSTORE_BLOCK_MULTIPLIER);
083    conf.setBoolean(CLIENT_SIDE_METRICS_ENABLED_KEY, true);
084    UTIL.startMiniCluster(1);
085    UTIL.createTable(tableName, family);
086  }
087
088  @AfterClass
089  public static void teardownCluster() throws Exception{
090    UTIL.shutdownMiniCluster();
091  }
092
093  @Test
094  public void testClientTracksServerPushback() throws Exception{
095    Configuration conf = UTIL.getConfiguration();
096
097    ClusterConnection conn = (ClusterConnection) ConnectionFactory.createConnection(conf);
098    BufferedMutatorImpl mutator = (BufferedMutatorImpl) conn.getBufferedMutator(tableName);
099
100    HRegionServer rs = UTIL.getHBaseCluster().getRegionServer(0);
101    Region region = rs.getRegions(tableName).get(0);
102
103    LOG.debug("Writing some data to "+tableName);
104    // write some data
105    Put p = new Put(Bytes.toBytes("row"));
106    p.addColumn(family, qualifier, Bytes.toBytes("value1"));
107    mutator.mutate(p);
108    mutator.flush();
109
110    // get the current load on RS. Hopefully memstore isn't flushed since we wrote the the data
111    int load = (int) ((region.getMemStoreHeapSize() * 100)
112        / flushSizeBytes);
113    LOG.debug("Done writing some data to "+tableName);
114
115    // get the stats for the region hosting our table
116    ClientBackoffPolicy backoffPolicy = conn.getBackoffPolicy();
117    assertTrue("Backoff policy is not correctly configured",
118      backoffPolicy instanceof ExponentialClientBackoffPolicy);
119
120    ServerStatisticTracker stats = conn.getStatisticsTracker();
121    assertNotNull( "No stats configured for the client!", stats);
122    // get the names so we can query the stats
123    ServerName server = rs.getServerName();
124    byte[] regionName = region.getRegionInfo().getRegionName();
125
126    // check to see we found some load on the memstore
127    ServerStatistics serverStats = stats.getServerStatsForTesting(server);
128    ServerStatistics.RegionStatistics regionStats = serverStats.getStatsForRegion(regionName);
129    assertEquals("We did not find some load on the memstore", load,
130      regionStats.getMemStoreLoadPercent());
131    // check that the load reported produces a nonzero delay
132    long backoffTime = backoffPolicy.getBackoffTime(server, regionName, serverStats);
133    assertNotEquals("Reported load does not produce a backoff", 0, backoffTime);
134    LOG.debug("Backoff calculated for " + region.getRegionInfo().getRegionNameAsString() + " @ " +
135      server + " is " + backoffTime);
136
137    // Reach into the connection and submit work directly to AsyncProcess so we can
138    // monitor how long the submission was delayed via a callback
139    List<Row> ops = new ArrayList<>(1);
140    ops.add(p);
141    final CountDownLatch latch = new CountDownLatch(1);
142    final AtomicLong endTime = new AtomicLong();
143    long startTime = EnvironmentEdgeManager.currentTime();
144    Batch.Callback<Result> callback = (byte[] r, byte[] row, Result result) -> {
145        endTime.set(EnvironmentEdgeManager.currentTime());
146        latch.countDown();
147    };
148    AsyncProcessTask<Result> task = AsyncProcessTask.newBuilder(callback)
149            .setPool(mutator.getPool())
150            .setTableName(tableName)
151            .setRowAccess(ops)
152            .setSubmittedRows(AsyncProcessTask.SubmittedRows.AT_LEAST_ONE)
153            .setOperationTimeout(conn.getConnectionConfiguration().getOperationTimeout())
154            .setRpcTimeout(60 * 1000)
155            .build();
156    mutator.getAsyncProcess().submit(task);
157    // Currently the ExponentialClientBackoffPolicy under these test conditions
158    // produces a backoffTime of 151 milliseconds. This is long enough so the
159    // wait and related checks below are reasonable. Revisit if the backoff
160    // time reported by above debug logging has significantly deviated.
161    String name = server.getServerName() + "," + Bytes.toStringBinary(regionName);
162    MetricsConnection.RegionStats rsStats = conn.getConnectionMetrics().
163            serverStats.get(server).get(regionName);
164    assertEquals(name, rsStats.name);
165    assertEquals(rsStats.heapOccupancyHist.getSnapshot().getMean(),
166        (double)regionStats.getHeapOccupancyPercent(), 0.1 );
167    assertEquals(rsStats.memstoreLoadHist.getSnapshot().getMean(),
168        (double)regionStats.getMemStoreLoadPercent(), 0.1);
169
170    MetricsConnection.RunnerStats runnerStats = conn.getConnectionMetrics().runnerStats;
171
172    assertEquals(1, runnerStats.delayRunners.getCount());
173    assertEquals(1, runnerStats.normalRunners.getCount());
174    assertEquals("", runnerStats.delayIntevalHist.getSnapshot().getMean(),
175      (double)backoffTime, 0.1);
176
177    latch.await(backoffTime * 2, TimeUnit.MILLISECONDS);
178    assertNotEquals("AsyncProcess did not submit the work time", 0, endTime.get());
179    assertTrue("AsyncProcess did not delay long enough", endTime.get() - startTime >= backoffTime);
180  }
181
182  @Test
183  public void testMutateRowStats() throws IOException {
184    Configuration conf = UTIL.getConfiguration();
185    ClusterConnection conn = (ClusterConnection) ConnectionFactory.createConnection(conf);
186    Table table = conn.getTable(tableName);
187    HRegionServer rs = UTIL.getHBaseCluster().getRegionServer(0);
188    Region region = rs.getRegions(tableName).get(0);
189
190    RowMutations mutations = new RowMutations(Bytes.toBytes("row"));
191    Put p = new Put(Bytes.toBytes("row"));
192    p.addColumn(family, qualifier, Bytes.toBytes("value2"));
193    mutations.add(p);
194    table.mutateRow(mutations);
195
196    ServerStatisticTracker stats = conn.getStatisticsTracker();
197    assertNotNull( "No stats configured for the client!", stats);
198    // get the names so we can query the stats
199    ServerName server = rs.getServerName();
200    byte[] regionName = region.getRegionInfo().getRegionName();
201
202    // check to see we found some load on the memstore
203    ServerStatistics serverStats = stats.getServerStatsForTesting(server);
204    ServerStatistics.RegionStatistics regionStats = serverStats.getStatsForRegion(regionName);
205
206    assertNotNull(regionStats);
207    assertTrue(regionStats.getMemStoreLoadPercent() > 0);
208    }
209}