001/**
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019package org.apache.hadoop.hbase;
020
021import org.apache.hbase.thirdparty.com.google.common.base.MoreObjects;
022import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
023import com.codahale.metrics.Histogram;
024import org.apache.hadoop.conf.Configuration;
025import org.apache.hadoop.hbase.chaos.actions.MoveRandomRegionOfTableAction;
026import org.apache.hadoop.hbase.chaos.actions.RestartRandomRsExceptMetaAction;
027import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey;
028import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy;
029import org.apache.hadoop.hbase.chaos.policies.Policy;
030import org.apache.hadoop.hbase.client.Admin;
031import org.apache.hadoop.hbase.ipc.RpcClient;
032import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
033import org.apache.hadoop.hbase.testclassification.IntegrationTests;
034import org.apache.hadoop.hbase.util.Bytes;
035import org.apache.hadoop.hbase.util.YammerHistogramUtils;
036import org.apache.hadoop.mapreduce.Counters;
037import org.apache.hadoop.mapreduce.Job;
038import org.apache.hadoop.util.ToolRunner;
039import org.junit.experimental.categories.Category;
040import org.slf4j.Logger;
041import org.slf4j.LoggerFactory;
042
043import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
044
045import java.util.*;
046import java.util.concurrent.Callable;
047
048import static java.lang.String.format;
049import static org.junit.Assert.assertEquals;
050import static org.junit.Assert.assertNotNull;
051import static org.junit.Assert.assertTrue;
052
053/**
054 * Test for comparing the performance impact of region replicas. Uses
055 * components of {@link PerformanceEvaluation}. Does not run from
056 * {@code IntegrationTestsDriver} because IntegrationTestBase is incompatible
057 * with the JUnit runner. Hence no @Test annotations either. See {@code -help}
058 * for full list of options.
059 */
060@Category(IntegrationTests.class)
061public class IntegrationTestRegionReplicaPerf extends IntegrationTestBase {
062
063  private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestRegionReplicaPerf.class);
064
065  private static final String SLEEP_TIME_KEY = "sleeptime";
066  // short default interval because tests don't run very long.
067  private static final String SLEEP_TIME_DEFAULT = "" + (10 * 1000l);
068  private static final String TABLE_NAME_KEY = "tableName";
069  private static final String TABLE_NAME_DEFAULT = "IntegrationTestRegionReplicaPerf";
070  private static final String REPLICA_COUNT_KEY = "replicas";
071  private static final String REPLICA_COUNT_DEFAULT = "" + 3;
072  private static final String PRIMARY_TIMEOUT_KEY = "timeout";
073  private static final String PRIMARY_TIMEOUT_DEFAULT = "" + 10 * 1000; // 10 ms
074  private static final String NUM_RS_KEY = "numRs";
075  private static final String NUM_RS_DEFAULT = "" + 3;
076  public static final String FAMILY_NAME = "info";
077
078  /** Extract a descriptive statistic from a {@link com.codahale.metrics.Histogram}. */
079  private enum Stat {
080    STDEV {
081      @Override
082      double apply(Histogram hist) {
083        return hist.getSnapshot().getStdDev();
084      }
085    },
086    FOUR_9S {
087      @Override
088      double apply(Histogram hist) {
089        return hist.getSnapshot().getValue(0.9999);
090      }
091    };
092
093    abstract double apply(Histogram hist);
094  }
095
096  private TableName tableName;
097  private long sleepTime;
098  private int replicaCount;
099  private int primaryTimeout;
100  private int clusterSize;
101
102  /**
103   * Wraps the invocation of {@link PerformanceEvaluation} in a {@code Callable}.
104   */
105  static class PerfEvalCallable implements Callable<TimingResult> {
106    private final Queue<String> argv = new LinkedList<>();
107    private final Admin admin;
108
109    public PerfEvalCallable(Admin admin, String argv) {
110      // TODO: this API is awkward, should take Connection, not Admin
111      this.admin = admin;
112      this.argv.addAll(Arrays.asList(argv.split(" ")));
113      LOG.debug("Created PerformanceEvaluationCallable with args: " + argv);
114    }
115
116    @Override
117    public TimingResult call() throws Exception {
118      PerformanceEvaluation.TestOptions opts = PerformanceEvaluation.parseOpts(argv);
119      PerformanceEvaluation.checkTable(admin, opts);
120      PerformanceEvaluation.RunResult results[] = null;
121      long numRows = opts.totalRows;
122      long elapsedTime = 0;
123      if (opts.nomapred) {
124        results = PerformanceEvaluation.doLocalClients(opts, admin.getConfiguration());
125        for (PerformanceEvaluation.RunResult r : results) {
126          elapsedTime = Math.max(elapsedTime, r.duration);
127        }
128      } else {
129        Job job = PerformanceEvaluation.doMapReduce(opts, admin.getConfiguration());
130        Counters counters = job.getCounters();
131        numRows = counters.findCounter(PerformanceEvaluation.Counter.ROWS).getValue();
132        elapsedTime = counters.findCounter(PerformanceEvaluation.Counter.ELAPSED_TIME).getValue();
133      }
134      return new TimingResult(numRows, elapsedTime, results);
135    }
136  }
137
138  /**
139   * Record the results from a single {@link PerformanceEvaluation} job run.
140   */
141  static class TimingResult {
142    public final long numRows;
143    public final long elapsedTime;
144    public final PerformanceEvaluation.RunResult results[];
145
146    public TimingResult(long numRows, long elapsedTime, PerformanceEvaluation.RunResult results[]) {
147      this.numRows = numRows;
148      this.elapsedTime = elapsedTime;
149      this.results = results;
150    }
151
152    @Override
153    public String toString() {
154      return MoreObjects.toStringHelper(this)
155        .add("numRows", numRows)
156        .add("elapsedTime", elapsedTime)
157        .toString();
158    }
159  }
160
161  @Override
162  public void setUp() throws Exception {
163    super.setUp();
164    Configuration conf = util.getConfiguration();
165
166    // sanity check cluster
167    // TODO: this should reach out to master and verify online state instead
168    assertEquals("Master must be configured with StochasticLoadBalancer",
169      "org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer",
170      conf.get("hbase.master.loadbalancer.class"));
171    // TODO: this should reach out to master and verify online state instead
172    assertTrue("hbase.regionserver.storefile.refresh.period must be greater than zero.",
173      conf.getLong("hbase.regionserver.storefile.refresh.period", 0) > 0);
174
175    // enable client-side settings
176    conf.setBoolean(RpcClient.SPECIFIC_WRITE_THREAD, true);
177    // TODO: expose these settings to CLI override
178    conf.setLong("hbase.client.primaryCallTimeout.get", primaryTimeout);
179    conf.setLong("hbase.client.primaryCallTimeout.multiget", primaryTimeout);
180  }
181
182  @Override
183  public void setUpCluster() throws Exception {
184    util = getTestingUtil(getConf());
185    util.initializeCluster(clusterSize);
186  }
187
188  @Override
189  public void setUpMonkey() throws Exception {
190    Policy p = new PeriodicRandomActionPolicy(sleepTime,
191      new RestartRandomRsExceptMetaAction(sleepTime),
192      new MoveRandomRegionOfTableAction(tableName));
193    this.monkey = new PolicyBasedChaosMonkey(util, p);
194    // don't start monkey right away
195  }
196
197  @Override
198  protected void addOptions() {
199    addOptWithArg(TABLE_NAME_KEY, "Alternate table name. Default: '"
200      + TABLE_NAME_DEFAULT + "'");
201    addOptWithArg(SLEEP_TIME_KEY, "How long the monkey sleeps between actions. Default: "
202      + SLEEP_TIME_DEFAULT);
203    addOptWithArg(REPLICA_COUNT_KEY, "Number of region replicas. Default: "
204      + REPLICA_COUNT_DEFAULT);
205    addOptWithArg(PRIMARY_TIMEOUT_KEY, "Overrides hbase.client.primaryCallTimeout. Default: "
206      + PRIMARY_TIMEOUT_DEFAULT + " (10ms)");
207    addOptWithArg(NUM_RS_KEY, "Specify the number of RegionServers to use. Default: "
208        + NUM_RS_DEFAULT);
209  }
210
211  @Override
212  protected void processOptions(CommandLine cmd) {
213    tableName = TableName.valueOf(cmd.getOptionValue(TABLE_NAME_KEY, TABLE_NAME_DEFAULT));
214    sleepTime = Long.parseLong(cmd.getOptionValue(SLEEP_TIME_KEY, SLEEP_TIME_DEFAULT));
215    replicaCount = Integer.parseInt(cmd.getOptionValue(REPLICA_COUNT_KEY, REPLICA_COUNT_DEFAULT));
216    primaryTimeout =
217      Integer.parseInt(cmd.getOptionValue(PRIMARY_TIMEOUT_KEY, PRIMARY_TIMEOUT_DEFAULT));
218    clusterSize = Integer.parseInt(cmd.getOptionValue(NUM_RS_KEY, NUM_RS_DEFAULT));
219    LOG.debug(MoreObjects.toStringHelper("Parsed Options")
220      .add(TABLE_NAME_KEY, tableName)
221      .add(SLEEP_TIME_KEY, sleepTime)
222      .add(REPLICA_COUNT_KEY, replicaCount)
223      .add(PRIMARY_TIMEOUT_KEY, primaryTimeout)
224      .add(NUM_RS_KEY, clusterSize)
225      .toString());
226  }
227
228  @Override
229  public int runTestFromCommandLine() throws Exception {
230    test();
231    return 0;
232  }
233
234  @Override
235  public TableName getTablename() {
236    return tableName;
237  }
238
239  @Override
240  protected Set<String> getColumnFamilies() {
241    return Sets.newHashSet(FAMILY_NAME);
242  }
243
244  /** Compute the mean of the given {@code stat} from a timing results. */
245  private static double calcMean(String desc, Stat stat, List<TimingResult> results) {
246    double sum = 0;
247    int count = 0;
248
249    for (TimingResult tr : results) {
250      for (PerformanceEvaluation.RunResult r : tr.results) {
251        assertNotNull("One of the run results is missing detailed run data.", r.hist);
252        sum += stat.apply(r.hist);
253        count += 1;
254        LOG.debug(desc + "{" + YammerHistogramUtils.getHistogramReport(r.hist) + "}");
255      }
256    }
257    return sum / count;
258  }
259
260  public void test() throws Exception {
261    int maxIters = 3;
262    String replicas = "--replicas=" + replicaCount;
263    // TODO: splits disabled until "phase 2" is complete.
264    String splitPolicy = "--splitPolicy=" + DisabledRegionSplitPolicy.class.getName();
265    String writeOpts = format("%s --nomapred --table=%s --presplit=16 sequentialWrite 4",
266      splitPolicy, tableName);
267    String readOpts =
268      format("--nomapred --table=%s --latency --sampleRate=0.1 randomRead 4", tableName);
269    String replicaReadOpts = format("%s %s", replicas, readOpts);
270
271    ArrayList<TimingResult> resultsWithoutReplicas = new ArrayList<>(maxIters);
272    ArrayList<TimingResult> resultsWithReplicas = new ArrayList<>(maxIters);
273
274    // create/populate the table, replicas disabled
275    LOG.debug("Populating table.");
276    new PerfEvalCallable(util.getAdmin(), writeOpts).call();
277
278    // one last sanity check, then send in the clowns!
279    assertEquals("Table must be created with DisabledRegionSplitPolicy. Broken test.",
280        DisabledRegionSplitPolicy.class.getName(),
281        util.getAdmin().getTableDescriptor(tableName).getRegionSplitPolicyClassName());
282    startMonkey();
283
284    // collect a baseline without region replicas.
285    for (int i = 0; i < maxIters; i++) {
286      LOG.debug("Launching non-replica job " + (i + 1) + "/" + maxIters);
287      resultsWithoutReplicas.add(new PerfEvalCallable(util.getAdmin(), readOpts).call());
288      // TODO: sleep to let cluster stabilize, though monkey continues. is it necessary?
289      Thread.sleep(5000l);
290    }
291
292    // disable monkey, enable region replicas, enable monkey
293    cleanUpMonkey("Altering table.");
294    LOG.debug("Altering " + tableName + " replica count to " + replicaCount);
295    IntegrationTestingUtility.setReplicas(util.getAdmin(), tableName, replicaCount);
296    setUpMonkey();
297    startMonkey();
298
299    // run test with region replicas.
300    for (int i = 0; i < maxIters; i++) {
301      LOG.debug("Launching replica job " + (i + 1) + "/" + maxIters);
302      resultsWithReplicas.add(new PerfEvalCallable(util.getAdmin(), replicaReadOpts).call());
303      // TODO: sleep to let cluster stabilize, though monkey continues. is it necessary?
304      Thread.sleep(5000l);
305    }
306
307    // compare the average of the stdev and 99.99pct across runs to determine if region replicas
308    // are having an overall improvement on response variance experienced by clients.
309    double withoutReplicasStdevMean =
310        calcMean("withoutReplicas", Stat.STDEV, resultsWithoutReplicas);
311    double withoutReplicas9999Mean =
312        calcMean("withoutReplicas", Stat.FOUR_9S, resultsWithoutReplicas);
313    double withReplicasStdevMean =
314        calcMean("withReplicas", Stat.STDEV, resultsWithReplicas);
315    double withReplicas9999Mean =
316        calcMean("withReplicas", Stat.FOUR_9S, resultsWithReplicas);
317
318    LOG.info(MoreObjects.toStringHelper(this)
319      .add("withoutReplicas", resultsWithoutReplicas)
320      .add("withReplicas", resultsWithReplicas)
321      .add("withoutReplicasStdevMean", withoutReplicasStdevMean)
322      .add("withoutReplicas99.99Mean", withoutReplicas9999Mean)
323      .add("withReplicasStdevMean", withReplicasStdevMean)
324      .add("withReplicas99.99Mean", withReplicas9999Mean)
325      .toString());
326
327    assertTrue(
328      "Running with region replicas under chaos should have less request variance than without. "
329      + "withReplicas.stdev.mean: " + withReplicasStdevMean + "ms "
330      + "withoutReplicas.stdev.mean: " + withoutReplicasStdevMean + "ms.",
331      withReplicasStdevMean <= withoutReplicasStdevMean);
332    assertTrue(
333        "Running with region replicas under chaos should improve 99.99pct latency. "
334            + "withReplicas.99.99.mean: " + withReplicas9999Mean + "ms "
335            + "withoutReplicas.99.99.mean: " + withoutReplicas9999Mean + "ms.",
336        withReplicas9999Mean <= withoutReplicas9999Mean);
337  }
338
339  public static void main(String[] args) throws Exception {
340    Configuration conf = HBaseConfiguration.create();
341    IntegrationTestingUtility.setUseDistributedCluster(conf);
342    int status = ToolRunner.run(conf, new IntegrationTestRegionReplicaPerf(), args);
343    System.exit(status);
344  }
345}