001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021
022import java.util.Collection;
023import org.apache.hadoop.conf.Configuration;
024import org.apache.hadoop.fs.FileSystem;
025import org.apache.hadoop.fs.Path;
026import org.apache.hadoop.hbase.*;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.client.*;
029import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
030import org.apache.hadoop.hbase.testclassification.MediumTests;
031import org.apache.hadoop.hbase.util.Bytes;
032import org.apache.hadoop.hbase.util.FSUtils;
033import org.junit.After;
034import org.junit.AfterClass;
035import org.junit.Assert;
036import org.junit.BeforeClass;
037import org.junit.ClassRule;
038import org.junit.Test;
039import org.junit.experimental.categories.Category;
040import org.slf4j.Logger;
041import org.slf4j.LoggerFactory;
042
043@Category(MediumTests.class)
044public class TestCompactSplitThread {
045
046  @ClassRule
047  public static final HBaseClassTestRule CLASS_RULE =
048      HBaseClassTestRule.forClass(TestCompactSplitThread.class);
049
050  private static final Logger LOG = LoggerFactory.getLogger(TestCompactSplitThread.class);
051  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
052  private final TableName tableName = TableName.valueOf(getClass().getSimpleName());
053  private final byte[] family = Bytes.toBytes("f");
054  private static final int NUM_RS = 1;
055  private static final int blockingStoreFiles = 3;
056  private static Path rootDir;
057  private static FileSystem fs;
058
059
060
061  /**
062   * Setup the config for the cluster
063   */
064  @BeforeClass
065  public static void setupCluster() throws Exception {
066    setupConf(TEST_UTIL.getConfiguration());
067    TEST_UTIL.startMiniCluster(NUM_RS);
068    fs = TEST_UTIL.getDFSCluster().getFileSystem();
069    rootDir = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
070
071  }
072
073  private static void setupConf(Configuration conf) {
074    // disable the ui
075    conf.setInt("hbase.regionsever.info.port", -1);
076    // so make sure we get a compaction when doing a load, but keep around some
077    // files in the store
078    conf.setInt("hbase.hstore.compaction.min", 2);
079    conf.setInt("hbase.hstore.compactionThreshold", 5);
080    // change the flush size to a small amount, regulating number of store files
081    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
082
083    // block writes if we get to blockingStoreFiles store files
084    conf.setInt("hbase.hstore.blockingStoreFiles", blockingStoreFiles);
085    // Ensure no extra cleaners on by default (e.g. TimeToLiveHFileCleaner)
086    conf.setInt(CompactSplit.LARGE_COMPACTION_THREADS, 3);
087    conf.setInt(CompactSplit.SMALL_COMPACTION_THREADS, 4);
088    conf.setInt(CompactSplit.SPLIT_THREADS, 5);
089  }
090
091  @After
092  public void tearDown() throws Exception {
093    TEST_UTIL.deleteTable(tableName);
094  }
095
096  @AfterClass
097  public static void cleanupTest() throws Exception {
098    try {
099      TEST_UTIL.shutdownMiniCluster();
100    } catch (Exception e) {
101      // NOOP;
102    }
103  }
104
105  @Test
106  public void testThreadPoolSizeTuning() throws Exception {
107    Configuration conf = TEST_UTIL.getConfiguration();
108    Connection conn = ConnectionFactory.createConnection(conf);
109    try {
110      HTableDescriptor htd = new HTableDescriptor(tableName);
111      htd.addFamily(new HColumnDescriptor(family));
112      htd.setCompactionEnabled(false);
113      TEST_UTIL.getAdmin().createTable(htd);
114      TEST_UTIL.waitTableAvailable(tableName);
115      HRegionServer regionServer = TEST_UTIL.getRSForFirstRegionInTable(tableName);
116
117      // check initial configuration of thread pool sizes
118      assertEquals(3, regionServer.compactSplitThread.getLargeCompactionThreadNum());
119      assertEquals(4, regionServer.compactSplitThread.getSmallCompactionThreadNum());
120      assertEquals(5, regionServer.compactSplitThread.getSplitThreadNum());
121
122      // change bigger configurations and do online update
123      conf.setInt(CompactSplit.LARGE_COMPACTION_THREADS, 4);
124      conf.setInt(CompactSplit.SMALL_COMPACTION_THREADS, 5);
125      conf.setInt(CompactSplit.SPLIT_THREADS, 6);
126      try {
127        regionServer.compactSplitThread.onConfigurationChange(conf);
128      } catch (IllegalArgumentException iae) {
129        Assert.fail("Update bigger configuration failed!");
130      }
131
132      // check again after online update
133      assertEquals(4, regionServer.compactSplitThread.getLargeCompactionThreadNum());
134      assertEquals(5, regionServer.compactSplitThread.getSmallCompactionThreadNum());
135      assertEquals(6, regionServer.compactSplitThread.getSplitThreadNum());
136
137      // change smaller configurations and do online update
138      conf.setInt(CompactSplit.LARGE_COMPACTION_THREADS, 2);
139      conf.setInt(CompactSplit.SMALL_COMPACTION_THREADS, 3);
140      conf.setInt(CompactSplit.SPLIT_THREADS, 4);
141      try {
142        regionServer.compactSplitThread.onConfigurationChange(conf);
143      } catch (IllegalArgumentException iae) {
144        Assert.fail("Update smaller configuration failed!");
145      }
146
147      // check again after online update
148      assertEquals(2, regionServer.compactSplitThread.getLargeCompactionThreadNum());
149      assertEquals(3, regionServer.compactSplitThread.getSmallCompactionThreadNum());
150      assertEquals(4, regionServer.compactSplitThread.getSplitThreadNum());
151    } finally {
152      conn.close();
153    }
154  }
155
156  @Test
157  public void testFlushWithTableCompactionDisabled() throws Exception {
158    HTableDescriptor htd = new HTableDescriptor(tableName);
159    htd.setCompactionEnabled(false);
160    TEST_UTIL.createTable(htd, new byte[][] { family }, null);
161
162    // load the table
163    for (int i = 0; i < blockingStoreFiles + 1; i ++) {
164      TEST_UTIL.loadTable(TEST_UTIL.getConnection().getTable(tableName), family);
165      TEST_UTIL.flush(tableName);
166    }
167
168    // Make sure that store file number is greater than blockingStoreFiles + 1
169    Path tableDir = FSUtils.getTableDir(rootDir, tableName);
170    Collection<String> hfiles =  SnapshotTestingUtils.listHFileNames(fs, tableDir);
171    assert(hfiles.size() > blockingStoreFiles + 1);
172  }
173}