Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.wal;
019
020import static org.junit.Assert.*;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertTrue;
025import static org.junit.Assert.fail;
026import static org.hamcrest.CoreMatchers.*;
027
028import java.io.IOException;
029import java.util.ArrayList;
030import java.util.Comparator;
031import java.util.List;
032import java.util.NavigableMap;
033import java.util.Set;
034import java.util.TreeMap;
035import java.util.UUID;
036import java.util.concurrent.atomic.AtomicBoolean;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.fs.FileStatus;
039import org.apache.hadoop.fs.FileSystem;
040import org.apache.hadoop.fs.Path;
041import org.apache.hadoop.hbase.CellScanner;
042import org.apache.hadoop.hbase.Coprocessor;
043import org.apache.hadoop.hbase.HBaseConfiguration;
044import org.apache.hadoop.hbase.HBaseTestingUtility;
045import org.apache.hadoop.hbase.HConstants;
046import org.apache.hadoop.hbase.KeyValue;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
049import org.apache.hadoop.hbase.client.Get;
050import org.apache.hadoop.hbase.client.Put;
051import org.apache.hadoop.hbase.client.RegionInfo;
052import org.apache.hadoop.hbase.client.RegionInfoBuilder;
053import org.apache.hadoop.hbase.client.Result;
054import org.apache.hadoop.hbase.client.TableDescriptor;
055import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
056import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
057import org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor;
058import org.apache.hadoop.hbase.regionserver.HRegion;
059import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
060import org.apache.hadoop.hbase.regionserver.SequenceId;
061import org.apache.hadoop.hbase.util.Bytes;
062import org.apache.hadoop.hbase.util.CommonFSUtils;
063import org.apache.hadoop.hbase.util.EnvironmentEdge;
064import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
065import org.apache.hadoop.hbase.util.Threads;
066import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
067import org.apache.hadoop.hbase.wal.WAL;
068import org.apache.hadoop.hbase.wal.WALEdit;
069import org.apache.hadoop.hbase.wal.WALKey;
070import org.apache.hadoop.hbase.wal.WALKeyImpl;
071import org.junit.AfterClass;
072import org.junit.Before;
073import org.junit.BeforeClass;
074import org.junit.Rule;
075import org.junit.Test;
076import org.junit.rules.TestName;
077import org.slf4j.Logger;
078import org.slf4j.LoggerFactory;
079
080public abstract class AbstractTestFSWAL {
081
082  protected static final Logger LOG = LoggerFactory.getLogger(AbstractTestFSWAL.class);
083
084  protected static Configuration CONF;
085  protected static FileSystem FS;
086  protected static Path DIR;
087  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
088
089  @Rule
090  public final TestName currentTest = new TestName();
091
092  @Before
093  public void setUp() throws Exception {
094    FileStatus[] entries = FS.listStatus(new Path("/"));
095    for (FileStatus dir : entries) {
096      FS.delete(dir.getPath(), true);
097    }
098    final Path hbaseDir = TEST_UTIL.createRootDir();
099    final Path hbaseWALDir = TEST_UTIL.createWALRootDir();
100    DIR = new Path(hbaseWALDir, currentTest.getMethodName());
101    assertNotEquals(hbaseDir, hbaseWALDir);
102  }
103
104  @BeforeClass
105  public static void setUpBeforeClass() throws Exception {
106    // Make block sizes small.
107    TEST_UTIL.getConfiguration().setInt("dfs.blocksize", 1024 * 1024);
108    // quicker heartbeat interval for faster DN death notification
109    TEST_UTIL.getConfiguration().setInt("dfs.namenode.heartbeat.recheck-interval", 5000);
110    TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1);
111    TEST_UTIL.getConfiguration().setInt("dfs.client.socket-timeout", 5000);
112
113    // faster failover with cluster.shutdown();fs.close() idiom
114    TEST_UTIL.getConfiguration().setInt("hbase.ipc.client.connect.max.retries", 1);
115    TEST_UTIL.getConfiguration().setInt("dfs.client.block.recovery.retries", 1);
116    TEST_UTIL.getConfiguration().setInt("hbase.ipc.client.connection.maxidletime", 500);
117    TEST_UTIL.getConfiguration().set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
118      SampleRegionWALCoprocessor.class.getName());
119    TEST_UTIL.startMiniDFSCluster(3);
120
121    CONF = TEST_UTIL.getConfiguration();
122    FS = TEST_UTIL.getDFSCluster().getFileSystem();
123  }
124
125  @AfterClass
126  public static void tearDownAfterClass() throws Exception {
127    TEST_UTIL.shutdownMiniCluster();
128  }
129
130  protected abstract AbstractFSWAL<?> newWAL(FileSystem fs, Path rootDir, String WALDir,
131      String archiveDir, Configuration conf, List<WALActionsListener> listeners,
132      boolean failIfWALExists, String prefix, String suffix) throws IOException;
133
134  protected abstract AbstractFSWAL<?> newSlowWAL(FileSystem fs, Path rootDir, String WALDir,
135      String archiveDir, Configuration conf, List<WALActionsListener> listeners,
136      boolean failIfWALExists, String prefix, String suffix, Runnable action) throws IOException;
137
138  /**
139   * A loaded WAL coprocessor won't break existing WAL test cases.
140   */
141  @Test
142  public void testWALCoprocessorLoaded() throws Exception {
143    // test to see whether the coprocessor is loaded or not.
144    AbstractFSWAL<?> wal = null;
145    try {
146      wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(),
147          HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null);
148      WALCoprocessorHost host = wal.getCoprocessorHost();
149      Coprocessor c = host.findCoprocessor(SampleRegionWALCoprocessor.class);
150      assertNotNull(c);
151    } finally {
152      if (wal != null) {
153        wal.close();
154      }
155    }
156  }
157
158  protected void addEdits(WAL log, RegionInfo hri, TableDescriptor htd, int times,
159      MultiVersionConcurrencyControl mvcc, NavigableMap<byte[], Integer> scopes)
160      throws IOException {
161    final byte[] row = Bytes.toBytes("row");
162    for (int i = 0; i < times; i++) {
163      long timestamp = System.currentTimeMillis();
164      WALEdit cols = new WALEdit();
165      cols.add(new KeyValue(row, row, row, timestamp, row));
166      WALKeyImpl key = new WALKeyImpl(hri.getEncodedNameAsBytes(), htd.getTableName(),
167          SequenceId.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE,
168          HConstants.NO_NONCE, mvcc, scopes);
169      log.append(hri, key, cols, true);
170    }
171    log.sync();
172  }
173
174  /**
175   * helper method to simulate region flush for a WAL.
176   * @param wal
177   * @param regionEncodedName
178   */
179  protected void flushRegion(WAL wal, byte[] regionEncodedName, Set<byte[]> flushedFamilyNames) {
180    wal.startCacheFlush(regionEncodedName, flushedFamilyNames);
181    wal.completeCacheFlush(regionEncodedName);
182  }
183
184  /**
185   * tests the log comparator. Ensure that we are not mixing meta logs with non-meta logs (throws
186   * exception if we do). Comparison is based on the timestamp present in the wal name.
187   * @throws Exception
188   */
189  @Test
190  public void testWALComparator() throws Exception {
191    AbstractFSWAL<?> wal1 = null;
192    AbstractFSWAL<?> walMeta = null;
193    try {
194      wal1 = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(),
195          HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null);
196      LOG.debug("Log obtained is: " + wal1);
197      Comparator<Path> comp = wal1.LOG_NAME_COMPARATOR;
198      Path p1 = wal1.computeFilename(11);
199      Path p2 = wal1.computeFilename(12);
200      // comparing with itself returns 0
201      assertTrue(comp.compare(p1, p1) == 0);
202      // comparing with different filenum.
203      assertTrue(comp.compare(p1, p2) < 0);
204      walMeta = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(),
205          HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null,
206          AbstractFSWALProvider.META_WAL_PROVIDER_ID);
207      Comparator<Path> compMeta = walMeta.LOG_NAME_COMPARATOR;
208
209      Path p1WithMeta = walMeta.computeFilename(11);
210      Path p2WithMeta = walMeta.computeFilename(12);
211      assertTrue(compMeta.compare(p1WithMeta, p1WithMeta) == 0);
212      assertTrue(compMeta.compare(p1WithMeta, p2WithMeta) < 0);
213      // mixing meta and non-meta logs gives error
214      boolean ex = false;
215      try {
216        comp.compare(p1WithMeta, p2);
217      } catch (IllegalArgumentException e) {
218        ex = true;
219      }
220      assertTrue("Comparator doesn't complain while checking meta log files", ex);
221      boolean exMeta = false;
222      try {
223        compMeta.compare(p1WithMeta, p2);
224      } catch (IllegalArgumentException e) {
225        exMeta = true;
226      }
227      assertTrue("Meta comparator doesn't complain while checking log files", exMeta);
228    } finally {
229      if (wal1 != null) {
230        wal1.close();
231      }
232      if (walMeta != null) {
233        walMeta.close();
234      }
235    }
236  }
237
238  /**
239   * On rolling a wal after reaching the threshold, {@link WAL#rollWriter()} returns the list of
240   * regions which should be flushed in order to archive the oldest wal file.
241   * <p>
242   * This method tests this behavior by inserting edits and rolling the wal enough times to reach
243   * the max number of logs threshold. It checks whether we get the "right regions" for flush on
244   * rolling the wal.
245   * @throws Exception
246   */
247  @Test
248  public void testFindMemStoresEligibleForFlush() throws Exception {
249    LOG.debug("testFindMemStoresEligibleForFlush");
250    Configuration conf1 = HBaseConfiguration.create(CONF);
251    conf1.setInt("hbase.regionserver.maxlogs", 1);
252    AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getWALRootDir(conf1), DIR.toString(),
253      HConstants.HREGION_OLDLOGDIR_NAME, conf1, null, true, null, null);
254    TableDescriptor t1 = TableDescriptorBuilder.newBuilder(TableName.valueOf("t1"))
255      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();
256    TableDescriptor t2 = TableDescriptorBuilder.newBuilder(TableName.valueOf("t2"))
257      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();
258    RegionInfo hri1 = RegionInfoBuilder.newBuilder(t1.getTableName()).build();
259    RegionInfo hri2 = RegionInfoBuilder.newBuilder(t2.getTableName()).build();
260    // add edits and roll the wal
261    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
262    NavigableMap<byte[], Integer> scopes1 = new TreeMap<>(Bytes.BYTES_COMPARATOR);
263    for (byte[] fam : t1.getColumnFamilyNames()) {
264      scopes1.put(fam, 0);
265    }
266    NavigableMap<byte[], Integer> scopes2 = new TreeMap<>(Bytes.BYTES_COMPARATOR);
267    for (byte[] fam : t2.getColumnFamilyNames()) {
268      scopes2.put(fam, 0);
269    }
270    try {
271      addEdits(wal, hri1, t1, 2, mvcc, scopes1);
272      wal.rollWriter();
273      // add some more edits and roll the wal. This would reach the log number threshold
274      addEdits(wal, hri1, t1, 2, mvcc, scopes1);
275      wal.rollWriter();
276      // with above rollWriter call, the max logs limit is reached.
277      assertTrue(wal.getNumRolledLogFiles() == 2);
278
279      // get the regions to flush; since there is only one region in the oldest wal, it should
280      // return only one region.
281      byte[][] regionsToFlush = wal.findRegionsToForceFlush();
282      assertEquals(1, regionsToFlush.length);
283      assertEquals(hri1.getEncodedNameAsBytes(), regionsToFlush[0]);
284      // insert edits in second region
285      addEdits(wal, hri2, t2, 2, mvcc, scopes2);
286      // get the regions to flush, it should still read region1.
287      regionsToFlush = wal.findRegionsToForceFlush();
288      assertEquals(1, regionsToFlush.length);
289      assertEquals(hri1.getEncodedNameAsBytes(), regionsToFlush[0]);
290      // flush region 1, and roll the wal file. Only last wal which has entries for region1 should
291      // remain.
292      flushRegion(wal, hri1.getEncodedNameAsBytes(), t1.getColumnFamilyNames());
293      wal.rollWriter();
294      // only one wal should remain now (that is for the second region).
295      assertEquals(1, wal.getNumRolledLogFiles());
296      // flush the second region
297      flushRegion(wal, hri2.getEncodedNameAsBytes(), t2.getColumnFamilyNames());
298      wal.rollWriter(true);
299      // no wal should remain now.
300      assertEquals(0, wal.getNumRolledLogFiles());
301      // add edits both to region 1 and region 2, and roll.
302      addEdits(wal, hri1, t1, 2, mvcc, scopes1);
303      addEdits(wal, hri2, t2, 2, mvcc, scopes2);
304      wal.rollWriter();
305      // add edits and roll the writer, to reach the max logs limit.
306      assertEquals(1, wal.getNumRolledLogFiles());
307      addEdits(wal, hri1, t1, 2, mvcc, scopes1);
308      wal.rollWriter();
309      // it should return two regions to flush, as the oldest wal file has entries
310      // for both regions.
311      regionsToFlush = wal.findRegionsToForceFlush();
312      assertEquals(2, regionsToFlush.length);
313      // flush both regions
314      flushRegion(wal, hri1.getEncodedNameAsBytes(), t1.getColumnFamilyNames());
315      flushRegion(wal, hri2.getEncodedNameAsBytes(), t2.getColumnFamilyNames());
316      wal.rollWriter(true);
317      assertEquals(0, wal.getNumRolledLogFiles());
318      // Add an edit to region1, and roll the wal.
319      addEdits(wal, hri1, t1, 2, mvcc, scopes1);
320      // tests partial flush: roll on a partial flush, and ensure that wal is not archived.
321      wal.startCacheFlush(hri1.getEncodedNameAsBytes(), t1.getColumnFamilyNames());
322      wal.rollWriter();
323      wal.completeCacheFlush(hri1.getEncodedNameAsBytes());
324      assertEquals(1, wal.getNumRolledLogFiles());
325    } finally {
326      if (wal != null) {
327        wal.close();
328      }
329    }
330  }
331
332  @Test(expected = IOException.class)
333  public void testFailedToCreateWALIfParentRenamed() throws IOException,
334      CommonFSUtils.StreamLacksCapabilityException {
335    final String name = "testFailedToCreateWALIfParentRenamed";
336    AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), name,
337      HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null);
338    long filenum = System.currentTimeMillis();
339    Path path = wal.computeFilename(filenum);
340    wal.createWriterInstance(path);
341    Path parent = path.getParent();
342    path = wal.computeFilename(filenum + 1);
343    Path newPath = new Path(parent.getParent(), parent.getName() + "-splitting");
344    FS.rename(parent, newPath);
345    wal.createWriterInstance(path);
346    fail("It should fail to create the new WAL");
347  }
348
349  /**
350   * Test flush for sure has a sequence id that is beyond the last edit appended. We do this by
351   * slowing appends in the background ring buffer thread while in foreground we call flush. The
352   * addition of the sync over HRegion in flush should fix an issue where flush was returning before
353   * all of its appends had made it out to the WAL (HBASE-11109).
354   * @throws IOException
355   * @see <a href="https://issues.apache.org/jira/browse/HBASE-11109">HBASE-11109</a>
356   */
357  @Test
358  public void testFlushSequenceIdIsGreaterThanAllEditsInHFile() throws IOException {
359    String testName = currentTest.getMethodName();
360    final TableName tableName = TableName.valueOf(testName);
361    final RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build();
362    final byte[] rowName = tableName.getName();
363    final TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
364      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
365    HRegion r = HBaseTestingUtility.createRegionAndWAL(hri, TEST_UTIL.getDefaultRootDirPath(),
366      TEST_UTIL.getConfiguration(), htd);
367    HBaseTestingUtility.closeRegionAndWAL(r);
368    final int countPerFamily = 10;
369    final AtomicBoolean goslow = new AtomicBoolean(false);
370    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
371    for (byte[] fam : htd.getColumnFamilyNames()) {
372      scopes.put(fam, 0);
373    }
374    // subclass and doctor a method.
375    AbstractFSWAL<?> wal = newSlowWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(),
376        testName, CONF, null, true, null, null, new Runnable() {
377
378          @Override
379          public void run() {
380            if (goslow.get()) {
381              Threads.sleep(100);
382              LOG.debug("Sleeping before appending 100ms");
383            }
384          }
385        });
386    HRegion region = HRegion.openHRegion(TEST_UTIL.getConfiguration(),
387      TEST_UTIL.getTestFileSystem(), TEST_UTIL.getDefaultRootDirPath(), hri, htd, wal);
388    EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
389    try {
390      List<Put> puts = null;
391      for (byte[] fam : htd.getColumnFamilyNames()) {
392        puts =
393            TestWALReplay.addRegionEdits(rowName, fam, countPerFamily, ee, region, "x");
394      }
395
396      // Now assert edits made it in.
397      final Get g = new Get(rowName);
398      Result result = region.get(g);
399      assertEquals(countPerFamily * htd.getColumnFamilyNames().size(), result.size());
400
401      // Construct a WALEdit and add it a few times to the WAL.
402      WALEdit edits = new WALEdit();
403      for (Put p : puts) {
404        CellScanner cs = p.cellScanner();
405        while (cs.advance()) {
406          edits.add(cs.current());
407        }
408      }
409      // Add any old cluster id.
410      List<UUID> clusterIds = new ArrayList<>(1);
411      clusterIds.add(TEST_UTIL.getRandomUUID());
412      // Now make appends run slow.
413      goslow.set(true);
414      for (int i = 0; i < countPerFamily; i++) {
415        final RegionInfo info = region.getRegionInfo();
416        final WALKeyImpl logkey = new WALKeyImpl(info.getEncodedNameAsBytes(), tableName,
417            System.currentTimeMillis(), clusterIds, -1, -1, region.getMVCC(), scopes);
418        wal.append(info, logkey, edits, true);
419        region.getMVCC().completeAndWait(logkey.getWriteEntry());
420      }
421      region.flush(true);
422      // FlushResult.flushSequenceId is not visible here so go get the current sequence id.
423      long currentSequenceId = region.getReadPoint(null);
424      // Now release the appends
425      goslow.set(false);
426      assertTrue(currentSequenceId >= region.getReadPoint(null));
427    } finally {
428      region.close(true);
429      wal.close();
430    }
431  }
432
433  @Test
434  public void testSyncNoAppend() throws IOException {
435    String testName = currentTest.getMethodName();
436    AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(), testName,
437        CONF, null, true, null, null);
438    try {
439      wal.sync();
440    } finally {
441      wal.close();
442    }
443  }
444
445  @Test
446  public void testWriteEntryCanBeNull() throws IOException {
447    String testName = currentTest.getMethodName();
448    AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(), testName,
449      CONF, null, true, null, null);
450    wal.close();
451    TableDescriptor td = TableDescriptorBuilder.newBuilder(TableName.valueOf("table"))
452      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();
453    RegionInfo ri = RegionInfoBuilder.newBuilder(td.getTableName()).build();
454    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
455    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
456    for (byte[] fam : td.getColumnFamilyNames()) {
457      scopes.put(fam, 0);
458    }
459    long timestamp = System.currentTimeMillis();
460    byte[] row = Bytes.toBytes("row");
461    WALEdit cols = new WALEdit();
462    cols.add(new KeyValue(row, row, row, timestamp, row));
463    WALKeyImpl key =
464        new WALKeyImpl(ri.getEncodedNameAsBytes(), td.getTableName(), SequenceId.NO_SEQUENCE_ID,
465          timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, scopes);
466    try {
467      wal.append(ri, key, cols, true);
468      fail("Should fail since the wal has already been closed");
469    } catch (IOException e) {
470      // expected
471      assertThat(e.getMessage(), containsString("log is closed"));
472      // the WriteEntry should be null since we fail before setting it.
473      assertNull(key.getWriteEntry());
474    }
475  }
476}