001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.replication;
019
020import static org.junit.Assert.assertArrayEquals;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertNotNull;
024import static org.junit.Assert.assertTrue;
025import static org.junit.Assert.fail;
026
027import java.io.IOException;
028import java.util.TreeMap;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileStatus;
031import org.apache.hadoop.fs.FileSystem;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.Cell;
034import org.apache.hadoop.hbase.CellUtil;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.HBaseTestingUtility;
037import org.apache.hadoop.hbase.HConstants;
038import org.apache.hadoop.hbase.TableName;
039import org.apache.hadoop.hbase.client.Admin;
040import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
041import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
042import org.apache.hadoop.hbase.client.Connection;
043import org.apache.hadoop.hbase.client.ConnectionFactory;
044import org.apache.hadoop.hbase.client.Delete;
045import org.apache.hadoop.hbase.client.Get;
046import org.apache.hadoop.hbase.client.Put;
047import org.apache.hadoop.hbase.client.Result;
048import org.apache.hadoop.hbase.client.ResultScanner;
049import org.apache.hadoop.hbase.client.Scan;
050import org.apache.hadoop.hbase.client.Table;
051import org.apache.hadoop.hbase.client.TableDescriptor;
052import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
053import org.apache.hadoop.hbase.mapreduce.replication.VerifyReplication;
054import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
055import org.apache.hadoop.hbase.testclassification.LargeTests;
056import org.apache.hadoop.hbase.testclassification.ReplicationTests;
057import org.apache.hadoop.hbase.util.Bytes;
058import org.apache.hadoop.hbase.util.FSUtils;
059import org.apache.hadoop.mapreduce.Job;
060import org.junit.Before;
061import org.junit.ClassRule;
062import org.junit.Rule;
063import org.junit.Test;
064import org.junit.experimental.categories.Category;
065import org.junit.rules.TestName;
066import org.slf4j.Logger;
067import org.slf4j.LoggerFactory;
068
069import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
070
071@Category({ ReplicationTests.class, LargeTests.class })
072public class TestVerifyReplication extends TestReplicationBase {
073
074  @ClassRule
075  public static final HBaseClassTestRule CLASS_RULE =
076      HBaseClassTestRule.forClass(TestVerifyReplication.class);
077
078  private static final Logger LOG = LoggerFactory.getLogger(TestVerifyReplication.class);
079
080  private static final String PEER_ID = "2";
081
082  @Rule
083  public TestName name = new TestName();
084
085  @Before
086  public void setUp() throws Exception {
087    cleanUp();
088  }
089
090  private void runVerifyReplication(String[] args, int expectedGoodRows, int expectedBadRows)
091      throws IOException, InterruptedException, ClassNotFoundException {
092    Job job = new VerifyReplication().createSubmittableJob(new Configuration(conf1), args);
093    if (job == null) {
094      fail("Job wasn't created, see the log");
095    }
096    if (!job.waitForCompletion(true)) {
097      fail("Job failed, see the log");
098    }
099    assertEquals(expectedGoodRows,
100      job.getCounters().findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue());
101    assertEquals(expectedBadRows,
102      job.getCounters().findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue());
103  }
104
105  /**
106   * Do a small loading into a table, make sure the data is really the same, then run the
107   * VerifyReplication job to check the results. Do a second comparison where all the cells are
108   * different.
109   */
110  @Test
111  public void testVerifyRepJob() throws Exception {
112    // Populate the tables, at the same time it guarantees that the tables are
113    // identical since it does the check
114    runSmallBatchTest();
115
116    String[] args = new String[] { PEER_ID, tableName.getNameAsString() };
117    runVerifyReplication(args, NB_ROWS_IN_BATCH, 0);
118
119    Scan scan = new Scan();
120    ResultScanner rs = htable2.getScanner(scan);
121    Put put = null;
122    for (Result result : rs) {
123      put = new Put(result.getRow());
124      Cell firstVal = result.rawCells()[0];
125      put.addColumn(CellUtil.cloneFamily(firstVal), CellUtil.cloneQualifier(firstVal),
126        Bytes.toBytes("diff data"));
127      htable2.put(put);
128    }
129    Delete delete = new Delete(put.getRow());
130    htable2.delete(delete);
131    runVerifyReplication(args, 0, NB_ROWS_IN_BATCH);
132  }
133
134  /**
135   * Load a row into a table, make sure the data is really the same, delete the row, make sure the
136   * delete marker is replicated, run verify replication with and without raw to check the results.
137   */
138  @Test
139  public void testVerifyRepJobWithRawOptions() throws Exception {
140    LOG.info(name.getMethodName());
141
142    final TableName tableName = TableName.valueOf(name.getMethodName());
143    byte[] familyname = Bytes.toBytes("fam_raw");
144    byte[] row = Bytes.toBytes("row_raw");
145
146    Table lHtable1 = null;
147    Table lHtable2 = null;
148
149    try {
150      ColumnFamilyDescriptor fam = ColumnFamilyDescriptorBuilder.newBuilder(familyname)
151          .setMaxVersions(100).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build();
152      TableDescriptor table =
153          TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(fam).build();
154      scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
155      for (ColumnFamilyDescriptor f : table.getColumnFamilies()) {
156        scopes.put(f.getName(), f.getScope());
157      }
158
159      Connection connection1 = ConnectionFactory.createConnection(conf1);
160      Connection connection2 = ConnectionFactory.createConnection(conf2);
161      try (Admin admin1 = connection1.getAdmin()) {
162        admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
163      }
164      try (Admin admin2 = connection2.getAdmin()) {
165        admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
166      }
167      utility1.waitUntilAllRegionsAssigned(tableName);
168      utility2.waitUntilAllRegionsAssigned(tableName);
169
170      lHtable1 = utility1.getConnection().getTable(tableName);
171      lHtable2 = utility2.getConnection().getTable(tableName);
172
173      Put put = new Put(row);
174      put.addColumn(familyname, row, row);
175      lHtable1.put(put);
176
177      Get get = new Get(row);
178      for (int i = 0; i < NB_RETRIES; i++) {
179        if (i == NB_RETRIES - 1) {
180          fail("Waited too much time for put replication");
181        }
182        Result res = lHtable2.get(get);
183        if (res.isEmpty()) {
184          LOG.info("Row not available");
185          Thread.sleep(SLEEP_TIME);
186        } else {
187          assertArrayEquals(res.value(), row);
188          break;
189        }
190      }
191
192      Delete del = new Delete(row);
193      lHtable1.delete(del);
194
195      get = new Get(row);
196      for (int i = 0; i < NB_RETRIES; i++) {
197        if (i == NB_RETRIES - 1) {
198          fail("Waited too much time for del replication");
199        }
200        Result res = lHtable2.get(get);
201        if (res.size() >= 1) {
202          LOG.info("Row not deleted");
203          Thread.sleep(SLEEP_TIME);
204        } else {
205          break;
206        }
207      }
208
209      // Checking verifyReplication for the default behavior.
210      String[] argsWithoutRaw = new String[] { PEER_ID, tableName.getNameAsString() };
211      runVerifyReplication(argsWithoutRaw, 0, 0);
212
213      // Checking verifyReplication with raw
214      String[] argsWithRawAsTrue = new String[] { "--raw", PEER_ID, tableName.getNameAsString() };
215      runVerifyReplication(argsWithRawAsTrue, 1, 0);
216    } finally {
217      if (lHtable1 != null) {
218        lHtable1.close();
219      }
220      if (lHtable2 != null) {
221        lHtable2.close();
222      }
223    }
224  }
225
226  // VerifyReplication should honor versions option
227  @Test
228  public void testHBase14905() throws Exception {
229    // normal Batch tests
230    byte[] qualifierName = Bytes.toBytes("f1");
231    Put put = new Put(Bytes.toBytes("r1"));
232    put.addColumn(famName, qualifierName, Bytes.toBytes("v1002"));
233    htable1.put(put);
234    put.addColumn(famName, qualifierName, Bytes.toBytes("v1001"));
235    htable1.put(put);
236    put.addColumn(famName, qualifierName, Bytes.toBytes("v1112"));
237    htable1.put(put);
238
239    Scan scan = new Scan();
240    scan.readVersions(100);
241    ResultScanner scanner1 = htable1.getScanner(scan);
242    Result[] res1 = scanner1.next(1);
243    scanner1.close();
244
245    assertEquals(1, res1.length);
246    assertEquals(3, res1[0].getColumnCells(famName, qualifierName).size());
247
248    for (int i = 0; i < NB_RETRIES; i++) {
249      scan = new Scan();
250      scan.readVersions(100);
251      scanner1 = htable2.getScanner(scan);
252      res1 = scanner1.next(1);
253      scanner1.close();
254      if (res1.length != 1) {
255        LOG.info("Only got " + res1.length + " rows");
256        Thread.sleep(SLEEP_TIME);
257      } else {
258        int cellNumber = res1[0].getColumnCells(famName, Bytes.toBytes("f1")).size();
259        if (cellNumber != 3) {
260          LOG.info("Only got " + cellNumber + " cells");
261          Thread.sleep(SLEEP_TIME);
262        } else {
263          break;
264        }
265      }
266      if (i == NB_RETRIES - 1) {
267        fail("Waited too much time for normal batch replication");
268      }
269    }
270
271    put.addColumn(famName, qualifierName, Bytes.toBytes("v1111"));
272    htable2.put(put);
273    put.addColumn(famName, qualifierName, Bytes.toBytes("v1112"));
274    htable2.put(put);
275
276    scan = new Scan();
277    scan.readVersions(100);
278    scanner1 = htable2.getScanner(scan);
279    res1 = scanner1.next(NB_ROWS_IN_BATCH);
280    scanner1.close();
281
282    assertEquals(1, res1.length);
283    assertEquals(5, res1[0].getColumnCells(famName, qualifierName).size());
284
285    String[] args = new String[] { "--versions=100", PEER_ID, tableName.getNameAsString() };
286    runVerifyReplication(args, 0, 1);
287  }
288
289  // VerifyReplication should honor versions option
290  @Test
291  public void testVersionMismatchHBase14905() throws Exception {
292    // normal Batch tests
293    byte[] qualifierName = Bytes.toBytes("f1");
294    Put put = new Put(Bytes.toBytes("r1"));
295    long ts = System.currentTimeMillis();
296    put.addColumn(famName, qualifierName, ts + 1, Bytes.toBytes("v1"));
297    htable1.put(put);
298    put.addColumn(famName, qualifierName, ts + 2, Bytes.toBytes("v2"));
299    htable1.put(put);
300    put.addColumn(famName, qualifierName, ts + 3, Bytes.toBytes("v3"));
301    htable1.put(put);
302
303    Scan scan = new Scan();
304    scan.readVersions(100);
305    ResultScanner scanner1 = htable1.getScanner(scan);
306    Result[] res1 = scanner1.next(1);
307    scanner1.close();
308
309    assertEquals(1, res1.length);
310    assertEquals(3, res1[0].getColumnCells(famName, qualifierName).size());
311
312    for (int i = 0; i < NB_RETRIES; i++) {
313      scan = new Scan();
314      scan.readVersions(100);
315      scanner1 = htable2.getScanner(scan);
316      res1 = scanner1.next(1);
317      scanner1.close();
318      if (res1.length != 1) {
319        LOG.info("Only got " + res1.length + " rows");
320        Thread.sleep(SLEEP_TIME);
321      } else {
322        int cellNumber = res1[0].getColumnCells(famName, Bytes.toBytes("f1")).size();
323        if (cellNumber != 3) {
324          LOG.info("Only got " + cellNumber + " cells");
325          Thread.sleep(SLEEP_TIME);
326        } else {
327          break;
328        }
329      }
330      if (i == NB_RETRIES - 1) {
331        fail("Waited too much time for normal batch replication");
332      }
333    }
334
335    try {
336      // Disabling replication and modifying the particular version of the cell to validate the
337      // feature.
338      hbaseAdmin.disableReplicationPeer(PEER_ID);
339      Put put2 = new Put(Bytes.toBytes("r1"));
340      put2.addColumn(famName, qualifierName, ts + 2, Bytes.toBytes("v99"));
341      htable2.put(put2);
342
343      scan = new Scan();
344      scan.readVersions(100);
345      scanner1 = htable2.getScanner(scan);
346      res1 = scanner1.next(NB_ROWS_IN_BATCH);
347      scanner1.close();
348      assertEquals(1, res1.length);
349      assertEquals(3, res1[0].getColumnCells(famName, qualifierName).size());
350
351      String[] args = new String[] { "--versions=100", PEER_ID, tableName.getNameAsString() };
352      runVerifyReplication(args, 0, 1);
353    } finally {
354      hbaseAdmin.enableReplicationPeer(PEER_ID);
355    }
356  }
357
358  @Test
359  public void testVerifyReplicationPrefixFiltering() throws Exception {
360    final byte[] prefixRow = Bytes.toBytes("prefixrow");
361    final byte[] prefixRow2 = Bytes.toBytes("secondrow");
362    loadData("prefixrow", prefixRow);
363    loadData("secondrow", prefixRow2);
364    loadData("aaa", row);
365    loadData("zzz", row);
366    waitForReplication(NB_ROWS_IN_BATCH * 4, NB_RETRIES * 4);
367    String[] args =
368        new String[] { "--row-prefixes=prefixrow,secondrow", PEER_ID, tableName.getNameAsString() };
369    runVerifyReplication(args, NB_ROWS_IN_BATCH * 2, 0);
370  }
371
372  @Test
373  public void testVerifyReplicationSnapshotArguments() {
374    String[] args =
375        new String[] { "--sourceSnapshotName=snapshot1", "2", tableName.getNameAsString() };
376    assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
377
378    args = new String[] { "--sourceSnapshotTmpDir=tmp", "2", tableName.getNameAsString() };
379    assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
380
381    args = new String[] { "--sourceSnapshotName=snapshot1", "--sourceSnapshotTmpDir=tmp", "2",
382        tableName.getNameAsString() };
383    assertTrue(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
384
385    args = new String[] { "--peerSnapshotName=snapshot1", "2", tableName.getNameAsString() };
386    assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
387
388    args = new String[] { "--peerSnapshotTmpDir=/tmp/", "2", tableName.getNameAsString() };
389    assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
390
391    args = new String[] { "--peerSnapshotName=snapshot1", "--peerSnapshotTmpDir=/tmp/",
392        "--peerFSAddress=tempfs", "--peerHBaseRootAddress=hdfs://tempfs:50070/hbase/", "2",
393        tableName.getNameAsString() };
394    assertTrue(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
395
396    args = new String[] { "--sourceSnapshotName=snapshot1", "--sourceSnapshotTmpDir=/tmp/",
397        "--peerSnapshotName=snapshot2", "--peerSnapshotTmpDir=/tmp/", "--peerFSAddress=tempfs",
398        "--peerHBaseRootAddress=hdfs://tempfs:50070/hbase/", "2", tableName.getNameAsString() };
399
400    assertTrue(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
401  }
402
403  private void checkRestoreTmpDir(Configuration conf, String restoreTmpDir, int expectedCount)
404      throws IOException {
405    FileSystem fs = FileSystem.get(conf);
406    FileStatus[] subDirectories = fs.listStatus(new Path(restoreTmpDir));
407    assertNotNull(subDirectories);
408    assertEquals(subDirectories.length, expectedCount);
409    for (int i = 0; i < expectedCount; i++) {
410      assertTrue(subDirectories[i].isDirectory());
411    }
412  }
413
414  @Test
415  public void testVerifyReplicationWithSnapshotSupport() throws Exception {
416    // Populate the tables, at the same time it guarantees that the tables are
417    // identical since it does the check
418    runSmallBatchTest();
419
420    // Take source and target tables snapshot
421    Path rootDir = FSUtils.getRootDir(conf1);
422    FileSystem fs = rootDir.getFileSystem(conf1);
423    String sourceSnapshotName = "sourceSnapshot-" + System.currentTimeMillis();
424    SnapshotTestingUtils.createSnapshotAndValidate(utility1.getAdmin(), tableName,
425      new String(famName), sourceSnapshotName, rootDir, fs, true);
426
427    // Take target snapshot
428    Path peerRootDir = FSUtils.getRootDir(conf2);
429    FileSystem peerFs = peerRootDir.getFileSystem(conf2);
430    String peerSnapshotName = "peerSnapshot-" + System.currentTimeMillis();
431    SnapshotTestingUtils.createSnapshotAndValidate(utility2.getAdmin(), tableName,
432      new String(famName), peerSnapshotName, peerRootDir, peerFs, true);
433
434    String peerFSAddress = peerFs.getUri().toString();
435    String temPath1 = utility1.getRandomDir().toString();
436    String temPath2 = "/tmp2";
437
438    String[] args = new String[] { "--sourceSnapshotName=" + sourceSnapshotName,
439        "--sourceSnapshotTmpDir=" + temPath1, "--peerSnapshotName=" + peerSnapshotName,
440        "--peerSnapshotTmpDir=" + temPath2, "--peerFSAddress=" + peerFSAddress,
441        "--peerHBaseRootAddress=" + FSUtils.getRootDir(conf2), "2", tableName.getNameAsString() };
442
443    Job job = new VerifyReplication().createSubmittableJob(conf1, args);
444    if (job == null) {
445      fail("Job wasn't created, see the log");
446    }
447    if (!job.waitForCompletion(true)) {
448      fail("Job failed, see the log");
449    }
450    assertEquals(NB_ROWS_IN_BATCH,
451      job.getCounters().findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue());
452    assertEquals(0,
453      job.getCounters().findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue());
454
455    checkRestoreTmpDir(conf1, temPath1, 1);
456    checkRestoreTmpDir(conf2, temPath2, 1);
457
458    Scan scan = new Scan();
459    ResultScanner rs = htable2.getScanner(scan);
460    Put put = null;
461    for (Result result : rs) {
462      put = new Put(result.getRow());
463      Cell firstVal = result.rawCells()[0];
464      put.addColumn(CellUtil.cloneFamily(firstVal), CellUtil.cloneQualifier(firstVal),
465        Bytes.toBytes("diff data"));
466      htable2.put(put);
467    }
468    Delete delete = new Delete(put.getRow());
469    htable2.delete(delete);
470
471    sourceSnapshotName = "sourceSnapshot-" + System.currentTimeMillis();
472    SnapshotTestingUtils.createSnapshotAndValidate(utility1.getAdmin(), tableName,
473      new String(famName), sourceSnapshotName, rootDir, fs, true);
474
475    peerSnapshotName = "peerSnapshot-" + System.currentTimeMillis();
476    SnapshotTestingUtils.createSnapshotAndValidate(utility2.getAdmin(), tableName,
477      new String(famName), peerSnapshotName, peerRootDir, peerFs, true);
478
479    args = new String[] { "--sourceSnapshotName=" + sourceSnapshotName,
480        "--sourceSnapshotTmpDir=" + temPath1, "--peerSnapshotName=" + peerSnapshotName,
481        "--peerSnapshotTmpDir=" + temPath2, "--peerFSAddress=" + peerFSAddress,
482        "--peerHBaseRootAddress=" + FSUtils.getRootDir(conf2), "2", tableName.getNameAsString() };
483
484    job = new VerifyReplication().createSubmittableJob(conf1, args);
485    if (job == null) {
486      fail("Job wasn't created, see the log");
487    }
488    if (!job.waitForCompletion(true)) {
489      fail("Job failed, see the log");
490    }
491    assertEquals(0,
492      job.getCounters().findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue());
493    assertEquals(NB_ROWS_IN_BATCH,
494      job.getCounters().findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue());
495
496    checkRestoreTmpDir(conf1, temPath1, 2);
497    checkRestoreTmpDir(conf2, temPath2, 2);
498  }
499}