001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.util.ArrayList;
028import java.util.HashMap;
029import java.util.Iterator;
030import java.util.List;
031import java.util.Map;
032import java.util.Optional;
033import java.util.concurrent.CompletionException;
034import org.apache.hadoop.hbase.AsyncMetaTableAccessor;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.HConstants;
037import org.apache.hadoop.hbase.HRegionLocation;
038import org.apache.hadoop.hbase.ServerName;
039import org.apache.hadoop.hbase.TableExistsException;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.TableNotFoundException;
042import org.apache.hadoop.hbase.master.LoadBalancer;
043import org.apache.hadoop.hbase.testclassification.ClientTests;
044import org.apache.hadoop.hbase.testclassification.LargeTests;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.junit.ClassRule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049import org.junit.runner.RunWith;
050import org.junit.runners.Parameterized;
051
052/**
053 * Class to test asynchronous table admin operations.
054 * @see TestAsyncTableAdminApi2 This test and it used to be joined it was taking longer than our
055 * ten minute timeout so they were split.
056 * @see TestAsyncTableAdminApi3 Another split out from this class so each runs under ten minutes.
057 */
058@RunWith(Parameterized.class)
059@Category({ LargeTests.class, ClientTests.class })
060public class TestAsyncTableAdminApi extends TestAsyncAdminBase {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064      HBaseClassTestRule.forClass(TestAsyncTableAdminApi.class);
065
066  @Test
067  public void testCreateTable() throws Exception {
068    List<TableDescriptor> tables = admin.listTableDescriptors().get();
069    int numTables = tables.size();
070    createTableWithDefaultConf(tableName);
071    tables = admin.listTableDescriptors().get();
072    assertEquals(numTables + 1, tables.size());
073    assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster().getMaster()
074        .getTableStateManager().isTableState(tableName, TableState.State.ENABLED));
075    assertEquals(TableState.State.ENABLED, getStateFromMeta(tableName));
076  }
077
078  static TableState.State getStateFromMeta(TableName table) throws Exception {
079    Optional<TableState> state = AsyncMetaTableAccessor
080        .getTableState(ASYNC_CONN.getTable(TableName.META_TABLE_NAME), table).get();
081    assertTrue(state.isPresent());
082    return state.get().getState();
083  }
084
085  @Test
086  public void testCreateTableNumberOfRegions() throws Exception {
087    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
088
089    createTableWithDefaultConf(tableName);
090    List<HRegionLocation> regionLocations =
091      AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName)).get();
092    assertEquals("Table should have only 1 region", 1, regionLocations.size());
093
094    final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "_2");
095    createTableWithDefaultConf(tableName2, new byte[][] { new byte[] { 42 } });
096    regionLocations =
097      AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName2)).get();
098    assertEquals("Table should have only 2 region", 2, regionLocations.size());
099
100    final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "_3");
101    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName3);
102    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
103    admin.createTable(builder.build(), "a".getBytes(), "z".getBytes(), 3).join();
104    regionLocations =
105      AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName3)).get();
106    assertEquals("Table should have only 3 region", 3, regionLocations.size());
107
108    final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "_4");
109    builder = TableDescriptorBuilder.newBuilder(tableName4);
110    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
111    try {
112      admin.createTable(builder.build(), "a".getBytes(), "z".getBytes(), 2).join();
113      fail("Should not be able to create a table with only 2 regions using this API.");
114    } catch (CompletionException e) {
115      assertTrue(e.getCause() instanceof IllegalArgumentException);
116    }
117
118    final TableName tableName5 = TableName.valueOf(tableName.getNameAsString() + "_5");
119    builder = TableDescriptorBuilder.newBuilder(tableName5);
120    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
121    admin.createTable(builder.build(), new byte[] { 1 }, new byte[] { 127 }, 16).join();
122    regionLocations =
123      AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName5)).get();
124    assertEquals("Table should have 16 region", 16, regionLocations.size());
125  }
126
127  @Test
128  public void testCreateTableWithRegions() throws Exception {
129    byte[][] splitKeys = { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 }, new byte[] { 3, 3, 3 },
130      new byte[] { 4, 4, 4 }, new byte[] { 5, 5, 5 }, new byte[] { 6, 6, 6 },
131      new byte[] { 7, 7, 7 }, new byte[] { 8, 8, 8 }, new byte[] { 9, 9, 9 }, };
132    int expectedRegions = splitKeys.length + 1;
133    boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
134    createTableWithDefaultConf(tableName, splitKeys);
135
136    boolean tableAvailable = admin.isTableAvailable(tableName, splitKeys).get();
137    assertTrue("Table should be created with splitKyes + 1 rows in META", tableAvailable);
138
139    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
140    List<HRegionLocation> regions =
141      AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName)).get();
142    Iterator<HRegionLocation> hris = regions.iterator();
143
144    assertEquals(
145      "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
146      expectedRegions, regions.size());
147    System.err.println("Found " + regions.size() + " regions");
148
149    RegionInfo hri;
150    hris = regions.iterator();
151    hri = hris.next().getRegion();
152    assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
153    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[0]));
154    hri = hris.next().getRegion();
155    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[0]));
156    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[1]));
157    hri = hris.next().getRegion();
158    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[1]));
159    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[2]));
160    hri = hris.next().getRegion();
161    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[2]));
162    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[3]));
163    hri = hris.next().getRegion();
164    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[3]));
165    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[4]));
166    hri = hris.next().getRegion();
167    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[4]));
168    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[5]));
169    hri = hris.next().getRegion();
170    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[5]));
171    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[6]));
172    hri = hris.next().getRegion();
173    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[6]));
174    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[7]));
175    hri = hris.next().getRegion();
176    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[7]));
177    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[8]));
178    hri = hris.next().getRegion();
179    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[8]));
180    assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
181    if (tablesOnMaster) {
182      verifyRoundRobinDistribution(regions, expectedRegions);
183    }
184
185    // Now test using start/end with a number of regions
186
187    // Use 80 bit numbers to make sure we aren't limited
188    byte[] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
189    byte[] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
190
191    // Splitting into 10 regions, we expect (null,1) ... (9, null)
192    // with (1,2) (2,3) (3,4) (4,5) (5,6) (6,7) (7,8) (8,9) in the middle
193    expectedRegions = 10;
194    final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "_2");
195    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName2);
196    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
197    admin.createTable(builder.build(), startKey, endKey, expectedRegions).join();
198
199    regions =
200      AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName2)).get();
201    assertEquals(
202      "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
203      expectedRegions, regions.size());
204    System.err.println("Found " + regions.size() + " regions");
205
206    hris = regions.iterator();
207    hri = hris.next().getRegion();
208    assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
209    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }));
210    hri = hris.next().getRegion();
211    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }));
212    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }));
213    hri = hris.next().getRegion();
214    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }));
215    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }));
216    hri = hris.next().getRegion();
217    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }));
218    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }));
219    hri = hris.next().getRegion();
220    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }));
221    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }));
222    hri = hris.next().getRegion();
223    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }));
224    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6 }));
225    hri = hris.next().getRegion();
226    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6 }));
227    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 }));
228    hri = hris.next().getRegion();
229    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 }));
230    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 }));
231    hri = hris.next().getRegion();
232    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 }));
233    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }));
234    hri = hris.next().getRegion();
235    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }));
236    assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
237    if (tablesOnMaster) {
238      // This don't work if master is not carrying regions. FIX. TODO.
239      verifyRoundRobinDistribution(regions, expectedRegions);
240    }
241
242    // Try once more with something that divides into something infinite
243    startKey = new byte[] { 0, 0, 0, 0, 0, 0 };
244    endKey = new byte[] { 1, 0, 0, 0, 0, 0 };
245
246    expectedRegions = 5;
247    final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "_3");
248    builder = TableDescriptorBuilder.newBuilder(tableName3);
249    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
250    admin.createTable(builder.build(), startKey, endKey, expectedRegions).join();
251
252    regions =
253      AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName3)).get();
254    assertEquals(
255      "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
256      expectedRegions, regions.size());
257    System.err.println("Found " + regions.size() + " regions");
258    if (tablesOnMaster) {
259      // This don't work if master is not carrying regions. FIX. TODO.
260      verifyRoundRobinDistribution(regions, expectedRegions);
261    }
262
263    // Try an invalid case where there are duplicate split keys
264    splitKeys = new byte[][] { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 },
265      new byte[] { 3, 3, 3 }, new byte[] { 2, 2, 2 } };
266    final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "_4");
267    try {
268      createTableWithDefaultConf(tableName4, splitKeys);
269      fail("Should not be able to create this table because of " + "duplicate split keys");
270    } catch (CompletionException e) {
271      assertTrue(e.getCause() instanceof IllegalArgumentException);
272    }
273  }
274
275  private void verifyRoundRobinDistribution(List<HRegionLocation> regions, int expectedRegions)
276      throws IOException {
277    int numRS = ((ClusterConnection) TEST_UTIL.getConnection()).getCurrentNrHRS();
278
279    Map<ServerName, List<RegionInfo>> server2Regions = new HashMap<>();
280    regions.stream().forEach((loc) -> {
281      ServerName server = loc.getServerName();
282      server2Regions.computeIfAbsent(server, (s) -> new ArrayList<>()).add(loc.getRegion());
283    });
284    if (numRS >= 2) {
285      // Ignore the master region server,
286      // which contains less regions by intention.
287      numRS--;
288    }
289    float average = (float) expectedRegions / numRS;
290    int min = (int) Math.floor(average);
291    int max = (int) Math.ceil(average);
292    server2Regions.values().forEach((regionList) -> {
293      assertTrue(regionList.size() == min || regionList.size() == max);
294    });
295  }
296
297  @Test
298  public void testCreateTableWithOnlyEmptyStartRow() throws Exception {
299    byte[][] splitKeys = new byte[1][];
300    splitKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
301    try {
302      createTableWithDefaultConf(tableName, splitKeys);
303      fail("Test case should fail as empty split key is passed.");
304    } catch (CompletionException e) {
305      assertTrue(e.getCause() instanceof IllegalArgumentException);
306    }
307  }
308
309  @Test
310  public void testCreateTableWithEmptyRowInTheSplitKeys() throws Exception {
311    byte[][] splitKeys = new byte[3][];
312    splitKeys[0] = "region1".getBytes();
313    splitKeys[1] = HConstants.EMPTY_BYTE_ARRAY;
314    splitKeys[2] = "region2".getBytes();
315    try {
316      createTableWithDefaultConf(tableName, splitKeys);
317      fail("Test case should fail as empty split key is passed.");
318    } catch (CompletionException e) {
319      assertTrue(e.getCause() instanceof IllegalArgumentException);
320    }
321  }
322
323  @Test
324  public void testDeleteTable() throws Exception {
325    createTableWithDefaultConf(tableName);
326    assertTrue(admin.tableExists(tableName).get());
327    TEST_UTIL.getAdmin().disableTable(tableName);
328    admin.deleteTable(tableName).join();
329    assertFalse(admin.tableExists(tableName).get());
330  }
331
332  @Test
333  public void testTruncateTable() throws Exception {
334    testTruncateTable(tableName, false);
335  }
336
337  @Test
338  public void testTruncateTablePreservingSplits() throws Exception {
339    testTruncateTable(tableName, true);
340  }
341
342  private void testTruncateTable(final TableName tableName, boolean preserveSplits)
343      throws Exception {
344    byte[][] splitKeys = new byte[2][];
345    splitKeys[0] = Bytes.toBytes(4);
346    splitKeys[1] = Bytes.toBytes(8);
347
348    // Create & Fill the table
349    createTableWithDefaultConf(tableName, splitKeys);
350    AsyncTable<?> table = ASYNC_CONN.getTable(tableName);
351    int expectedRows = 10;
352    for (int i = 0; i < expectedRows; i++) {
353      byte[] data = Bytes.toBytes(String.valueOf(i));
354      Put put = new Put(data);
355      put.addColumn(FAMILY, null, data);
356      table.put(put).join();
357    }
358    assertEquals(10, table.scanAll(new Scan()).get().size());
359    assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
360
361    // Truncate & Verify
362    admin.disableTable(tableName).join();
363    admin.truncateTable(tableName, preserveSplits).join();
364    assertEquals(0, table.scanAll(new Scan()).get().size());
365    if (preserveSplits) {
366      assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
367    } else {
368      assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
369    }
370  }
371
372  @Test
373  public void testCloneTableSchema() throws Exception {
374    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
375    testCloneTableSchema(tableName, newTableName, false);
376  }
377
378  @Test
379  public void testCloneTableSchemaPreservingSplits() throws Exception {
380    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
381    testCloneTableSchema(tableName, newTableName, true);
382  }
383
384  private void testCloneTableSchema(final TableName tableName,
385      final TableName newTableName, boolean preserveSplits) throws Exception {
386    byte[][] splitKeys = new byte[2][];
387    splitKeys[0] = Bytes.toBytes(4);
388    splitKeys[1] = Bytes.toBytes(8);
389    int NUM_FAMILYS = 2;
390    int NUM_REGIONS = 3;
391    int BLOCK_SIZE = 1024;
392    int TTL = 86400;
393    boolean BLOCK_CACHE = false;
394
395    // Create the table
396    TableDescriptor tableDesc = TableDescriptorBuilder
397        .newBuilder(tableName)
398        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_0))
399        .setColumnFamily(ColumnFamilyDescriptorBuilder
400            .newBuilder(FAMILY_1)
401            .setBlocksize(BLOCK_SIZE)
402            .setBlockCacheEnabled(BLOCK_CACHE)
403            .setTimeToLive(TTL)
404            .build()).build();
405    admin.createTable(tableDesc, splitKeys).join();
406
407    assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
408    assertTrue("Table should be created with splitKyes + 1 rows in META",
409        admin.isTableAvailable(tableName, splitKeys).get());
410
411    // Clone & Verify
412    admin.cloneTableSchema(tableName, newTableName, preserveSplits).join();
413    TableDescriptor newTableDesc = admin.getDescriptor(newTableName).get();
414
415    assertEquals(NUM_FAMILYS, newTableDesc.getColumnFamilyCount());
416    assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize());
417    assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled());
418    assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive());
419    TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc);
420
421    if (preserveSplits) {
422      assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size());
423      assertTrue("New table should be created with splitKyes + 1 rows in META",
424          admin.isTableAvailable(newTableName, splitKeys).get());
425    } else {
426      assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size());
427    }
428  }
429
430  @Test
431  public void testCloneTableSchemaWithNonExistentSourceTable() throws Exception {
432    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
433    // test for non-existent source table
434    try {
435      admin.cloneTableSchema(tableName, newTableName, false).join();
436      fail("Should have failed when source table doesn't exist.");
437    } catch (CompletionException e) {
438      assertTrue(e.getCause() instanceof TableNotFoundException);
439    }
440  }
441
442  @Test
443  public void testCloneTableSchemaWithExistentDestinationTable() throws Exception {
444    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
445    byte[] FAMILY_0 = Bytes.toBytes("cf0");
446    TEST_UTIL.createTable(tableName, FAMILY_0);
447    TEST_UTIL.createTable(newTableName, FAMILY_0);
448    // test for existent destination table
449    try {
450      admin.cloneTableSchema(tableName, newTableName, false).join();
451      fail("Should have failed when destination table exists.");
452    } catch (CompletionException e) {
453      assertTrue(e.getCause() instanceof TableExistsException);
454    }
455  }
456}