001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.apache.hadoop.hbase.client.AsyncProcess.START_LOG_ERRORS_AFTER_COUNT_KEY;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertNotNull;
024import static org.junit.Assert.assertNull;
025import static org.junit.Assert.assertTrue;
026import static org.junit.Assert.fail;
027
028import java.io.IOException;
029import java.util.ArrayList;
030import java.util.HashMap;
031import java.util.HashSet;
032import java.util.List;
033import java.util.Map;
034import java.util.Set;
035import java.util.concurrent.CompletionException;
036import org.apache.hadoop.hbase.HBaseClassTestRule;
037import org.apache.hadoop.hbase.HConstants;
038import org.apache.hadoop.hbase.ServerName;
039import org.apache.hadoop.hbase.TableName;
040import org.apache.hadoop.hbase.replication.ReplicationException;
041import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
042import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
043import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
044import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
045import org.apache.hadoop.hbase.testclassification.ClientTests;
046import org.apache.hadoop.hbase.testclassification.LargeTests;
047import org.junit.After;
048import org.junit.BeforeClass;
049import org.junit.ClassRule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052import org.junit.runner.RunWith;
053import org.junit.runners.Parameterized;
054
055/**
056 * Class to test asynchronous replication admin operations.
057 */
058@RunWith(Parameterized.class)
059@Category({LargeTests.class, ClientTests.class})
060public class TestAsyncReplicationAdminApi extends TestAsyncAdminBase {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064      HBaseClassTestRule.forClass(TestAsyncReplicationAdminApi.class);
065
066  private final String ID_ONE = "1";
067  private final String KEY_ONE = "127.0.0.1:2181:/hbase";
068  private final String ID_TWO = "2";
069  private final String KEY_TWO = "127.0.0.1:2181:/hbase2";
070
071  @BeforeClass
072  public static void setUpBeforeClass() throws Exception {
073    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 60000);
074    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 120000);
075    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
076    TEST_UTIL.getConfiguration().setInt(START_LOG_ERRORS_AFTER_COUNT_KEY, 0);
077    TEST_UTIL.startMiniCluster();
078    ASYNC_CONN = ConnectionFactory.createAsyncConnection(TEST_UTIL.getConfiguration()).get();
079  }
080
081  @After
082  public void clearPeerAndQueues() throws IOException, ReplicationException {
083    try {
084      admin.removeReplicationPeer(ID_ONE).join();
085    } catch (Exception e) {
086    }
087    try {
088      admin.removeReplicationPeer(ID_TWO).join();
089    } catch (Exception e) {
090    }
091    ReplicationQueueStorage queueStorage = ReplicationStorageFactory
092        .getReplicationQueueStorage(TEST_UTIL.getZooKeeperWatcher(), TEST_UTIL.getConfiguration());
093    for (ServerName serverName : queueStorage.getListOfReplicators()) {
094      for (String queue : queueStorage.getAllQueues(serverName)) {
095        queueStorage.removeQueue(serverName, queue);
096      }
097    }
098  }
099
100  @Test
101  public void testAddRemovePeer() throws Exception {
102    ReplicationPeerConfig rpc1 = new ReplicationPeerConfig();
103    rpc1.setClusterKey(KEY_ONE);
104    ReplicationPeerConfig rpc2 = new ReplicationPeerConfig();
105    rpc2.setClusterKey(KEY_TWO);
106    // Add a valid peer
107    admin.addReplicationPeer(ID_ONE, rpc1).join();
108    // try adding the same (fails)
109    try {
110      admin.addReplicationPeer(ID_ONE, rpc1).join();
111      fail("Test case should fail as adding a same peer.");
112    } catch (CompletionException e) {
113      // OK!
114    }
115    assertEquals(1, admin.listReplicationPeers().get().size());
116    // Try to remove an inexisting peer
117    try {
118      admin.removeReplicationPeer(ID_TWO).join();
119      fail("Test case should fail as removing a inexisting peer.");
120    } catch (CompletionException e) {
121      // OK!
122    }
123    assertEquals(1, admin.listReplicationPeers().get().size());
124    // Add a second since multi-slave is supported
125    admin.addReplicationPeer(ID_TWO, rpc2).join();
126    assertEquals(2, admin.listReplicationPeers().get().size());
127    // Remove the first peer we added
128    admin.removeReplicationPeer(ID_ONE).join();
129    assertEquals(1, admin.listReplicationPeers().get().size());
130    admin.removeReplicationPeer(ID_TWO).join();
131    assertEquals(0, admin.listReplicationPeers().get().size());
132  }
133
134  @Test
135  public void testPeerConfig() throws Exception {
136    ReplicationPeerConfig config = new ReplicationPeerConfig();
137    config.setClusterKey(KEY_ONE);
138    config.getConfiguration().put("key1", "value1");
139    config.getConfiguration().put("key2", "value2");
140    admin.addReplicationPeer(ID_ONE, config).join();
141
142    List<ReplicationPeerDescription> peers = admin.listReplicationPeers().get();
143    assertEquals(1, peers.size());
144    ReplicationPeerDescription peerOne = peers.get(0);
145    assertNotNull(peerOne);
146    assertEquals("value1", peerOne.getPeerConfig().getConfiguration().get("key1"));
147    assertEquals("value2", peerOne.getPeerConfig().getConfiguration().get("key2"));
148
149    admin.removeReplicationPeer(ID_ONE).join();
150  }
151
152  @Test
153  public void testEnableDisablePeer() throws Exception {
154    ReplicationPeerConfig rpc1 = new ReplicationPeerConfig();
155    rpc1.setClusterKey(KEY_ONE);
156    admin.addReplicationPeer(ID_ONE, rpc1).join();
157    List<ReplicationPeerDescription> peers = admin.listReplicationPeers().get();
158    assertEquals(1, peers.size());
159    assertTrue(peers.get(0).isEnabled());
160
161    admin.disableReplicationPeer(ID_ONE).join();
162    peers = admin.listReplicationPeers().get();
163    assertEquals(1, peers.size());
164    assertFalse(peers.get(0).isEnabled());
165    admin.removeReplicationPeer(ID_ONE).join();
166  }
167
168  @Test
169  public void testAppendPeerTableCFs() throws Exception {
170    ReplicationPeerConfig rpc1 = new ReplicationPeerConfig();
171    rpc1.setClusterKey(KEY_ONE);
172    final TableName tableName1 = TableName.valueOf(tableName.getNameAsString() + "t1");
173    final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "t2");
174    final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "t3");
175    final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "t4");
176    final TableName tableName5 = TableName.valueOf(tableName.getNameAsString() + "t5");
177    final TableName tableName6 = TableName.valueOf(tableName.getNameAsString() + "t6");
178
179    // Add a valid peer
180    admin.addReplicationPeer(ID_ONE, rpc1).join();
181    rpc1.setReplicateAllUserTables(false);
182    admin.updateReplicationPeerConfig(ID_ONE, rpc1).join();
183
184    Map<TableName, List<String>> tableCFs = new HashMap<>();
185
186    // append table t1 to replication
187    tableCFs.put(tableName1, null);
188    admin.appendReplicationPeerTableCFs(ID_ONE, tableCFs).join();
189    Map<TableName, List<String>> result = admin.getReplicationPeerConfig(ID_ONE).get()
190        .getTableCFsMap();
191    assertEquals(1, result.size());
192    assertEquals(true, result.containsKey(tableName1));
193    assertNull(result.get(tableName1));
194
195    // append table t2 to replication
196    tableCFs.clear();
197    tableCFs.put(tableName2, null);
198    admin.appendReplicationPeerTableCFs(ID_ONE, tableCFs).join();
199    result = admin.getReplicationPeerConfig(ID_ONE).get().getTableCFsMap();
200    assertEquals(2, result.size());
201    assertTrue("Should contain t1", result.containsKey(tableName1));
202    assertTrue("Should contain t2", result.containsKey(tableName2));
203    assertNull(result.get(tableName1));
204    assertNull(result.get(tableName2));
205
206    // append table column family: f1 of t3 to replication
207    tableCFs.clear();
208    tableCFs.put(tableName3, new ArrayList<>());
209    tableCFs.get(tableName3).add("f1");
210    admin.appendReplicationPeerTableCFs(ID_ONE, tableCFs).join();
211    result = admin.getReplicationPeerConfig(ID_ONE).get().getTableCFsMap();
212    assertEquals(3, result.size());
213    assertTrue("Should contain t1", result.containsKey(tableName1));
214    assertTrue("Should contain t2", result.containsKey(tableName2));
215    assertTrue("Should contain t3", result.containsKey(tableName3));
216    assertNull(result.get(tableName1));
217    assertNull(result.get(tableName2));
218    assertEquals(1, result.get(tableName3).size());
219    assertEquals("f1", result.get(tableName3).get(0));
220
221    // append table column family: f1,f2 of t4 to replication
222    tableCFs.clear();
223    tableCFs.put(tableName4, new ArrayList<>());
224    tableCFs.get(tableName4).add("f1");
225    tableCFs.get(tableName4).add("f2");
226    admin.appendReplicationPeerTableCFs(ID_ONE, tableCFs).join();
227    result = admin.getReplicationPeerConfig(ID_ONE).get().getTableCFsMap();
228    assertEquals(4, result.size());
229    assertTrue("Should contain t1", result.containsKey(tableName1));
230    assertTrue("Should contain t2", result.containsKey(tableName2));
231    assertTrue("Should contain t3", result.containsKey(tableName3));
232    assertTrue("Should contain t4", result.containsKey(tableName4));
233    assertNull(result.get(tableName1));
234    assertNull(result.get(tableName2));
235    assertEquals(1, result.get(tableName3).size());
236    assertEquals("f1", result.get(tableName3).get(0));
237    assertEquals(2, result.get(tableName4).size());
238    assertEquals("f1", result.get(tableName4).get(0));
239    assertEquals("f2", result.get(tableName4).get(1));
240
241    // append "table5" => [], then append "table5" => ["f1"]
242    tableCFs.clear();
243    tableCFs.put(tableName5, new ArrayList<>());
244    admin.appendReplicationPeerTableCFs(ID_ONE, tableCFs).join();
245    tableCFs.clear();
246    tableCFs.put(tableName5, new ArrayList<>());
247    tableCFs.get(tableName5).add("f1");
248    admin.appendReplicationPeerTableCFs(ID_ONE, tableCFs).join();
249    result = admin.getReplicationPeerConfig(ID_ONE).get().getTableCFsMap();
250    assertEquals(5, result.size());
251    assertTrue("Should contain t5", result.containsKey(tableName5));
252    // null means replication all cfs of tab5
253    assertNull(result.get(tableName5));
254
255    // append "table6" => ["f1"], then append "table6" => []
256    tableCFs.clear();
257    tableCFs.put(tableName6, new ArrayList<>());
258    tableCFs.get(tableName6).add("f1");
259    admin.appendReplicationPeerTableCFs(ID_ONE, tableCFs).join();
260    tableCFs.clear();
261    tableCFs.put(tableName6, new ArrayList<>());
262    admin.appendReplicationPeerTableCFs(ID_ONE, tableCFs).join();
263    result = admin.getReplicationPeerConfig(ID_ONE).get().getTableCFsMap();
264    assertEquals(6, result.size());
265    assertTrue("Should contain t6", result.containsKey(tableName6));
266    // null means replication all cfs of tab6
267    assertNull(result.get(tableName6));
268
269    admin.removeReplicationPeer(ID_ONE).join();
270  }
271
272  @Test
273  public void testRemovePeerTableCFs() throws Exception {
274    ReplicationPeerConfig rpc1 = new ReplicationPeerConfig();
275    rpc1.setClusterKey(KEY_ONE);
276    final TableName tableName1 = TableName.valueOf(tableName.getNameAsString() + "t1");
277    final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "t2");
278    final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "t3");
279    final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "t4");
280    // Add a valid peer
281    admin.addReplicationPeer(ID_ONE, rpc1).join();
282    rpc1.setReplicateAllUserTables(false);
283    admin.updateReplicationPeerConfig(ID_ONE, rpc1).join();
284
285    Map<TableName, List<String>> tableCFs = new HashMap<>();
286    try {
287      tableCFs.put(tableName3, null);
288      admin.removeReplicationPeerTableCFs(ID_ONE, tableCFs).join();
289      fail("Test case should fail as removing table-cfs from a peer whose table-cfs is null");
290    } catch (CompletionException e) {
291      assertTrue(e.getCause() instanceof ReplicationException);
292    }
293    assertNull(admin.getReplicationPeerConfig(ID_ONE).get().getTableCFsMap());
294
295    tableCFs.clear();
296    tableCFs.put(tableName1, null);
297    tableCFs.put(tableName2, new ArrayList<>());
298    tableCFs.get(tableName2).add("cf1");
299    admin.appendReplicationPeerTableCFs(ID_ONE, tableCFs).join();
300    try {
301      tableCFs.clear();
302      tableCFs.put(tableName3, null);
303      admin.removeReplicationPeerTableCFs(ID_ONE, tableCFs).join();
304      fail("Test case should fail as removing table-cfs from a peer whose table-cfs didn't contain t3");
305    } catch (CompletionException e) {
306      assertTrue(e.getCause() instanceof ReplicationException);
307    }
308    Map<TableName, List<String>> result = admin.getReplicationPeerConfig(ID_ONE).get()
309        .getTableCFsMap();
310    assertEquals(2, result.size());
311    assertTrue("Should contain t1", result.containsKey(tableName1));
312    assertTrue("Should contain t2", result.containsKey(tableName2));
313    assertNull(result.get(tableName1));
314    assertEquals(1, result.get(tableName2).size());
315    assertEquals("cf1", result.get(tableName2).get(0));
316
317    try {
318      tableCFs.clear();
319      tableCFs.put(tableName1, new ArrayList<>());
320      tableCFs.get(tableName1).add("cf1");
321      admin.removeReplicationPeerTableCFs(ID_ONE, tableCFs).join();
322      fail("Test case should fail, because table t1 didn't specify cfs in peer config");
323    } catch (CompletionException e) {
324      assertTrue(e.getCause() instanceof ReplicationException);
325    }
326    tableCFs.clear();
327    tableCFs.put(tableName1, null);
328    admin.removeReplicationPeerTableCFs(ID_ONE, tableCFs).join();
329    result = admin.getReplicationPeerConfig(ID_ONE).get().getTableCFsMap();
330    assertEquals(1, result.size());
331    assertEquals(1, result.get(tableName2).size());
332    assertEquals("cf1", result.get(tableName2).get(0));
333
334    try {
335      tableCFs.clear();
336      tableCFs.put(tableName2, null);
337      admin.removeReplicationPeerTableCFs(ID_ONE, tableCFs).join();
338      fail("Test case should fail, because table t2 hase specified cfs in peer config");
339    } catch (CompletionException e) {
340      assertTrue(e.getCause() instanceof ReplicationException);
341    }
342    tableCFs.clear();
343    tableCFs.put(tableName2, new ArrayList<>());
344    tableCFs.get(tableName2).add("cf1");
345    admin.removeReplicationPeerTableCFs(ID_ONE, tableCFs).join();
346    assertNull(admin.getReplicationPeerConfig(ID_ONE).get().getTableCFsMap());
347
348    tableCFs.clear();
349    tableCFs.put(tableName4, new ArrayList<>());
350    admin.appendReplicationPeerTableCFs(ID_ONE, tableCFs).join();
351    admin.removeReplicationPeerTableCFs(ID_ONE, tableCFs).join();
352    assertNull(admin.getReplicationPeerConfig(ID_ONE).get().getTableCFsMap());
353
354    admin.removeReplicationPeer(ID_ONE);
355  }
356
357  @Test
358  public void testSetPeerNamespaces() throws Exception {
359    String ns1 = "ns1";
360    String ns2 = "ns2";
361
362    ReplicationPeerConfig rpc = new ReplicationPeerConfig();
363    rpc.setClusterKey(KEY_ONE);
364    admin.addReplicationPeer(ID_ONE, rpc).join();
365    rpc.setReplicateAllUserTables(false);
366    admin.updateReplicationPeerConfig(ID_ONE, rpc).join();
367
368    // add ns1 and ns2 to peer config
369    rpc = admin.getReplicationPeerConfig(ID_ONE).get();
370    Set<String> namespaces = new HashSet<>();
371    namespaces.add(ns1);
372    namespaces.add(ns2);
373    rpc.setNamespaces(namespaces);
374    admin.updateReplicationPeerConfig(ID_ONE, rpc).join();
375    namespaces = admin.getReplicationPeerConfig(ID_ONE).get().getNamespaces();
376    assertEquals(2, namespaces.size());
377    assertTrue(namespaces.contains(ns1));
378    assertTrue(namespaces.contains(ns2));
379
380    // update peer config only contains ns1
381    rpc = admin.getReplicationPeerConfig(ID_ONE).get();
382    namespaces = new HashSet<>();
383    namespaces.add(ns1);
384    rpc.setNamespaces(namespaces);
385    admin.updateReplicationPeerConfig(ID_ONE, rpc).join();
386    namespaces = admin.getReplicationPeerConfig(ID_ONE).get().getNamespaces();
387    assertEquals(1, namespaces.size());
388    assertTrue(namespaces.contains(ns1));
389
390    admin.removeReplicationPeer(ID_ONE).join();
391  }
392
393  @Test
394  public void testNamespacesAndTableCfsConfigConflict() throws Exception {
395    String ns1 = "ns1";
396    String ns2 = "ns2";
397    final TableName tableName1 = TableName.valueOf(ns1 + ":" + tableName.getNameAsString() + "1");
398    final TableName tableName2 = TableName.valueOf(ns2 + ":" + tableName.getNameAsString() + "2");
399
400    ReplicationPeerConfig rpc = new ReplicationPeerConfig();
401    rpc.setClusterKey(KEY_ONE);
402    admin.addReplicationPeer(ID_ONE, rpc).join();
403    rpc.setReplicateAllUserTables(false);
404    admin.updateReplicationPeerConfig(ID_ONE, rpc).join();
405
406    rpc = admin.getReplicationPeerConfig(ID_ONE).get();
407    Set<String> namespaces = new HashSet<String>();
408    namespaces.add(ns1);
409    rpc.setNamespaces(namespaces);
410    admin.updateReplicationPeerConfig(ID_ONE, rpc).get();
411    rpc = admin.getReplicationPeerConfig(ID_ONE).get();
412    Map<TableName, List<String>> tableCfs = new HashMap<>();
413    tableCfs.put(tableName1, new ArrayList<>());
414    rpc.setTableCFsMap(tableCfs);
415    try {
416      admin.updateReplicationPeerConfig(ID_ONE, rpc).join();
417      fail("Test case should fail, because table " + tableName1 + " conflict with namespace " + ns1);
418    } catch (CompletionException e) {
419      // OK
420    }
421
422    rpc = admin.getReplicationPeerConfig(ID_ONE).get();
423    tableCfs.clear();
424    tableCfs.put(tableName2, new ArrayList<>());
425    rpc.setTableCFsMap(tableCfs);
426    admin.updateReplicationPeerConfig(ID_ONE, rpc).get();
427    rpc = admin.getReplicationPeerConfig(ID_ONE).get();
428    namespaces.clear();
429    namespaces.add(ns2);
430    rpc.setNamespaces(namespaces);
431    try {
432      admin.updateReplicationPeerConfig(ID_ONE, rpc).join();
433      fail("Test case should fail, because namespace " + ns2 + " conflict with table " + tableName2);
434    } catch (CompletionException e) {
435      // OK
436    }
437
438    admin.removeReplicationPeer(ID_ONE).join();
439  }
440
441  @Test
442  public void testPeerBandwidth() throws Exception {
443    ReplicationPeerConfig rpc = new ReplicationPeerConfig();
444    rpc.setClusterKey(KEY_ONE);
445
446    admin.addReplicationPeer(ID_ONE, rpc).join();
447    rpc = admin.getReplicationPeerConfig(ID_ONE).get();
448    assertEquals(0, rpc.getBandwidth());
449
450    rpc.setBandwidth(2097152);
451    admin.updateReplicationPeerConfig(ID_ONE, rpc).join();
452    assertEquals(2097152, admin.getReplicationPeerConfig(ID_ONE).join().getBandwidth());
453
454    admin.removeReplicationPeer(ID_ONE).join();
455  }
456}