Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.security.SecureRandom;
028import java.util.ArrayList;
029import java.util.EnumSet;
030import java.util.HashSet;
031import java.util.Iterator;
032import java.util.LinkedList;
033import java.util.List;
034import java.util.Map;
035import java.util.Set;
036import java.util.TreeMap;
037import org.apache.hadoop.hbase.ClusterMetrics;
038import org.apache.hadoop.hbase.ClusterMetrics.Option;
039import org.apache.hadoop.hbase.HBaseCluster;
040import org.apache.hadoop.hbase.HBaseTestingUtility;
041import org.apache.hadoop.hbase.HColumnDescriptor;
042import org.apache.hadoop.hbase.HTableDescriptor;
043import org.apache.hadoop.hbase.NamespaceDescriptor;
044import org.apache.hadoop.hbase.RegionMetrics;
045import org.apache.hadoop.hbase.ServerMetrics;
046import org.apache.hadoop.hbase.ServerName;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.TableNotFoundException;
049import org.apache.hadoop.hbase.Waiter;
050import org.apache.hadoop.hbase.client.Admin;
051import org.apache.hadoop.hbase.client.ClusterConnection;
052import org.apache.hadoop.hbase.client.RegionInfo;
053import org.apache.hadoop.hbase.constraint.ConstraintException;
054import org.apache.hadoop.hbase.master.HMaster;
055import org.apache.hadoop.hbase.net.Address;
056import org.apache.hadoop.hbase.util.Bytes;
057import org.junit.Assert;
058import org.junit.Before;
059import org.junit.Rule;
060import org.junit.Test;
061import org.junit.rules.TestName;
062import org.slf4j.Logger;
063import org.slf4j.LoggerFactory;
064
065import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
066import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
067import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
068import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
069import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
070import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest;
071
072public abstract class TestRSGroupsBase {
073  protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsBase.class);
074  @Rule
075  public TestName name = new TestName();
076
077  //shared
078  protected final static String groupPrefix = "Group";
079  protected final static String tablePrefix = "Group";
080  protected final static SecureRandom rand = new SecureRandom();
081
082  //shared, cluster type specific
083  protected static HBaseTestingUtility TEST_UTIL;
084  protected static Admin admin;
085  protected static HBaseCluster cluster;
086  protected static RSGroupAdmin rsGroupAdmin;
087  protected static HMaster master;
088
089  public final static long WAIT_TIMEOUT = 60000*5;
090  public final static int NUM_SLAVES_BASE = 4; //number of slaves for the smallest cluster
091  public static int NUM_DEAD_SERVERS = 0;
092
093  // Per test variables
094  TableName tableName;
095  @Before
096  public void setup() {
097    LOG.info(name.getMethodName());
098    tableName = TableName.valueOf(tablePrefix + "_" + name.getMethodName());
099  }
100
101  protected RSGroupInfo addGroup(String groupName, int serverCount)
102      throws IOException, InterruptedException {
103    RSGroupInfo defaultInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
104    assertTrue(defaultInfo != null);
105    assertTrue(defaultInfo.getServers().size() >= serverCount);
106    rsGroupAdmin.addRSGroup(groupName);
107
108    Set<Address> set = new HashSet<>();
109    for(Address server: defaultInfo.getServers()) {
110      if(set.size() == serverCount) {
111        break;
112      }
113      set.add(server);
114    }
115    rsGroupAdmin.moveServers(set, groupName);
116    RSGroupInfo result = rsGroupAdmin.getRSGroupInfo(groupName);
117    assertTrue(result.getServers().size() >= serverCount);
118    return result;
119  }
120
121  void removeGroup(String groupName) throws IOException {
122    RSGroupInfo RSGroupInfo = rsGroupAdmin.getRSGroupInfo(groupName);
123    rsGroupAdmin.moveTables(RSGroupInfo.getTables(), RSGroupInfo.DEFAULT_GROUP);
124    rsGroupAdmin.moveServers(RSGroupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
125    rsGroupAdmin.removeRSGroup(groupName);
126  }
127
128  protected void deleteTableIfNecessary() throws IOException {
129    for (HTableDescriptor desc : TEST_UTIL.getAdmin().listTables(tablePrefix+".*")) {
130      TEST_UTIL.deleteTable(desc.getTableName());
131    }
132  }
133
134  protected void deleteNamespaceIfNecessary() throws IOException {
135    for (NamespaceDescriptor desc : TEST_UTIL.getAdmin().listNamespaceDescriptors()) {
136      if(desc.getName().startsWith(tablePrefix)) {
137        admin.deleteNamespace(desc.getName());
138      }
139    }
140  }
141
142  protected void deleteGroups() throws IOException {
143    RSGroupAdmin groupAdmin =
144        new RSGroupAdminClient(TEST_UTIL.getConnection());
145    for(RSGroupInfo group: groupAdmin.listRSGroups()) {
146      if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
147        groupAdmin.moveTables(group.getTables(), RSGroupInfo.DEFAULT_GROUP);
148        groupAdmin.moveServers(group.getServers(), RSGroupInfo.DEFAULT_GROUP);
149        groupAdmin.removeRSGroup(group.getName());
150      }
151    }
152  }
153
154  public Map<TableName, List<String>> getTableRegionMap() throws IOException {
155    Map<TableName, List<String>> map = Maps.newTreeMap();
156    Map<TableName, Map<ServerName, List<String>>> tableServerRegionMap
157        = getTableServerRegionMap();
158    for(TableName tableName : tableServerRegionMap.keySet()) {
159      if(!map.containsKey(tableName)) {
160        map.put(tableName, new LinkedList<>());
161      }
162      for(List<String> subset: tableServerRegionMap.get(tableName).values()) {
163        map.get(tableName).addAll(subset);
164      }
165    }
166    return map;
167  }
168
169  public Map<TableName, Map<ServerName, List<String>>> getTableServerRegionMap()
170      throws IOException {
171    Map<TableName, Map<ServerName, List<String>>> map = Maps.newTreeMap();
172    ClusterMetrics status = TEST_UTIL.getHBaseClusterInterface().getClusterMetrics();
173    for (Map.Entry<ServerName, ServerMetrics> entry : status.getLiveServerMetrics().entrySet()) {
174      ServerName serverName = entry.getKey();
175      for(RegionMetrics rl : entry.getValue().getRegionMetrics().values()) {
176        TableName tableName = null;
177        try {
178          tableName = RegionInfo.getTable(rl.getRegionName());
179        } catch (IllegalArgumentException e) {
180          LOG.warn("Failed parse a table name from regionname=" +
181            Bytes.toStringBinary(rl.getRegionName()));
182          continue;
183        }
184        if(!map.containsKey(tableName)) {
185          map.put(tableName, new TreeMap<>());
186        }
187        if(!map.get(tableName).containsKey(serverName)) {
188          map.get(tableName).put(serverName, new LinkedList<>());
189        }
190        map.get(tableName).get(serverName).add(rl.getNameAsString());
191      }
192    }
193    return map;
194  }
195
196  @Test
197  public void testBogusArgs() throws Exception {
198    assertNull(rsGroupAdmin.getRSGroupInfoOfTable(TableName.valueOf("nonexistent")));
199    assertNull(rsGroupAdmin.getRSGroupOfServer(Address.fromParts("bogus",123)));
200    assertNull(rsGroupAdmin.getRSGroupInfo("bogus"));
201
202    try {
203      rsGroupAdmin.removeRSGroup("bogus");
204      fail("Expected removing bogus group to fail");
205    } catch(ConstraintException ex) {
206      //expected
207    }
208
209    try {
210      rsGroupAdmin.moveTables(Sets.newHashSet(TableName.valueOf("bogustable")), "bogus");
211      fail("Expected move with bogus group to fail");
212    } catch(ConstraintException|TableNotFoundException ex) {
213      //expected
214    }
215
216    try {
217      rsGroupAdmin.moveServers(Sets.newHashSet(Address.fromParts("bogus",123)), "bogus");
218      fail("Expected move with bogus group to fail");
219    } catch(ConstraintException ex) {
220      //expected
221    }
222
223    try {
224      admin.setBalancerRunning(true,true);
225      rsGroupAdmin.balanceRSGroup("bogus");
226      admin.setBalancerRunning(false,true);
227      fail("Expected move with bogus group to fail");
228    } catch(ConstraintException ex) {
229      //expected
230    }
231  }
232
233  @Test
234  public void testCreateMultiRegion() throws IOException {
235    byte[] end = {1,3,5,7,9};
236    byte[] start = {0,2,4,6,8};
237    byte[][] f = {Bytes.toBytes("f")};
238    TEST_UTIL.createTable(tableName, f,1,start,end,10);
239  }
240
241  @Test
242  public void testCreateAndDrop() throws Exception {
243    TEST_UTIL.createTable(tableName, Bytes.toBytes("cf"));
244    //wait for created table to be assigned
245    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
246      @Override
247      public boolean evaluate() throws Exception {
248        return getTableRegionMap().get(tableName) != null;
249      }
250    });
251    TEST_UTIL.deleteTable(tableName);
252  }
253
254
255  @Test
256  public void testSimpleRegionServerMove() throws IOException,
257      InterruptedException {
258    int initNumGroups = rsGroupAdmin.listRSGroups().size();
259    RSGroupInfo appInfo = addGroup(getGroupName(name.getMethodName()), 1);
260    RSGroupInfo adminInfo = addGroup(getGroupName(name.getMethodName()), 1);
261    RSGroupInfo dInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
262    Assert.assertEquals(initNumGroups + 2, rsGroupAdmin.listRSGroups().size());
263    assertEquals(1, adminInfo.getServers().size());
264    assertEquals(1, appInfo.getServers().size());
265    assertEquals(getNumServers() - 2, dInfo.getServers().size());
266    rsGroupAdmin.moveServers(appInfo.getServers(),
267        RSGroupInfo.DEFAULT_GROUP);
268    rsGroupAdmin.removeRSGroup(appInfo.getName());
269    rsGroupAdmin.moveServers(adminInfo.getServers(),
270        RSGroupInfo.DEFAULT_GROUP);
271    rsGroupAdmin.removeRSGroup(adminInfo.getName());
272    Assert.assertEquals(rsGroupAdmin.listRSGroups().size(), initNumGroups);
273  }
274
275  // return the real number of region servers, excluding the master embedded region server in 2.0+
276  public int getNumServers() throws IOException {
277    ClusterMetrics status =
278        admin.getClusterMetrics(EnumSet.of(Option.MASTER, Option.LIVE_SERVERS));
279    ServerName masterName = status.getMasterName();
280    int count = 0;
281    for (ServerName sn : status.getLiveServerMetrics().keySet()) {
282      if (!sn.equals(masterName)) {
283        count++;
284      }
285    }
286    return count;
287  }
288
289  @Test
290  public void testMoveServers() throws Exception {
291    //create groups and assign servers
292    addGroup("bar", 3);
293    rsGroupAdmin.addRSGroup("foo");
294
295    RSGroupInfo barGroup = rsGroupAdmin.getRSGroupInfo("bar");
296    RSGroupInfo fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
297    assertEquals(3, barGroup.getServers().size());
298    assertEquals(0, fooGroup.getServers().size());
299
300    //test fail bogus server move
301    try {
302      rsGroupAdmin.moveServers(Sets.newHashSet(Address.fromString("foo:9999")),"foo");
303      fail("Bogus servers shouldn't have been successfully moved.");
304    } catch(IOException ex) {
305      String exp = "Source RSGroup for server foo:9999 does not exist.";
306      String msg = "Expected '"+exp+"' in exception message: ";
307      assertTrue(msg+" "+ex.getMessage(), ex.getMessage().contains(exp));
308    }
309
310    //test success case
311    LOG.info("moving servers "+barGroup.getServers()+" to group foo");
312    rsGroupAdmin.moveServers(barGroup.getServers(), fooGroup.getName());
313
314    barGroup = rsGroupAdmin.getRSGroupInfo("bar");
315    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
316    assertEquals(0,barGroup.getServers().size());
317    assertEquals(3,fooGroup.getServers().size());
318
319    LOG.info("moving servers "+fooGroup.getServers()+" to group default");
320    rsGroupAdmin.moveServers(fooGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
321
322    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
323      @Override
324      public boolean evaluate() throws Exception {
325        return getNumServers() ==
326        rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size();
327      }
328    });
329
330    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
331    assertEquals(0,fooGroup.getServers().size());
332
333    //test group removal
334    LOG.info("Remove group "+barGroup.getName());
335    rsGroupAdmin.removeRSGroup(barGroup.getName());
336    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(barGroup.getName()));
337    LOG.info("Remove group "+fooGroup.getName());
338    rsGroupAdmin.removeRSGroup(fooGroup.getName());
339    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName()));
340  }
341
342  @Test
343  public void testMoveServersFromDefaultGroup() throws Exception {
344    //create groups and assign servers
345    rsGroupAdmin.addRSGroup("foo");
346
347    RSGroupInfo fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
348    assertEquals(0, fooGroup.getServers().size());
349    RSGroupInfo defaultGroup = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
350
351    //test remove all servers from default
352    try {
353      rsGroupAdmin.moveServers(defaultGroup.getServers(), fooGroup.getName());
354      fail(RSGroupAdminServer.KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE);
355    } catch (ConstraintException ex) {
356      assertTrue(ex.getMessage().contains(RSGroupAdminServer
357              .KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE));
358    }
359
360    //test success case, remove one server from default ,keep at least one server
361    if (defaultGroup.getServers().size() > 1) {
362      Address serverInDefaultGroup = defaultGroup.getServers().iterator().next();
363      LOG.info("moving server " + serverInDefaultGroup + " from group default to group " +
364              fooGroup.getName());
365      rsGroupAdmin.moveServers(Sets.newHashSet(serverInDefaultGroup), fooGroup.getName());
366    }
367
368    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
369    LOG.info("moving servers " + fooGroup.getServers() + " to group default");
370    rsGroupAdmin.moveServers(fooGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
371
372    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
373      @Override
374      public boolean evaluate() throws Exception {
375        return getNumServers() ==
376                rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size();
377      }
378    });
379
380    fooGroup = rsGroupAdmin.getRSGroupInfo("foo");
381    assertEquals(0, fooGroup.getServers().size());
382
383    //test group removal
384    LOG.info("Remove group " + fooGroup.getName());
385    rsGroupAdmin.removeRSGroup(fooGroup.getName());
386    Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName()));
387  }
388
389  @Test
390  public void testTableMoveTruncateAndDrop() throws Exception {
391    final byte[] familyNameBytes = Bytes.toBytes("f");
392    String newGroupName = getGroupName(name.getMethodName());
393    final RSGroupInfo newGroup = addGroup(newGroupName, 2);
394
395    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5);
396    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
397      @Override
398      public boolean evaluate() throws Exception {
399        List<String> regions = getTableRegionMap().get(tableName);
400        if (regions == null) {
401          return false;
402        }
403
404        return getTableRegionMap().get(tableName).size() >= 5;
405      }
406    });
407
408    RSGroupInfo tableGrp = rsGroupAdmin.getRSGroupInfoOfTable(tableName);
409    assertTrue(tableGrp.getName().equals(RSGroupInfo.DEFAULT_GROUP));
410
411    //change table's group
412    LOG.info("Moving table "+tableName+" to "+newGroup.getName());
413    rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName());
414
415    //verify group change
416    Assert.assertEquals(newGroup.getName(),
417        rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName());
418
419    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
420      @Override
421      public boolean evaluate() throws Exception {
422        Map<ServerName, List<String>> serverMap = getTableServerRegionMap().get(tableName);
423        int count = 0;
424        if (serverMap != null) {
425          for (ServerName rs : serverMap.keySet()) {
426            if (newGroup.containsServer(rs.getAddress())) {
427              count += serverMap.get(rs).size();
428            }
429          }
430        }
431        return count == 5;
432      }
433    });
434
435    //test truncate
436    admin.disableTable(tableName);
437    admin.truncateTable(tableName, true);
438    Assert.assertEquals(1, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size());
439    Assert.assertEquals(tableName, rsGroupAdmin.getRSGroupInfo(
440        newGroup.getName()).getTables().first());
441
442    //verify removed table is removed from group
443    TEST_UTIL.deleteTable(tableName);
444    Assert.assertEquals(0, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size());
445  }
446
447  @Test
448  public void testGroupBalance() throws Exception {
449    LOG.info(name.getMethodName());
450    String newGroupName = getGroupName(name.getMethodName());
451    final RSGroupInfo newGroup = addGroup(newGroupName, 3);
452
453    final TableName tableName = TableName.valueOf(tablePrefix+"_ns", name.getMethodName());
454    admin.createNamespace(
455        NamespaceDescriptor.create(tableName.getNamespaceAsString())
456            .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, newGroupName).build());
457    final byte[] familyNameBytes = Bytes.toBytes("f");
458    final HTableDescriptor desc = new HTableDescriptor(tableName);
459    desc.addFamily(new HColumnDescriptor("f"));
460    byte [] startKey = Bytes.toBytes("aaaaa");
461    byte [] endKey = Bytes.toBytes("zzzzz");
462    admin.createTable(desc, startKey, endKey, 6);
463    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
464      @Override
465      public boolean evaluate() throws Exception {
466        List<String> regions = getTableRegionMap().get(tableName);
467        if (regions == null) {
468          return false;
469        }
470        return regions.size() >= 6;
471      }
472    });
473
474    //make assignment uneven, move all regions to one server
475    Map<ServerName,List<String>> assignMap =
476        getTableServerRegionMap().get(tableName);
477    final ServerName first = assignMap.entrySet().iterator().next().getKey();
478    for(RegionInfo region: admin.getTableRegions(tableName)) {
479      if(!assignMap.get(first).contains(region.getRegionNameAsString())) {
480        admin.move(region.getEncodedNameAsBytes(), Bytes.toBytes(first.getServerName()));
481      }
482    }
483    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
484      @Override
485      public boolean evaluate() throws Exception {
486        Map<ServerName, List<String>> map = getTableServerRegionMap().get(tableName);
487        if (map == null) {
488          return true;
489        }
490        List<String> regions = map.get(first);
491        if (regions == null) {
492          return true;
493        }
494        return regions.size() >= 6;
495      }
496    });
497
498    //balance the other group and make sure it doesn't affect the new group
499    admin.setBalancerRunning(true,true);
500    rsGroupAdmin.balanceRSGroup(RSGroupInfo.DEFAULT_GROUP);
501    assertEquals(6, getTableServerRegionMap().get(tableName).get(first).size());
502
503    //disable balance, balancer will not be run and return false
504    admin.setBalancerRunning(false,true);
505    assertFalse(rsGroupAdmin.balanceRSGroup(newGroupName));
506    assertEquals(6, getTableServerRegionMap().get(tableName).get(first).size());
507
508    //enable balance
509    admin.setBalancerRunning(true,true);
510    rsGroupAdmin.balanceRSGroup(newGroupName);
511    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
512      @Override
513      public boolean evaluate() throws Exception {
514        for (List<String> regions : getTableServerRegionMap().get(tableName).values()) {
515          if (2 != regions.size()) {
516            return false;
517          }
518        }
519        return true;
520      }
521    });
522    admin.setBalancerRunning(false,true);
523  }
524
525  @Test
526  public void testRegionMove() throws Exception {
527    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
528    final byte[] familyNameBytes = Bytes.toBytes("f");
529    // All the regions created below will be assigned to the default group.
530    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 6);
531    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
532      @Override
533      public boolean evaluate() throws Exception {
534        List<String> regions = getTableRegionMap().get(tableName);
535        if (regions == null) {
536          return false;
537        }
538        return getTableRegionMap().get(tableName).size() >= 6;
539      }
540    });
541
542    //get target region to move
543    Map<ServerName,List<String>> assignMap =
544        getTableServerRegionMap().get(tableName);
545    String targetRegion = null;
546    for(ServerName server : assignMap.keySet()) {
547      targetRegion = assignMap.get(server).size() > 0 ? assignMap.get(server).get(0) : null;
548      if(targetRegion != null) {
549        break;
550      }
551    }
552    //get server which is not a member of new group
553    ServerName targetServer = null;
554    for (ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
555                                  .getLiveServerMetrics().keySet()) {
556      if (!newGroup.containsServer(server.getAddress())) {
557        targetServer = server;
558        break;
559      }
560    }
561
562    final AdminProtos.AdminService.BlockingInterface targetRS =
563      ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
564
565    //move target server to group
566    rsGroupAdmin.moveServers(Sets.newHashSet(targetServer.getAddress()),
567        newGroup.getName());
568    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
569      @Override
570      public boolean evaluate() throws Exception {
571        return ProtobufUtil.getOnlineRegions(targetRS).size() <= 0;
572      }
573    });
574
575    // Lets move this region to the new group.
576    TEST_UTIL.getAdmin().move(Bytes.toBytes(RegionInfo.encodeRegionName(
577        Bytes.toBytes(targetRegion))), Bytes.toBytes(targetServer.getServerName()));
578    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
579      @Override
580      public boolean evaluate() throws Exception {
581        return
582            getTableRegionMap().get(tableName) != null &&
583                getTableRegionMap().get(tableName).size() == 6 &&
584                admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION))
585                     .getRegionStatesInTransition().size() < 1;
586      }
587    });
588
589    //verify that targetServer didn't open it
590    for (RegionInfo region: ProtobufUtil.getOnlineRegions(targetRS)) {
591      if (targetRegion.equals(region.getRegionNameAsString())) {
592        fail("Target server opened region");
593      }
594    }
595  }
596
597  @Test
598  public void testFailRemoveGroup() throws IOException, InterruptedException {
599    int initNumGroups = rsGroupAdmin.listRSGroups().size();
600    addGroup("bar", 3);
601    TEST_UTIL.createTable(tableName, Bytes.toBytes("f"));
602    rsGroupAdmin.moveTables(Sets.newHashSet(tableName), "bar");
603    RSGroupInfo barGroup = rsGroupAdmin.getRSGroupInfo("bar");
604    //group is not empty therefore it should fail
605    try {
606      rsGroupAdmin.removeRSGroup(barGroup.getName());
607      fail("Expected remove group to fail");
608    } catch(IOException e) {
609    }
610    //group cannot lose all it's servers therefore it should fail
611    try {
612      rsGroupAdmin.moveServers(barGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
613      fail("Expected move servers to fail");
614    } catch(IOException e) {
615    }
616
617    rsGroupAdmin.moveTables(barGroup.getTables(), RSGroupInfo.DEFAULT_GROUP);
618    try {
619      rsGroupAdmin.removeRSGroup(barGroup.getName());
620      fail("Expected move servers to fail");
621    } catch(IOException e) {
622    }
623
624    rsGroupAdmin.moveServers(barGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
625    rsGroupAdmin.removeRSGroup(barGroup.getName());
626
627    Assert.assertEquals(initNumGroups, rsGroupAdmin.listRSGroups().size());
628  }
629
630  @Test
631  public void testKillRS() throws Exception {
632    RSGroupInfo appInfo = addGroup("appInfo", 1);
633
634    final TableName tableName = TableName.valueOf(tablePrefix+"_ns", name.getMethodName());
635    admin.createNamespace(
636        NamespaceDescriptor.create(tableName.getNamespaceAsString())
637            .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build());
638    final HTableDescriptor desc = new HTableDescriptor(tableName);
639    desc.addFamily(new HColumnDescriptor("f"));
640    admin.createTable(desc);
641    //wait for created table to be assigned
642    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
643      @Override
644      public boolean evaluate() throws Exception {
645        return getTableRegionMap().get(desc.getTableName()) != null;
646      }
647    });
648
649    ServerName targetServer = ServerName.parseServerName(
650        appInfo.getServers().iterator().next().toString());
651    AdminProtos.AdminService.BlockingInterface targetRS =
652      ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
653    RegionInfo targetRegion = ProtobufUtil.getOnlineRegions(targetRS).get(0);
654    Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(targetRS).size());
655
656    try {
657      //stopping may cause an exception
658      //due to the connection loss
659      targetRS.stopServer(null,
660          AdminProtos.StopServerRequest.newBuilder().setReason("Die").build());
661    } catch(Exception e) {
662    }
663    assertFalse(cluster.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer));
664
665    //wait for created table to be assigned
666    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
667      @Override
668      public boolean evaluate() throws Exception {
669        return cluster.getClusterMetrics().getRegionStatesInTransition().isEmpty();
670      }
671    });
672    Set<Address> newServers = Sets.newHashSet();
673    newServers.add(
674        rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().iterator().next());
675    rsGroupAdmin.moveServers(newServers, appInfo.getName());
676
677    //Make sure all the table's regions get reassigned
678    //disabling the table guarantees no conflicting assign/unassign (ie SSH) happens
679    admin.disableTable(tableName);
680    admin.enableTable(tableName);
681
682    //wait for region to be assigned
683    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
684      @Override
685      public boolean evaluate() throws Exception {
686        return cluster.getClusterMetrics().getRegionStatesInTransition().isEmpty();
687      }
688    });
689
690    targetServer = ServerName.parseServerName(
691        newServers.iterator().next().toString());
692    targetRS =
693      ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
694    Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(targetRS).size());
695    Assert.assertEquals(tableName,
696        ProtobufUtil.getOnlineRegions(targetRS).get(0).getTable());
697  }
698
699  @Test
700  public void testValidGroupNames() throws IOException {
701    String[] badNames = {"foo*","foo@","-"};
702    String[] goodNames = {"foo_123"};
703
704    for(String entry: badNames) {
705      try {
706        rsGroupAdmin.addRSGroup(entry);
707        fail("Expected a constraint exception for: "+entry);
708      } catch(ConstraintException ex) {
709        //expected
710      }
711    }
712
713    for(String entry: goodNames) {
714      rsGroupAdmin.addRSGroup(entry);
715    }
716  }
717
718  private String getGroupName(String baseName) {
719    return groupPrefix+"_"+baseName+"_"+rand.nextInt(Integer.MAX_VALUE);
720  }
721
722  @Test
723  public void testMultiTableMove() throws Exception {
724    final TableName tableNameA = TableName.valueOf(tablePrefix + name.getMethodName() + "A");
725    final TableName tableNameB = TableName.valueOf(tablePrefix + name.getMethodName() + "B");
726    final byte[] familyNameBytes = Bytes.toBytes("f");
727    String newGroupName = getGroupName(name.getMethodName());
728    final RSGroupInfo newGroup = addGroup(newGroupName, 1);
729
730    TEST_UTIL.createTable(tableNameA, familyNameBytes);
731    TEST_UTIL.createTable(tableNameB, familyNameBytes);
732    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
733      @Override
734      public boolean evaluate() throws Exception {
735        List<String> regionsA = getTableRegionMap().get(tableNameA);
736        if (regionsA == null) {
737          return false;
738        }
739
740        List<String> regionsB = getTableRegionMap().get(tableNameB);
741        if (regionsB == null) {
742          return false;
743        }
744
745        return getTableRegionMap().get(tableNameA).size() >= 1
746                && getTableRegionMap().get(tableNameB).size() >= 1;
747      }
748    });
749
750    RSGroupInfo tableGrpA = rsGroupAdmin.getRSGroupInfoOfTable(tableNameA);
751    assertTrue(tableGrpA.getName().equals(RSGroupInfo.DEFAULT_GROUP));
752
753    RSGroupInfo tableGrpB = rsGroupAdmin.getRSGroupInfoOfTable(tableNameB);
754    assertTrue(tableGrpB.getName().equals(RSGroupInfo.DEFAULT_GROUP));
755    //change table's group
756    LOG.info("Moving table [" + tableNameA + "," + tableNameB + "] to " + newGroup.getName());
757    rsGroupAdmin.moveTables(Sets.newHashSet(tableNameA, tableNameB), newGroup.getName());
758
759    //verify group change
760    Assert.assertEquals(newGroup.getName(),
761            rsGroupAdmin.getRSGroupInfoOfTable(tableNameA).getName());
762
763    Assert.assertEquals(newGroup.getName(),
764            rsGroupAdmin.getRSGroupInfoOfTable(tableNameB).getName());
765
766    //verify tables' not exist in old group
767    Set<TableName> DefaultTables = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
768        .getTables();
769    assertFalse(DefaultTables.contains(tableNameA));
770    assertFalse(DefaultTables.contains(tableNameB));
771
772    //verify tables' exist in new group
773    Set<TableName> newGroupTables = rsGroupAdmin.getRSGroupInfo(newGroupName).getTables();
774    assertTrue(newGroupTables.contains(tableNameA));
775    assertTrue(newGroupTables.contains(tableNameB));
776  }
777
778  @Test
779  public void testDisabledTableMove() throws Exception {
780    final byte[] familyNameBytes = Bytes.toBytes("f");
781    String newGroupName = getGroupName(name.getMethodName());
782    final RSGroupInfo newGroup = addGroup(newGroupName, 2);
783
784    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5);
785    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
786      @Override
787      public boolean evaluate() throws Exception {
788        List<String> regions = getTableRegionMap().get(tableName);
789        if (regions == null) {
790          return false;
791        }
792        return getTableRegionMap().get(tableName).size() >= 5;
793      }
794    });
795
796    RSGroupInfo tableGrp = rsGroupAdmin.getRSGroupInfoOfTable(tableName);
797    assertTrue(tableGrp.getName().equals(RSGroupInfo.DEFAULT_GROUP));
798
799    //test disable table
800    admin.disableTable(tableName);
801
802    //change table's group
803    LOG.info("Moving table "+ tableName + " to " + newGroup.getName());
804    rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName());
805
806    //verify group change
807    Assert.assertEquals(newGroup.getName(),
808        rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName());
809  }
810
811  @Test
812  public void testNonExistentTableMove() throws Exception {
813    TableName tableName = TableName.valueOf(tablePrefix + name.getMethodName());
814
815    RSGroupInfo tableGrp = rsGroupAdmin.getRSGroupInfoOfTable(tableName);
816    assertNull(tableGrp);
817
818    //test if table exists already.
819    boolean exist = admin.tableExists(tableName);
820    assertFalse(exist);
821
822    LOG.info("Moving table "+ tableName + " to " + RSGroupInfo.DEFAULT_GROUP);
823    try {
824      rsGroupAdmin.moveTables(Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
825      fail("Table " + tableName + " shouldn't have been successfully moved.");
826    } catch(IOException ex) {
827      assertTrue(ex instanceof TableNotFoundException);
828    }
829
830    try {
831      rsGroupAdmin.moveServersAndTables(
832          Sets.newHashSet(Address.fromParts("bogus",123)),
833          Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
834      fail("Table " + tableName + " shouldn't have been successfully moved.");
835    } catch(IOException ex) {
836      assertTrue(ex instanceof TableNotFoundException);
837    }
838    //verify group change
839    assertNull(rsGroupAdmin.getRSGroupInfoOfTable(tableName));
840  }
841
842  @Test
843  public void testMoveServersAndTables() throws Exception {
844    LOG.info("testMoveServersAndTables");
845    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
846    //create table
847    final byte[] familyNameBytes = Bytes.toBytes("f");
848    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5);
849    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
850      @Override
851      public boolean evaluate() throws Exception {
852        List<String> regions = getTableRegionMap().get(tableName);
853        if (regions == null) {
854          return false;
855        }
856        return getTableRegionMap().get(tableName).size() >= 5;
857      }
858    });
859
860    //get server which is not a member of new group
861    ServerName targetServer = null;
862    for(ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
863      .getLiveServerMetrics().keySet()) {
864      if(!newGroup.containsServer(server.getAddress()) &&
865           !rsGroupAdmin.getRSGroupInfo("master").containsServer(server.getAddress())) {
866        targetServer = server;
867        break;
868      }
869    }
870
871    LOG.debug("Print group info : " + rsGroupAdmin.listRSGroups());
872    int oldDefaultGroupServerSize =
873            rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size();
874    int oldDefaultGroupTableSize =
875            rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size();
876
877    //test fail bogus server move
878    try {
879      rsGroupAdmin.moveServersAndTables(Sets.newHashSet(Address.fromString("foo:9999")),
880              Sets.newHashSet(tableName), newGroup.getName());
881      fail("Bogus servers shouldn't have been successfully moved.");
882    } catch(IOException ex) {
883      String exp = "Source RSGroup for server foo:9999 does not exist.";
884      String msg = "Expected '" + exp + "' in exception message: ";
885      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
886    }
887
888    //test fail server move
889    try {
890      rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()),
891              Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
892      fail("servers shouldn't have been successfully moved.");
893    } catch(IOException ex) {
894      String exp = "Target RSGroup " + RSGroupInfo.DEFAULT_GROUP +
895              " is same as source " + RSGroupInfo.DEFAULT_GROUP + " RSGroup.";
896      String msg = "Expected '" + exp + "' in exception message: ";
897      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
898    }
899
900    //verify default group info
901    Assert.assertEquals(oldDefaultGroupServerSize,
902            rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size());
903    Assert.assertEquals(oldDefaultGroupTableSize,
904            rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size());
905
906    //verify new group info
907    Assert.assertEquals(1,
908            rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers().size());
909    Assert.assertEquals(0,
910            rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size());
911
912    //get all region to move targetServer
913    List<String> regionList = getTableRegionMap().get(tableName);
914    for(String region : regionList) {
915      // Lets move this region to the targetServer
916      TEST_UTIL.getAdmin().move(Bytes.toBytes(RegionInfo.encodeRegionName(Bytes.toBytes(region))),
917              Bytes.toBytes(targetServer.getServerName()));
918    }
919
920    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
921      @Override
922      public boolean evaluate() throws Exception {
923        return getTableRegionMap().get(tableName) != null &&
924                getTableRegionMap().get(tableName).size() == 5 &&
925                getTableServerRegionMap().get(tableName).size() == 1 &&
926                admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION))
927                     .getRegionStatesInTransition().size() < 1;
928      }
929    });
930
931    //verify that all region move to targetServer
932    Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size());
933
934    //move targetServer and table to newGroup
935    LOG.info("moving server and table to newGroup");
936    rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()),
937            Sets.newHashSet(tableName), newGroup.getName());
938
939    //verify group change
940    Assert.assertEquals(newGroup.getName(),
941            rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName());
942
943    //verify servers' not exist in old group
944    Set<Address> defaultServers = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
945        .getServers();
946    assertFalse(defaultServers.contains(targetServer.getAddress()));
947
948    //verify servers' exist in new group
949    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
950    assertTrue(newGroupServers.contains(targetServer.getAddress()));
951
952    //verify tables' not exist in old group
953    Set<TableName> defaultTables = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
954        .getTables();
955    assertFalse(defaultTables.contains(tableName));
956
957    //verify tables' exist in new group
958    Set<TableName> newGroupTables = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables();
959    assertTrue(newGroupTables.contains(tableName));
960
961    //verify that all region still assgin on targetServer
962    Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size());
963  }
964
965  @Test
966  public void testClearDeadServers() throws Exception {
967    LOG.info("testClearDeadServers");
968    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 3);
969    NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size();
970
971    ServerName targetServer = ServerName.parseServerName(
972        newGroup.getServers().iterator().next().toString());
973    AdminProtos.AdminService.BlockingInterface targetRS =
974        ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
975    try {
976      targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null,
977          GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName());
978      //stopping may cause an exception
979      //due to the connection loss
980      targetRS.stopServer(null,
981          AdminProtos.StopServerRequest.newBuilder().setReason("Die").build());
982      NUM_DEAD_SERVERS ++;
983    } catch(Exception e) {
984    }
985    //wait for stopped regionserver to dead server list
986    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
987      @Override
988      public boolean evaluate() throws Exception {
989        return !master.getServerManager().areDeadServersInProgress()
990            && cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS;
991      }
992    });
993    assertFalse(cluster.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer));
994    assertTrue(cluster.getClusterMetrics().getDeadServerNames().contains(targetServer));
995    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
996
997    //clear dead servers list
998    List<ServerName> notClearedServers = admin.clearDeadServers(Lists.newArrayList(targetServer));
999    assertEquals(0, notClearedServers.size());
1000
1001    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
1002    assertFalse(newGroupServers.contains(targetServer.getAddress()));
1003    assertEquals(2, newGroupServers.size());
1004  }
1005
1006  @Test
1007  public void testRemoveServers() throws Exception {
1008    LOG.info("testRemoveServers");
1009    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 3);
1010    Iterator<Address> iterator = newGroup.getServers().iterator();
1011    ServerName targetServer = ServerName.parseServerName(iterator.next().toString());
1012
1013    // remove online servers
1014    try {
1015      rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
1016      fail("Online servers shouldn't have been successfully removed.");
1017    } catch(IOException ex) {
1018      String exp = "Server " + targetServer.getAddress()
1019          + " is an online server, not allowed to remove.";
1020      String msg = "Expected '" + exp + "' in exception message: ";
1021      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
1022    }
1023    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
1024
1025    // remove dead servers
1026    NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size();
1027    AdminProtos.AdminService.BlockingInterface targetRS =
1028        ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
1029    try {
1030      targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null,
1031          GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName());
1032      //stopping may cause an exception
1033      //due to the connection loss
1034      LOG.info("stopping server " + targetServer.getHostAndPort());
1035      targetRS.stopServer(null,
1036          AdminProtos.StopServerRequest.newBuilder().setReason("Die").build());
1037      NUM_DEAD_SERVERS ++;
1038    } catch(Exception e) {
1039    }
1040
1041    //wait for stopped regionserver to dead server list
1042    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
1043      @Override
1044      public boolean evaluate() throws Exception {
1045        return !master.getServerManager().areDeadServersInProgress()
1046            && cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS;
1047      }
1048    });
1049
1050    try {
1051      rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
1052      fail("Dead servers shouldn't have been successfully removed.");
1053    } catch(IOException ex) {
1054      String exp = "Server " + targetServer.getAddress() + " is on the dead servers list,"
1055          + " Maybe it will come back again, not allowed to remove.";
1056      String msg = "Expected '" + exp + "' in exception message: ";
1057      assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
1058    }
1059    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
1060
1061    // remove decommissioned servers
1062    List<ServerName> serversToDecommission = new ArrayList<>();
1063    targetServer = ServerName.parseServerName(iterator.next().toString());
1064    targetRS = ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
1065    targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null,
1066          GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName());
1067    assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer));
1068    serversToDecommission.add(targetServer);
1069
1070    admin.decommissionRegionServers(serversToDecommission, true);
1071    assertEquals(1, admin.listDecommissionedRegionServers().size());
1072
1073    assertTrue(newGroup.getServers().contains(targetServer.getAddress()));
1074    rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
1075    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
1076    assertFalse(newGroupServers.contains(targetServer.getAddress()));
1077    assertEquals(2, newGroupServers.size());
1078  }
1079
1080  @Test
1081  public void testCreateWhenRsgroupNoOnlineServers() throws Exception {
1082    LOG.info("testCreateWhenRsgroupNoOnlineServers");
1083
1084    // set rsgroup has no online servers and test create table
1085    final RSGroupInfo appInfo = addGroup("appInfo", 1);
1086    Iterator<Address> iterator = appInfo.getServers().iterator();
1087    List<ServerName> serversToDecommission = new ArrayList<>();
1088    ServerName targetServer = ServerName.parseServerName(iterator.next().toString());
1089    AdminProtos.AdminService.BlockingInterface targetRS =
1090        ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
1091    targetServer = ProtobufUtil.toServerName(
1092        targetRS.getServerInfo(null, GetServerInfoRequest.newBuilder().build()).getServerInfo()
1093            .getServerName());
1094    assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer));
1095    serversToDecommission.add(targetServer);
1096    admin.decommissionRegionServers(serversToDecommission, true);
1097    assertEquals(1, admin.listDecommissionedRegionServers().size());
1098
1099    final TableName tableName = TableName.valueOf(tablePrefix + "_ns", name.getMethodName());
1100    admin.createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString())
1101        .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build());
1102    final HTableDescriptor desc = new HTableDescriptor(tableName);
1103    desc.addFamily(new HColumnDescriptor("f"));
1104    try {
1105      admin.createTable(desc);
1106      fail("Shouldn't create table successfully!");
1107    } catch (Exception e) {
1108      LOG.debug("create table error", e);
1109    }
1110
1111    // recommission and test create table
1112    admin.recommissionRegionServer(targetServer, null);
1113    assertEquals(0, admin.listDecommissionedRegionServers().size());
1114    admin.createTable(desc);
1115    // wait for created table to be assigned
1116    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
1117      @Override public boolean evaluate() throws Exception {
1118        return getTableRegionMap().get(desc.getTableName()) != null;
1119      }
1120    });
1121  }
1122}