Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.replication;
019
020import static org.junit.Assert.assertNull;
021import static org.junit.Assert.assertTrue;
022import static org.mockito.Mockito.mock;
023import static org.mockito.Mockito.when;
024
025import java.util.ArrayList;
026import java.util.HashMap;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Map;
030import java.util.Set;
031import java.util.TreeMap;
032import org.apache.hadoop.hbase.Cell;
033import org.apache.hadoop.hbase.CellComparatorImpl;
034import org.apache.hadoop.hbase.HBaseClassTestRule;
035import org.apache.hadoop.hbase.HConstants;
036import org.apache.hadoop.hbase.KeyValue;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.RegionInfoBuilder;
039import org.apache.hadoop.hbase.testclassification.ReplicationTests;
040import org.apache.hadoop.hbase.testclassification.SmallTests;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.apache.hadoop.hbase.wal.WAL.Entry;
043import org.apache.hadoop.hbase.wal.WALEdit;
044import org.apache.hadoop.hbase.wal.WALKeyImpl;
045import org.junit.Assert;
046import org.junit.ClassRule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049
050import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
051
052@Category({ ReplicationTests.class, SmallTests.class })
053public class TestReplicationWALEntryFilters {
054
055  @ClassRule
056  public static final HBaseClassTestRule CLASS_RULE =
057      HBaseClassTestRule.forClass(TestReplicationWALEntryFilters.class);
058
059  static byte[] a = new byte[] {'a'};
060  static byte[] b = new byte[] {'b'};
061  static byte[] c = new byte[] {'c'};
062  static byte[] d = new byte[] {'d'};
063
064  @Test
065  public void testSystemTableWALEntryFilter() {
066    SystemTableWALEntryFilter filter = new SystemTableWALEntryFilter();
067
068    // meta
069    WALKeyImpl key1 =
070      new WALKeyImpl(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
071        TableName.META_TABLE_NAME, System.currentTimeMillis());
072    Entry metaEntry = new Entry(key1, null);
073
074    assertNull(filter.filter(metaEntry));
075
076    // ns table
077    WALKeyImpl key2 =
078        new WALKeyImpl(new byte[0], TableName.NAMESPACE_TABLE_NAME, System.currentTimeMillis());
079    Entry nsEntry = new Entry(key2, null);
080    assertNull(filter.filter(nsEntry));
081
082    // user table
083
084    WALKeyImpl key3 = new WALKeyImpl(new byte[0], TableName.valueOf("foo"),
085        System.currentTimeMillis());
086    Entry userEntry = new Entry(key3, null);
087
088    assertEquals(userEntry, filter.filter(userEntry));
089  }
090
091  @Test
092  public void testScopeWALEntryFilter() {
093    WALEntryFilter filter = new ChainWALEntryFilter(new ScopeWALEntryFilter());
094
095    Entry userEntry = createEntry(null, a, b);
096    Entry userEntryA = createEntry(null, a);
097    Entry userEntryB = createEntry(null, b);
098    Entry userEntryEmpty = createEntry(null);
099
100    // no scopes
101    // now we will not filter out entries without a replication scope since serial replication still
102    // need the sequence id, but the cells will all be filtered out.
103    assertTrue(filter.filter(userEntry).getEdit().isEmpty());
104
105    // empty scopes
106    // ditto
107    TreeMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
108    userEntry = createEntry(scopes, a, b);
109    assertTrue(filter.filter(userEntry).getEdit().isEmpty());
110
111    // different scope
112    scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
113    scopes.put(c, HConstants.REPLICATION_SCOPE_GLOBAL);
114    userEntry = createEntry(scopes, a, b);
115    // all kvs should be filtered
116    assertEquals(userEntryEmpty, filter.filter(userEntry));
117
118    // local scope
119    scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
120    scopes.put(a, HConstants.REPLICATION_SCOPE_LOCAL);
121    userEntry = createEntry(scopes, a, b);
122    assertEquals(userEntryEmpty, filter.filter(userEntry));
123    scopes.put(b, HConstants.REPLICATION_SCOPE_LOCAL);
124    assertEquals(userEntryEmpty, filter.filter(userEntry));
125
126    // only scope a
127    scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
128    scopes.put(a, HConstants.REPLICATION_SCOPE_GLOBAL);
129    userEntry = createEntry(scopes, a, b);
130    assertEquals(userEntryA, filter.filter(userEntry));
131    scopes.put(b, HConstants.REPLICATION_SCOPE_LOCAL);
132    assertEquals(userEntryA, filter.filter(userEntry));
133
134    // only scope b
135    scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
136    scopes.put(b, HConstants.REPLICATION_SCOPE_GLOBAL);
137    userEntry = createEntry(scopes, a, b);
138    assertEquals(userEntryB, filter.filter(userEntry));
139    scopes.put(a, HConstants.REPLICATION_SCOPE_LOCAL);
140    assertEquals(userEntryB, filter.filter(userEntry));
141
142    // scope a and b
143    scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
144    scopes.put(b, HConstants.REPLICATION_SCOPE_GLOBAL);
145    userEntry = createEntry(scopes, a, b);
146    assertEquals(userEntryB, filter.filter(userEntry));
147    scopes.put(a, HConstants.REPLICATION_SCOPE_LOCAL);
148    assertEquals(userEntryB, filter.filter(userEntry));
149  }
150
151  WALEntryFilter nullFilter = new WALEntryFilter() {
152    @Override
153    public Entry filter(Entry entry) {
154      return null;
155    }
156  };
157
158  WALEntryFilter passFilter = new WALEntryFilter() {
159    @Override
160    public Entry filter(Entry entry) {
161      return entry;
162    }
163  };
164
165  @Test
166  public void testChainWALEntryFilter() {
167    Entry userEntry = createEntry(null, a, b, c);
168
169    ChainWALEntryFilter filter = new ChainWALEntryFilter(passFilter);
170    assertEquals(createEntry(null, a,b,c), filter.filter(userEntry));
171
172    filter = new ChainWALEntryFilter(passFilter, passFilter);
173    assertEquals(createEntry(null, a,b,c), filter.filter(userEntry));
174
175    filter = new ChainWALEntryFilter(passFilter, passFilter, passFilter);
176    assertEquals(createEntry(null, a,b,c), filter.filter(userEntry));
177
178    filter = new ChainWALEntryFilter(nullFilter);
179    assertEquals(null, filter.filter(userEntry));
180
181    filter = new ChainWALEntryFilter(nullFilter, passFilter);
182    assertEquals(null, filter.filter(userEntry));
183
184    filter = new ChainWALEntryFilter(passFilter, nullFilter);
185    assertEquals(null, filter.filter(userEntry));
186
187    filter = new ChainWALEntryFilter(nullFilter, passFilter, nullFilter);
188    assertEquals(null, filter.filter(userEntry));
189
190    filter = new ChainWALEntryFilter(nullFilter, nullFilter);
191    assertEquals(null, filter.filter(userEntry));
192
193    // flatten
194    filter =
195        new ChainWALEntryFilter(
196          new ChainWALEntryFilter(passFilter,
197            new ChainWALEntryFilter(passFilter, passFilter),
198          new ChainWALEntryFilter(passFilter),
199          new ChainWALEntryFilter(passFilter)),
200          new ChainWALEntryFilter(passFilter));
201    assertEquals(createEntry(null, a,b,c), filter.filter(userEntry));
202
203
204    filter =
205        new ChainWALEntryFilter(
206          new ChainWALEntryFilter(passFilter,
207            new ChainWALEntryFilter(passFilter,
208              new ChainWALEntryFilter(nullFilter))),
209          new ChainWALEntryFilter(passFilter));
210    assertEquals(null, filter.filter(userEntry));
211  }
212
213  @Test
214  public void testNamespaceTableCfWALEntryFilter() {
215    ReplicationPeer peer = mock(ReplicationPeer.class);
216    ReplicationPeerConfig peerConfig = mock(ReplicationPeerConfig.class);
217
218    // 1. replicate_all flag is false, no namespaces and table-cfs config
219    when(peerConfig.replicateAllUserTables()).thenReturn(false);
220    when(peerConfig.getNamespaces()).thenReturn(null);
221    when(peerConfig.getTableCFsMap()).thenReturn(null);
222    when(peer.getPeerConfig()).thenReturn(peerConfig);
223    Entry userEntry = createEntry(null, a, b, c);
224    ChainWALEntryFilter filter =
225        new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
226    assertEquals(null, filter.filter(userEntry));
227
228    // 2. replicate_all flag is false, and only config table-cfs in peer
229    // empty map
230    userEntry = createEntry(null, a, b, c);
231    Map<TableName, List<String>> tableCfs = new HashMap<>();
232    when(peerConfig.replicateAllUserTables()).thenReturn(false);
233    when(peerConfig.getTableCFsMap()).thenReturn(tableCfs);
234    when(peer.getPeerConfig()).thenReturn(peerConfig);
235    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
236    assertEquals(null, filter.filter(userEntry));
237
238    // table bar
239    userEntry = createEntry(null, a, b, c);
240    tableCfs = new HashMap<>();
241    tableCfs.put(TableName.valueOf("bar"), null);
242    when(peerConfig.replicateAllUserTables()).thenReturn(false);
243    when(peerConfig.getTableCFsMap()).thenReturn(tableCfs);
244    when(peer.getPeerConfig()).thenReturn(peerConfig);
245    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
246    assertEquals(null, filter.filter(userEntry));
247
248    // table foo:a
249    userEntry = createEntry(null, a, b, c);
250    tableCfs = new HashMap<>();
251    tableCfs.put(TableName.valueOf("foo"), Lists.newArrayList("a"));
252    when(peerConfig.replicateAllUserTables()).thenReturn(false);
253    when(peerConfig.getTableCFsMap()).thenReturn(tableCfs);
254    when(peer.getPeerConfig()).thenReturn(peerConfig);
255    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
256    assertEquals(createEntry(null, a), filter.filter(userEntry));
257
258    // table foo:a,c
259    userEntry = createEntry(null, a, b, c, d);
260    tableCfs = new HashMap<>();
261    tableCfs.put(TableName.valueOf("foo"), Lists.newArrayList("a", "c"));
262    when(peerConfig.replicateAllUserTables()).thenReturn(false);
263    when(peerConfig.getTableCFsMap()).thenReturn(tableCfs);
264    when(peer.getPeerConfig()).thenReturn(peerConfig);
265    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
266    assertEquals(createEntry(null, a,c), filter.filter(userEntry));
267
268    // 3. replicate_all flag is false, and only config namespaces in peer
269    when(peer.getTableCFs()).thenReturn(null);
270    // empty set
271    Set<String> namespaces = new HashSet<>();
272    when(peerConfig.replicateAllUserTables()).thenReturn(false);
273    when(peerConfig.getNamespaces()).thenReturn(namespaces);
274    when(peerConfig.getTableCFsMap()).thenReturn(null);
275    when(peer.getPeerConfig()).thenReturn(peerConfig);
276    userEntry = createEntry(null, a, b, c);
277    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
278    assertEquals(null, filter.filter(userEntry));
279
280    // namespace default
281    namespaces.add("default");
282    when(peerConfig.replicateAllUserTables()).thenReturn(false);
283    when(peerConfig.getNamespaces()).thenReturn(namespaces);
284    when(peer.getPeerConfig()).thenReturn(peerConfig);
285    userEntry = createEntry(null, a, b, c);
286    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
287    assertEquals(createEntry(null, a,b,c), filter.filter(userEntry));
288
289    // namespace ns1
290    namespaces = new HashSet<>();
291    namespaces.add("ns1");
292    when(peerConfig.replicateAllUserTables()).thenReturn(false);
293    when(peerConfig.getNamespaces()).thenReturn(namespaces);
294    when(peer.getPeerConfig()).thenReturn(peerConfig);
295    userEntry = createEntry(null, a, b, c);
296    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
297    assertEquals(null, filter.filter(userEntry));
298
299    // 4. replicate_all flag is false, and config namespaces and table-cfs both
300    // Namespaces config should not confict with table-cfs config
301    namespaces = new HashSet<>();
302    tableCfs = new HashMap<>();
303    namespaces.add("ns1");
304    tableCfs.put(TableName.valueOf("foo"), Lists.newArrayList("a", "c"));
305    when(peerConfig.replicateAllUserTables()).thenReturn(false);
306    when(peerConfig.getNamespaces()).thenReturn(namespaces);
307    when(peerConfig.getTableCFsMap()).thenReturn(tableCfs);
308    when(peer.getPeerConfig()).thenReturn(peerConfig);
309    userEntry = createEntry(null, a, b, c);
310    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
311    assertEquals(createEntry(null, a, c), filter.filter(userEntry));
312
313    namespaces = new HashSet<>();
314    tableCfs = new HashMap<>();
315    namespaces.add("default");
316    tableCfs.put(TableName.valueOf("ns1:foo"), Lists.newArrayList("a", "c"));
317    when(peerConfig.replicateAllUserTables()).thenReturn(false);
318    when(peerConfig.getNamespaces()).thenReturn(namespaces);
319    when(peerConfig.getTableCFsMap()).thenReturn(tableCfs);
320    when(peer.getPeerConfig()).thenReturn(peerConfig);
321    userEntry = createEntry(null, a, b, c);
322    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
323    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
324
325    namespaces = new HashSet<>();
326    tableCfs = new HashMap<>();
327    namespaces.add("ns1");
328    tableCfs.put(TableName.valueOf("bar"), null);
329    when(peerConfig.replicateAllUserTables()).thenReturn(false);
330    when(peerConfig.getNamespaces()).thenReturn(namespaces);
331    when(peerConfig.getTableCFsMap()).thenReturn(tableCfs);
332    when(peer.getPeerConfig()).thenReturn(peerConfig);
333    userEntry = createEntry(null, a, b, c);
334    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
335    assertEquals(null, filter.filter(userEntry));
336  }
337
338  @Test
339  public void testNamespaceTableCfWALEntryFilter2() {
340    ReplicationPeer peer = mock(ReplicationPeer.class);
341    ReplicationPeerConfig peerConfig = mock(ReplicationPeerConfig.class);
342
343    // 1. replicate_all flag is true
344    // and no exclude namespaces and no exclude table-cfs config
345    when(peerConfig.replicateAllUserTables()).thenReturn(true);
346    when(peerConfig.getExcludeNamespaces()).thenReturn(null);
347    when(peerConfig.getExcludeTableCFsMap()).thenReturn(null);
348    when(peer.getPeerConfig()).thenReturn(peerConfig);
349    Entry userEntry = createEntry(null, a, b, c);
350    ChainWALEntryFilter filter =
351        new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
352    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
353
354    // 2. replicate_all flag is true, and only config exclude namespaces
355    // empty set
356    Set<String> namespaces = new HashSet<String>();
357    when(peerConfig.getExcludeNamespaces()).thenReturn(namespaces);
358    when(peerConfig.getExcludeTableCFsMap()).thenReturn(null);
359    when(peer.getPeerConfig()).thenReturn(peerConfig);
360    userEntry = createEntry(null, a, b, c);
361    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
362    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
363
364    // exclude namespace default
365    namespaces.add("default");
366    when(peerConfig.getExcludeNamespaces()).thenReturn(namespaces);
367    when(peerConfig.getExcludeTableCFsMap()).thenReturn(null);
368    when(peer.getPeerConfig()).thenReturn(peerConfig);
369    userEntry = createEntry(null, a, b, c);
370    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
371    assertEquals(null, filter.filter(userEntry));
372
373    // exclude namespace ns1
374    namespaces = new HashSet<String>();
375    namespaces.add("ns1");
376    when(peerConfig.getExcludeNamespaces()).thenReturn(namespaces);
377    when(peerConfig.getExcludeTableCFsMap()).thenReturn(null);
378    when(peer.getPeerConfig()).thenReturn(peerConfig);
379    userEntry = createEntry(null, a, b, c);
380    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
381    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
382
383    // 3. replicate_all flag is true, and only config exclude table-cfs
384    // empty table-cfs map
385    Map<TableName, List<String>> tableCfs = new HashMap<TableName, List<String>>();
386    when(peerConfig.getExcludeNamespaces()).thenReturn(null);
387    when(peerConfig.getExcludeTableCFsMap()).thenReturn(tableCfs);
388    when(peer.getPeerConfig()).thenReturn(peerConfig);
389    userEntry = createEntry(null, a, b, c);
390    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
391    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
392
393    // exclude table bar
394    tableCfs = new HashMap<TableName, List<String>>();
395    tableCfs.put(TableName.valueOf("bar"), null);
396    when(peerConfig.getExcludeNamespaces()).thenReturn(null);
397    when(peerConfig.getExcludeTableCFsMap()).thenReturn(tableCfs);
398    when(peer.getPeerConfig()).thenReturn(peerConfig);
399    userEntry = createEntry(null, a, b, c);
400    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
401    assertEquals(createEntry(null, a, b, c), filter.filter(userEntry));
402
403    // exclude table foo:a
404    tableCfs = new HashMap<TableName, List<String>>();
405    tableCfs.put(TableName.valueOf("foo"), Lists.newArrayList("a"));
406    when(peerConfig.getExcludeNamespaces()).thenReturn(null);
407    when(peerConfig.getExcludeTableCFsMap()).thenReturn(tableCfs);
408    when(peer.getPeerConfig()).thenReturn(peerConfig);
409    userEntry = createEntry(null, a, b, c);
410    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
411    assertEquals(createEntry(null, b, c), filter.filter(userEntry));
412
413    // 4. replicate_all flag is true, and config exclude namespaces and table-cfs both
414    // exclude ns1 and table foo:a,c
415    namespaces = new HashSet<String>();
416    tableCfs = new HashMap<TableName, List<String>>();
417    namespaces.add("ns1");
418    tableCfs.put(TableName.valueOf("foo"), Lists.newArrayList("a", "c"));
419    when(peerConfig.getExcludeNamespaces()).thenReturn(namespaces);
420    when(peerConfig.getExcludeTableCFsMap()).thenReturn(tableCfs);
421    when(peer.getPeerConfig()).thenReturn(peerConfig);
422    userEntry = createEntry(null, a, b, c);
423    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
424    assertEquals(createEntry(null, b), filter.filter(userEntry));
425
426    // exclude namespace default and table ns1:bar
427    namespaces = new HashSet<String>();
428    tableCfs = new HashMap<TableName, List<String>>();
429    namespaces.add("default");
430    tableCfs.put(TableName.valueOf("ns1:bar"), new ArrayList<String>());
431    when(peerConfig.getExcludeNamespaces()).thenReturn(namespaces);
432    when(peerConfig.getExcludeTableCFsMap()).thenReturn(tableCfs);
433    when(peer.getPeerConfig()).thenReturn(peerConfig);
434    userEntry = createEntry(null, a, b, c);
435    filter = new ChainWALEntryFilter(new NamespaceTableCfWALEntryFilter(peer));
436    assertEquals(null, filter.filter(userEntry));
437  }
438
439  private Entry createEntry(TreeMap<byte[], Integer> scopes, byte[]... kvs) {
440    WALKeyImpl key1 = new WALKeyImpl(new byte[0], TableName.valueOf("foo"),
441      System.currentTimeMillis(), scopes);
442    WALEdit edit1 = new WALEdit();
443
444    for (byte[] kv : kvs) {
445      edit1.add(new KeyValue(kv, kv, kv));
446    }
447    return new Entry(key1, edit1);
448  }
449
450  private void assertEquals(Entry e1, Entry e2) {
451    Assert.assertEquals(e1 == null, e2 == null);
452    if (e1 == null) {
453      return;
454    }
455
456    // do not compare WALKeys
457
458    // compare kvs
459    Assert.assertEquals(e1.getEdit() == null, e2.getEdit() == null);
460    if (e1.getEdit() == null) {
461      return;
462    }
463    List<Cell> cells1 = e1.getEdit().getCells();
464    List<Cell> cells2 = e2.getEdit().getCells();
465    Assert.assertEquals(cells1.size(), cells2.size());
466    for (int i = 0; i < cells1.size(); i++) {
467      CellComparatorImpl.COMPARATOR.compare(cells1.get(i), cells2.get(i));
468    }
469  }
470}