001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.List;
028import org.apache.hadoop.hbase.Cell;
029import org.apache.hadoop.hbase.CellComparatorImpl;
030import org.apache.hadoop.hbase.CompareOperator;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtility;
033import org.apache.hadoop.hbase.HColumnDescriptor;
034import org.apache.hadoop.hbase.HRegionInfo;
035import org.apache.hadoop.hbase.HTableDescriptor;
036import org.apache.hadoop.hbase.KeyValue;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.Put;
039import org.apache.hadoop.hbase.client.Scan;
040import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
041import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
042import org.apache.hadoop.hbase.regionserver.HRegion;
043import org.apache.hadoop.hbase.regionserver.InternalScanner;
044import org.apache.hadoop.hbase.testclassification.FilterTests;
045import org.apache.hadoop.hbase.testclassification.SmallTests;
046import org.apache.hadoop.hbase.util.Bytes;
047import org.junit.After;
048import org.junit.Before;
049import org.junit.ClassRule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052import org.slf4j.Logger;
053import org.slf4j.LoggerFactory;
054
055@Category({FilterTests.class, SmallTests.class})
056public class TestDependentColumnFilter {
057
058  @ClassRule
059  public static final HBaseClassTestRule CLASS_RULE =
060      HBaseClassTestRule.forClass(TestDependentColumnFilter.class);
061
062  private static final Logger LOG = LoggerFactory.getLogger(TestDependentColumnFilter.class);
063  private static final byte[][] ROWS = {
064    Bytes.toBytes("test1"),Bytes.toBytes("test2")
065  };
066  private static final byte[][] FAMILIES = {
067    Bytes.toBytes("familyOne"),Bytes.toBytes("familyTwo")
068  };
069  private static final long STAMP_BASE = System.currentTimeMillis();
070  private static final long[] STAMPS = {
071    STAMP_BASE-100, STAMP_BASE-200, STAMP_BASE-300
072  };
073  private static final byte[] QUALIFIER = Bytes.toBytes("qualifier");
074  private static final byte[][] BAD_VALS = {
075    Bytes.toBytes("bad1"), Bytes.toBytes("bad2"), Bytes.toBytes("bad3")
076  };
077  private static final byte[] MATCH_VAL = Bytes.toBytes("match");
078  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
079
080  List<KeyValue> testVals;
081  private HRegion region;
082
083  @Before
084  public void setUp() throws Exception {
085    testVals = makeTestVals();
086
087    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(this.getClass().getSimpleName()));
088    HColumnDescriptor hcd0 = new HColumnDescriptor(FAMILIES[0]);
089    hcd0.setMaxVersions(3);
090    htd.addFamily(hcd0);
091    HColumnDescriptor hcd1 = new HColumnDescriptor(FAMILIES[1]);
092    hcd1.setMaxVersions(3);
093    htd.addFamily(hcd1);
094    HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
095    this.region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
096        TEST_UTIL.getConfiguration(), htd);
097    addData();
098  }
099
100  @After
101  public void tearDown() throws Exception {
102    HBaseTestingUtility.closeRegionAndWAL(this.region);
103  }
104
105  private void addData() throws IOException {
106    Put put = new Put(ROWS[0]);
107    // add in an entry for each stamp, with 2 as a "good" value
108    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]);
109    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]);
110    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
111    // add in entries for stamps 0 and 2.
112    // without a value check both will be "accepted"
113    // with one 2 will be accepted(since the corresponding ts entry
114    // has a matching value
115    put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[0], BAD_VALS[0]);
116    put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
117
118    this.region.put(put);
119
120    put = new Put(ROWS[1]);
121    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]);
122    // there is no corresponding timestamp for this so it should never pass
123    put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
124    // if we reverse the qualifiers this one should pass
125    put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL);
126    // should pass
127    put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]);
128
129    this.region.put(put);
130  }
131
132  private List<KeyValue> makeTestVals() {
133    List<KeyValue> testVals = new ArrayList<>();
134    testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]));
135    testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]));
136    testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]));
137    testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL));
138    testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]));
139
140    return testVals;
141  }
142
143  /**
144   * This shouldn't be confused with TestFilter#verifyScan
145   * as expectedKeys is not the per row total, but the scan total
146   *
147   * @param s
148   * @param expectedRows
149   * @param expectedCells
150   * @throws IOException
151   */
152  private void verifyScan(Scan s, long expectedRows, long expectedCells)
153  throws IOException {
154    InternalScanner scanner = this.region.getScanner(s);
155    List<Cell> results = new ArrayList<>();
156    int i = 0;
157    int cells = 0;
158    for (boolean done = true; done; i++) {
159      done = scanner.next(results);
160      Arrays.sort(results.toArray(new Cell[results.size()]),
161          CellComparatorImpl.COMPARATOR);
162      LOG.info("counter=" + i + ", " + results);
163      if (results.isEmpty()) break;
164      cells += results.size();
165      assertTrue("Scanned too many rows! Only expected " + expectedRows +
166          " total but already scanned " + (i+1), expectedRows > i);
167      assertTrue("Expected " + expectedCells + " cells total but " +
168          "already scanned " + cells, expectedCells >= cells);
169      results.clear();
170    }
171    assertEquals("Expected " + expectedRows + " rows but scanned " + i +
172        " rows", expectedRows, i);
173    assertEquals("Expected " + expectedCells + " cells but scanned " + cells +
174            " cells", expectedCells, cells);
175  }
176
177  /**
178   * Test scans using a DependentColumnFilter
179   */
180  @Test
181  public void testScans() throws Exception {
182    Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
183
184    Scan scan = new Scan();
185    scan.setFilter(filter);
186    scan.setMaxVersions(Integer.MAX_VALUE);
187
188    verifyScan(scan, 2, 8);
189
190    // drop the filtering cells
191    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true);
192    scan = new Scan();
193    scan.setFilter(filter);
194    scan.setMaxVersions(Integer.MAX_VALUE);
195
196    verifyScan(scan, 2, 3);
197
198    // include a comparator operation
199    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, false,
200    CompareOperator.EQUAL, new BinaryComparator(MATCH_VAL));
201    scan = new Scan();
202    scan.setFilter(filter);
203    scan.setMaxVersions(Integer.MAX_VALUE);
204
205    /*
206     * expecting to get the following 3 cells
207     * row 0
208     *   put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
209     *   put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
210     * row 1
211     *   put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
212     */
213    verifyScan(scan, 2, 3);
214
215    // include a comparator operation and drop comparator
216    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true,
217    CompareOperator.EQUAL, new BinaryComparator(MATCH_VAL));
218    scan = new Scan();
219    scan.setFilter(filter);
220    scan.setMaxVersions(Integer.MAX_VALUE);
221
222    /*
223     * expecting to get the following 1 cell
224     * row 0
225     *   put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
226     */
227    verifyScan(scan, 1, 1);
228
229  }
230
231  /**
232   * Test that the filter correctly drops rows without a corresponding timestamp
233   *
234   * @throws Exception
235   */
236  @Test
237  public void testFilterDropping() throws Exception {
238    Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
239    List<Cell> accepted = new ArrayList<>();
240    for(Cell val : testVals) {
241      if(filter.filterCell(val) == ReturnCode.INCLUDE) {
242        accepted.add(val);
243      }
244    }
245    assertEquals("check all values accepted from filterCell", 5, accepted.size());
246
247    filter.filterRowCells(accepted);
248    assertEquals("check filterRow(List<KeyValue>) dropped cell without corresponding column entry", 4, accepted.size());
249
250    // start do it again with dependent column dropping on
251    filter = new DependentColumnFilter(FAMILIES[1], QUALIFIER, true);
252    accepted.clear();
253    for(KeyValue val : testVals) {
254        if(filter.filterCell(val) == ReturnCode.INCLUDE) {
255          accepted.add(val);
256        }
257      }
258      assertEquals("check the filtering column cells got dropped", 2, accepted.size());
259
260      filter.filterRowCells(accepted);
261      assertEquals("check cell retention", 2, accepted.size());
262  }
263
264  /**
265   * Test for HBASE-8794. Avoid NullPointerException in DependentColumnFilter.toString().
266   */
267  @Test
268  public void testToStringWithNullComparator() {
269    // Test constructor that implicitly sets a null comparator
270    Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
271    assertNotNull(filter.toString());
272    assertTrue("check string contains 'null' as compatator is null",
273      filter.toString().contains("null"));
274
275    // Test constructor with explicit null comparator
276    filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOperator.EQUAL, null);
277    assertNotNull(filter.toString());
278    assertTrue("check string contains 'null' as compatator is null",
279      filter.toString().contains("null"));
280  }
281
282  @Test
283  public void testToStringWithNonNullComparator() {
284    Filter filter =
285        new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOperator.EQUAL,
286            new BinaryComparator(MATCH_VAL));
287    assertNotNull(filter.toString());
288    assertTrue("check string contains comparator value", filter.toString().contains("match"));
289  }
290
291}
292