001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.shaded.protobuf;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.nio.ByteBuffer;
025import org.apache.hadoop.hbase.ByteBufferKeyValue;
026import org.apache.hadoop.hbase.Cell;
027import org.apache.hadoop.hbase.CellBuilderType;
028import org.apache.hadoop.hbase.CellComparatorImpl;
029import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.KeyValue;
032import org.apache.hadoop.hbase.client.Append;
033import org.apache.hadoop.hbase.client.Delete;
034import org.apache.hadoop.hbase.client.Get;
035import org.apache.hadoop.hbase.client.Increment;
036import org.apache.hadoop.hbase.client.Put;
037import org.apache.hadoop.hbase.io.TimeRange;
038import org.apache.hadoop.hbase.testclassification.SmallTests;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.junit.ClassRule;
041import org.junit.Test;
042import org.junit.experimental.categories.Category;
043
044import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
045import org.apache.hbase.thirdparty.com.google.protobuf.Any;
046import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
047import org.apache.hbase.thirdparty.com.google.protobuf.BytesValue;
048
049import org.apache.hadoop.hbase.shaded.protobuf.generated.CellProtos;
050import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
051import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.Column;
052import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto;
053import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.ColumnValue;
054import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.ColumnValue.QualifierValue;
055import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.DeleteType;
056import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType;
057import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameBytesPair;
058import org.apache.hadoop.hbase.shaded.protobuf.generated.LockServiceProtos;
059import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
060
061@Category(SmallTests.class)
062public class TestProtobufUtil {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066      HBaseClassTestRule.forClass(TestProtobufUtil.class);
067
068  public TestProtobufUtil() {
069  }
070
071  @Test
072  public void testException() throws IOException {
073    NameBytesPair.Builder builder = NameBytesPair.newBuilder();
074    final String omg = "OMG!!!";
075    builder.setName("java.io.IOException");
076    builder.setValue(ByteString.copyFrom(Bytes.toBytes(omg)));
077    Throwable t = ProtobufUtil.toException(builder.build());
078    assertEquals(omg, t.getMessage());
079    builder.clear();
080    builder.setName("org.apache.hadoop.ipc.RemoteException");
081    builder.setValue(ByteString.copyFrom(Bytes.toBytes(omg)));
082    t = ProtobufUtil.toException(builder.build());
083    assertEquals(omg, t.getMessage());
084  }
085
086  /**
087   * Test basic Get conversions.
088   *
089   * @throws IOException
090   */
091  @Test
092  public void testGet() throws IOException {
093    ClientProtos.Get.Builder getBuilder = ClientProtos.Get.newBuilder();
094    getBuilder.setRow(ByteString.copyFromUtf8("row"));
095    Column.Builder columnBuilder = Column.newBuilder();
096    columnBuilder.setFamily(ByteString.copyFromUtf8("f1"));
097    columnBuilder.addQualifier(ByteString.copyFromUtf8("c1"));
098    columnBuilder.addQualifier(ByteString.copyFromUtf8("c2"));
099    getBuilder.addColumn(columnBuilder.build());
100
101    columnBuilder.clear();
102    columnBuilder.setFamily(ByteString.copyFromUtf8("f2"));
103    getBuilder.addColumn(columnBuilder.build());
104    getBuilder.setLoadColumnFamiliesOnDemand(true);
105    ClientProtos.Get proto = getBuilder.build();
106    // default fields
107    assertEquals(1, proto.getMaxVersions());
108    assertEquals(true, proto.getCacheBlocks());
109
110    // set the default value for equal comparison
111    getBuilder = ClientProtos.Get.newBuilder(proto);
112    getBuilder.setMaxVersions(1);
113    getBuilder.setCacheBlocks(true);
114    getBuilder.setTimeRange(ProtobufUtil.toTimeRange(TimeRange.allTime()));
115    Get get = ProtobufUtil.toGet(proto);
116    assertEquals(getBuilder.build(), ProtobufUtil.toGet(get));
117  }
118
119  /**
120   * Test Delete Mutate conversions.
121   *
122   * @throws IOException
123   */
124  @Test
125  public void testDelete() throws IOException {
126    MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
127    mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
128    mutateBuilder.setMutateType(MutationType.DELETE);
129    mutateBuilder.setTimestamp(111111);
130    ColumnValue.Builder valueBuilder = ColumnValue.newBuilder();
131    valueBuilder.setFamily(ByteString.copyFromUtf8("f1"));
132    QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
133    qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
134    qualifierBuilder.setDeleteType(DeleteType.DELETE_ONE_VERSION);
135    qualifierBuilder.setTimestamp(111222);
136    valueBuilder.addQualifierValue(qualifierBuilder.build());
137    qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
138    qualifierBuilder.setDeleteType(DeleteType.DELETE_MULTIPLE_VERSIONS);
139    qualifierBuilder.setTimestamp(111333);
140    valueBuilder.addQualifierValue(qualifierBuilder.build());
141    mutateBuilder.addColumnValue(valueBuilder.build());
142
143    MutationProto proto = mutateBuilder.build();
144    // default fields
145    assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
146
147    // set the default value for equal comparison
148    mutateBuilder = MutationProto.newBuilder(proto);
149    mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
150
151    Delete delete = ProtobufUtil.toDelete(proto);
152
153    // delete always have empty value,
154    // add empty value to the original mutate
155    for (ColumnValue.Builder column:
156        mutateBuilder.getColumnValueBuilderList()) {
157      for (QualifierValue.Builder qualifier:
158          column.getQualifierValueBuilderList()) {
159        qualifier.setValue(ByteString.EMPTY);
160      }
161    }
162    assertEquals(mutateBuilder.build(),
163      ProtobufUtil.toMutation(MutationType.DELETE, delete));
164  }
165
166  /**
167   * Test Put Mutate conversions.
168   *
169   * @throws IOException
170   */
171  @Test
172  public void testPut() throws IOException {
173    MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
174    mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
175    mutateBuilder.setMutateType(MutationType.PUT);
176    mutateBuilder.setTimestamp(111111);
177    ColumnValue.Builder valueBuilder = ColumnValue.newBuilder();
178    valueBuilder.setFamily(ByteString.copyFromUtf8("f1"));
179    QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
180    qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
181    qualifierBuilder.setValue(ByteString.copyFromUtf8("v1"));
182    valueBuilder.addQualifierValue(qualifierBuilder.build());
183    qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
184    qualifierBuilder.setValue(ByteString.copyFromUtf8("v2"));
185    qualifierBuilder.setTimestamp(222222);
186    valueBuilder.addQualifierValue(qualifierBuilder.build());
187    mutateBuilder.addColumnValue(valueBuilder.build());
188
189    MutationProto proto = mutateBuilder.build();
190    // default fields
191    assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
192
193    // set the default value for equal comparison
194    mutateBuilder = MutationProto.newBuilder(proto);
195    mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
196
197    Put put = ProtobufUtil.toPut(proto);
198
199    // put value always use the default timestamp if no
200    // value level timestamp specified,
201    // add the timestamp to the original mutate
202    long timestamp = put.getTimestamp();
203    for (ColumnValue.Builder column:
204        mutateBuilder.getColumnValueBuilderList()) {
205      for (QualifierValue.Builder qualifier:
206          column.getQualifierValueBuilderList()) {
207        if (!qualifier.hasTimestamp()) {
208          qualifier.setTimestamp(timestamp);
209        }
210      }
211    }
212    assertEquals(mutateBuilder.build(),
213      ProtobufUtil.toMutation(MutationType.PUT, put));
214  }
215
216  /**
217   * Test basic Scan conversions.
218   *
219   * @throws IOException
220   */
221  @Test
222  public void testScan() throws IOException {
223    ClientProtos.Scan.Builder scanBuilder = ClientProtos.Scan.newBuilder();
224    scanBuilder.setStartRow(ByteString.copyFromUtf8("row1"));
225    scanBuilder.setStopRow(ByteString.copyFromUtf8("row2"));
226    Column.Builder columnBuilder = Column.newBuilder();
227    columnBuilder.setFamily(ByteString.copyFromUtf8("f1"));
228    columnBuilder.addQualifier(ByteString.copyFromUtf8("c1"));
229    columnBuilder.addQualifier(ByteString.copyFromUtf8("c2"));
230    scanBuilder.addColumn(columnBuilder.build());
231
232    columnBuilder.clear();
233    columnBuilder.setFamily(ByteString.copyFromUtf8("f2"));
234    scanBuilder.addColumn(columnBuilder.build());
235
236    ClientProtos.Scan proto = scanBuilder.build();
237
238    // Verify default values
239    assertEquals(1, proto.getMaxVersions());
240    assertEquals(true, proto.getCacheBlocks());
241
242    // Verify fields survive ClientProtos.Scan -> Scan -> ClientProtos.Scan
243    // conversion
244    scanBuilder = ClientProtos.Scan.newBuilder(proto);
245    scanBuilder.setMaxVersions(2);
246    scanBuilder.setCacheBlocks(false);
247    scanBuilder.setCaching(1024);
248    scanBuilder.setTimeRange(ProtobufUtil.toTimeRange(TimeRange.allTime()));
249    scanBuilder.setIncludeStopRow(false);
250    ClientProtos.Scan expectedProto = scanBuilder.build();
251
252    ClientProtos.Scan actualProto = ProtobufUtil.toScan(
253        ProtobufUtil.toScan(expectedProto));
254    assertEquals(expectedProto, actualProto);
255  }
256
257  @Test
258  public void testToCell() throws Exception {
259    KeyValue kv1 =
260        new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
261    KeyValue kv2 =
262        new KeyValue(Bytes.toBytes("bbb"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
263    KeyValue kv3 =
264        new KeyValue(Bytes.toBytes("ccc"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
265    byte[] arr = new byte[kv1.getLength() + kv2.getLength() + kv3.getLength()];
266    System.arraycopy(kv1.getBuffer(), kv1.getOffset(), arr, 0, kv1.getLength());
267    System.arraycopy(kv2.getBuffer(), kv2.getOffset(), arr, kv1.getLength(), kv2.getLength());
268    System.arraycopy(kv3.getBuffer(), kv3.getOffset(), arr, kv1.getLength() + kv2.getLength(),
269      kv3.getLength());
270    ByteBuffer dbb = ByteBuffer.allocateDirect(arr.length);
271    dbb.put(arr);
272    ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(dbb, kv1.getLength(), kv2.getLength());
273    CellProtos.Cell cell = ProtobufUtil.toCell(offheapKV);
274    Cell newOffheapKV = ProtobufUtil.toCell(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY), cell);
275    assertTrue(CellComparatorImpl.COMPARATOR.compare(offheapKV, newOffheapKV) == 0);
276  }
277
278  /**
279   * Test Increment Mutate conversions.
280   *
281   * @throws IOException
282   */
283  @Test
284  public void testIncrement() throws IOException {
285    long timeStamp = 111111;
286    MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
287    mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
288    mutateBuilder.setMutateType(MutationProto.MutationType.INCREMENT);
289    ColumnValue.Builder valueBuilder = ColumnValue.newBuilder();
290    valueBuilder.setFamily(ByteString.copyFromUtf8("f1"));
291    QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
292    qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
293    qualifierBuilder.setValue(ByteString.copyFrom(Bytes.toBytes(11L)));
294    qualifierBuilder.setTimestamp(timeStamp);
295    valueBuilder.addQualifierValue(qualifierBuilder.build());
296    qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
297    qualifierBuilder.setValue(ByteString.copyFrom(Bytes.toBytes(22L)));
298    valueBuilder.addQualifierValue(qualifierBuilder.build());
299    mutateBuilder.addColumnValue(valueBuilder.build());
300
301    MutationProto proto = mutateBuilder.build();
302    // default fields
303    assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
304
305    // set the default value for equal comparison
306    mutateBuilder = MutationProto.newBuilder(proto);
307    mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
308
309    Increment increment = ProtobufUtil.toIncrement(proto, null);
310    mutateBuilder.setTimestamp(increment.getTimestamp());
311    mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(increment.getTimeRange()));
312    assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.INCREMENT, increment));
313  }
314
315  /**
316   * Test Append Mutate conversions.
317   *
318   * @throws IOException
319   */
320  @Test
321  public void testAppend() throws IOException {
322    long timeStamp = 111111;
323    MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
324    mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
325    mutateBuilder.setMutateType(MutationType.APPEND);
326    mutateBuilder.setTimestamp(timeStamp);
327    ColumnValue.Builder valueBuilder = ColumnValue.newBuilder();
328    valueBuilder.setFamily(ByteString.copyFromUtf8("f1"));
329    QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
330    qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
331    qualifierBuilder.setValue(ByteString.copyFromUtf8("v1"));
332    qualifierBuilder.setTimestamp(timeStamp);
333    valueBuilder.addQualifierValue(qualifierBuilder.build());
334    qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
335    qualifierBuilder.setValue(ByteString.copyFromUtf8("v2"));
336    valueBuilder.addQualifierValue(qualifierBuilder.build());
337    mutateBuilder.addColumnValue(valueBuilder.build());
338
339    MutationProto proto = mutateBuilder.build();
340    // default fields
341    assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
342
343    // set the default value for equal comparison
344    mutateBuilder = MutationProto.newBuilder(proto);
345    mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
346
347    Append append = ProtobufUtil.toAppend(proto, null);
348
349    // append always use the latest timestamp,
350    // reset the timestamp to the original mutate
351    mutateBuilder.setTimestamp(append.getTimestamp());
352    mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(append.getTimeRange()));
353    assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.APPEND, append));
354  }
355
356  private static ProcedureProtos.Procedure.Builder createProcedureBuilder(long procId) {
357    ProcedureProtos.Procedure.Builder builder = ProcedureProtos.Procedure.newBuilder();
358    builder.setProcId(procId);
359    builder.setClassName("java.lang.Object");
360    builder.setSubmittedTime(0);
361    builder.setState(ProcedureProtos.ProcedureState.RUNNABLE);
362    builder.setLastUpdate(0);
363
364    return builder;
365  }
366
367  private static ProcedureProtos.Procedure createProcedure(long procId) {
368    return createProcedureBuilder(procId).build();
369  }
370
371  private static LockServiceProtos.LockedResource createLockedResource(
372      LockServiceProtos.LockedResourceType resourceType, String resourceName,
373      LockServiceProtos.LockType lockType,
374      ProcedureProtos.Procedure exclusiveLockOwnerProcedure, int sharedLockCount) {
375    LockServiceProtos.LockedResource.Builder build = LockServiceProtos.LockedResource.newBuilder();
376    build.setResourceType(resourceType);
377    build.setResourceName(resourceName);
378    build.setLockType(lockType);
379    if (exclusiveLockOwnerProcedure != null) {
380      build.setExclusiveLockOwnerProcedure(exclusiveLockOwnerProcedure);
381    }
382    build.setSharedLockCount(sharedLockCount);
383
384    return build.build();
385  }
386
387  @Test
388  public void testProcedureInfo() {
389    ProcedureProtos.Procedure.Builder builder = createProcedureBuilder(1);
390    ByteString stateBytes = ByteString.copyFrom(new byte[] { 65 });
391    BytesValue state = BytesValue.newBuilder().setValue(stateBytes).build();
392    builder.addStateMessage(Any.pack(state));
393    ProcedureProtos.Procedure procedure = builder.build();
394
395    String procJson = ProtobufUtil.toProcedureJson(Lists.newArrayList(procedure));
396    assertEquals("[{"
397        + "\"className\":\"java.lang.Object\","
398        + "\"procId\":\"1\","
399        + "\"submittedTime\":\"0\","
400        + "\"state\":\"RUNNABLE\","
401        + "\"lastUpdate\":\"0\","
402        + "\"stateMessage\":[{\"value\":\"QQ==\"}]"
403        + "}]", procJson);
404  }
405
406  @Test
407  public void testServerLockInfo() {
408    LockServiceProtos.LockedResource resource = createLockedResource(
409        LockServiceProtos.LockedResourceType.SERVER, "server",
410        LockServiceProtos.LockType.SHARED, null, 2);
411
412    String lockJson = ProtobufUtil.toLockJson(Lists.newArrayList(resource));
413    assertEquals("[{"
414        + "\"resourceType\":\"SERVER\","
415        + "\"resourceName\":\"server\","
416        + "\"lockType\":\"SHARED\","
417        + "\"sharedLockCount\":2"
418        + "}]", lockJson);
419  }
420
421  @Test
422  public void testNamespaceLockInfo() {
423    LockServiceProtos.LockedResource resource = createLockedResource(
424        LockServiceProtos.LockedResourceType.NAMESPACE, "ns",
425        LockServiceProtos.LockType.EXCLUSIVE, createProcedure(2), 0);
426
427    String lockJson = ProtobufUtil.toLockJson(Lists.newArrayList(resource));
428    assertEquals("[{"
429        + "\"resourceType\":\"NAMESPACE\","
430        + "\"resourceName\":\"ns\","
431        + "\"lockType\":\"EXCLUSIVE\","
432        + "\"exclusiveLockOwnerProcedure\":{"
433          + "\"className\":\"java.lang.Object\","
434          + "\"procId\":\"2\","
435          + "\"submittedTime\":\"0\","
436          + "\"state\":\"RUNNABLE\","
437          + "\"lastUpdate\":\"0\""
438        + "},"
439        + "\"sharedLockCount\":0"
440        + "}]", lockJson);
441  }
442
443  @Test
444  public void testTableLockInfo() {
445    LockServiceProtos.LockedResource resource = createLockedResource(
446        LockServiceProtos.LockedResourceType.TABLE, "table",
447        LockServiceProtos.LockType.SHARED, null, 2);
448
449    String lockJson = ProtobufUtil.toLockJson(Lists.newArrayList(resource));
450    assertEquals("[{"
451        + "\"resourceType\":\"TABLE\","
452        + "\"resourceName\":\"table\","
453        + "\"lockType\":\"SHARED\","
454        + "\"sharedLockCount\":2"
455        + "}]", lockJson);
456  }
457
458  @Test
459  public void testRegionLockInfo() {
460    LockServiceProtos.LockedResource resource = createLockedResource(
461        LockServiceProtos.LockedResourceType.REGION, "region",
462        LockServiceProtos.LockType.EXCLUSIVE, createProcedure(3), 0);
463
464    String lockJson = ProtobufUtil.toLockJson(Lists.newArrayList(resource));
465    assertEquals("[{"
466        + "\"resourceType\":\"REGION\","
467        + "\"resourceName\":\"region\","
468        + "\"lockType\":\"EXCLUSIVE\","
469        + "\"exclusiveLockOwnerProcedure\":{"
470          + "\"className\":\"java.lang.Object\","
471          + "\"procId\":\"3\","
472          + "\"submittedTime\":\"0\","
473          + "\"state\":\"RUNNABLE\","
474          + "\"lastUpdate\":\"0\""
475        + "},"
476        + "\"sharedLockCount\":0"
477        + "}]", lockJson);
478  }
479}