001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.shaded.protobuf;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.HashMap;
023import java.util.List;
024import java.util.Map;
025
026import org.apache.hadoop.hbase.Cell;
027import org.apache.hadoop.hbase.CellScanner;
028import org.apache.hadoop.hbase.DoNotRetryIOException;
029import org.apache.hadoop.hbase.ServerName;
030import org.apache.hadoop.hbase.client.RegionInfo;
031import org.apache.hadoop.hbase.client.Result;
032import org.apache.hadoop.hbase.client.SingleResponse;
033import org.apache.hadoop.hbase.ipc.ServerRpcController;
034import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
035import org.apache.hadoop.util.StringUtils;
036import org.apache.yetus.audience.InterfaceAudience;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
040import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
041import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse;
042import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
043import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoResponse;
044import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ServerInfo;
045import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
046import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MultiRequest;
047import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MultiResponse;
048import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionAction;
049import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionActionResult;
050import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ResultOrException;
051import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanResponse;
052import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds;
053import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
054import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameBytesPair;
055import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameInt64Pair;
056import org.apache.hadoop.hbase.shaded.protobuf.generated.MapReduceProtos.ScanMetrics;
057import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.EnableCatalogJanitorResponse;
058import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RunCatalogScanResponse;
059import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RunCleanerChoreResponse;
060import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
061
062import edu.umd.cs.findbugs.annotations.Nullable;
063
064/**
065 * Helper utility to build protocol buffer responses,
066 * or retrieve data from protocol buffer responses.
067 */
068@InterfaceAudience.Private
069public final class ResponseConverter {
070  private static final Logger LOG = LoggerFactory.getLogger(ResponseConverter.class);
071
072  private ResponseConverter() {
073  }
074
075  // Start utilities for Client
076  public static SingleResponse getResult(final ClientProtos.MutateRequest request,
077      final ClientProtos.MutateResponse response,
078      final CellScanner cells)
079          throws IOException {
080    SingleResponse singleResponse = new SingleResponse();
081    SingleResponse.Entry entry = new SingleResponse.Entry();
082    entry.setResult(ProtobufUtil.toResult(response.getResult(), cells));
083    entry.setProcessed(response.getProcessed());
084    singleResponse.setEntry(entry);
085    return singleResponse;
086  }
087
088  /**
089   * Get the results from a protocol buffer MultiResponse
090   *
091   * @param request the original protocol buffer MultiRequest
092   * @param response the protocol buffer MultiResponse to convert
093   * @param cells Cells to go with the passed in <code>proto</code>.  Can be null.
094   * @return the results that were in the MultiResponse (a Result or an Exception).
095   * @throws IOException
096   */
097  public static org.apache.hadoop.hbase.client.MultiResponse getResults(final MultiRequest request,
098      final MultiResponse response, final CellScanner cells)
099  throws IOException {
100    return getResults(request, null, response, cells);
101  }
102
103  /**
104   * Get the results from a protocol buffer MultiResponse
105   *
106   * @param request the original protocol buffer MultiRequest
107   * @param rowMutationsIndexMap Used to support RowMutations in batch
108   * @param response the protocol buffer MultiResponse to convert
109   * @param cells Cells to go with the passed in <code>proto</code>.  Can be null.
110   * @return the results that were in the MultiResponse (a Result or an Exception).
111   * @throws IOException
112   */
113  public static org.apache.hadoop.hbase.client.MultiResponse getResults(final MultiRequest request,
114      final Map<Integer, Integer> rowMutationsIndexMap, final MultiResponse response,
115      final CellScanner cells) throws IOException {
116    int requestRegionActionCount = request.getRegionActionCount();
117    int responseRegionActionResultCount = response.getRegionActionResultCount();
118    if (requestRegionActionCount != responseRegionActionResultCount) {
119      throw new IllegalStateException("Request mutation count=" + requestRegionActionCount +
120          " does not match response mutation result count=" + responseRegionActionResultCount);
121    }
122
123    org.apache.hadoop.hbase.client.MultiResponse results =
124      new org.apache.hadoop.hbase.client.MultiResponse();
125
126    for (int i = 0; i < responseRegionActionResultCount; i++) {
127      RegionAction actions = request.getRegionAction(i);
128      RegionActionResult actionResult = response.getRegionActionResult(i);
129      HBaseProtos.RegionSpecifier rs = actions.getRegion();
130      if (rs.hasType() &&
131          (rs.getType() != HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME)){
132        throw new IllegalArgumentException(
133            "We support only encoded types for protobuf multi response.");
134      }
135      byte[] regionName = rs.getValue().toByteArray();
136
137      if (actionResult.hasException()) {
138        Throwable regionException =  ProtobufUtil.toException(actionResult.getException());
139        results.addException(regionName, regionException);
140        continue;
141      }
142
143      if (actions.getActionCount() != actionResult.getResultOrExceptionCount()) {
144        throw new IllegalStateException("actions.getActionCount=" + actions.getActionCount() +
145            ", actionResult.getResultOrExceptionCount=" +
146            actionResult.getResultOrExceptionCount() + " for region " + actions.getRegion());
147      }
148
149      Object responseValue;
150
151      // For RowMutations action, if there is an exception, the exception is set
152      // at the RegionActionResult level and the ResultOrException is null at the original index
153      Integer rowMutationsIndex =
154          (rowMutationsIndexMap == null ? null : rowMutationsIndexMap.get(i));
155      if (rowMutationsIndex != null) {
156        // This RegionAction is from a RowMutations in a batch.
157        // If there is an exception from the server, the exception is set at
158        // the RegionActionResult level, which has been handled above.
159        responseValue = response.getProcessed() ?
160            ProtobufUtil.EMPTY_RESULT_EXISTS_TRUE :
161            ProtobufUtil.EMPTY_RESULT_EXISTS_FALSE;
162        results.add(regionName, rowMutationsIndex, responseValue);
163        continue;
164      }
165
166      for (ResultOrException roe : actionResult.getResultOrExceptionList()) {
167        if (roe.hasException()) {
168          responseValue = ProtobufUtil.toException(roe.getException());
169        } else if (roe.hasResult()) {
170          responseValue = ProtobufUtil.toResult(roe.getResult(), cells);
171        } else if (roe.hasServiceResult()) {
172          responseValue = roe.getServiceResult();
173        } else{
174          // Sometimes, the response is just "it was processed". Generally, this occurs for things
175          // like mutateRows where either we get back 'processed' (or not) and optionally some
176          // statistics about the regions we touched.
177          responseValue = response.getProcessed() ?
178                          ProtobufUtil.EMPTY_RESULT_EXISTS_TRUE :
179                          ProtobufUtil.EMPTY_RESULT_EXISTS_FALSE;
180        }
181        results.add(regionName, roe.getIndex(), responseValue);
182      }
183    }
184
185    if (response.hasRegionStatistics()) {
186      ClientProtos.MultiRegionLoadStats stats = response.getRegionStatistics();
187      for (int i = 0; i < stats.getRegionCount(); i++) {
188        results.addStatistic(stats.getRegion(i).getValue().toByteArray(), stats.getStat(i));
189      }
190    }
191
192    return results;
193  }
194
195  /**
196   * Wrap a throwable to an action result.
197   *
198   * @param t
199   * @return an action result builder
200   */
201  public static ResultOrException.Builder buildActionResult(final Throwable t) {
202    ResultOrException.Builder builder = ResultOrException.newBuilder();
203    if (t != null) builder.setException(buildException(t));
204    return builder;
205  }
206
207  /**
208   * Wrap a throwable to an action result.
209   *
210   * @param r
211   * @return an action result builder
212   */
213  public static ResultOrException.Builder buildActionResult(final ClientProtos.Result r) {
214    ResultOrException.Builder builder = ResultOrException.newBuilder();
215    if (r != null) builder.setResult(r);
216    return builder;
217  }
218
219  /**
220   * @param t
221   * @return NameValuePair of the exception name to stringified version os exception.
222   */
223  public static NameBytesPair buildException(final Throwable t) {
224    NameBytesPair.Builder parameterBuilder = NameBytesPair.newBuilder();
225    parameterBuilder.setName(t.getClass().getName());
226    parameterBuilder.setValue(
227      ByteString.copyFromUtf8(StringUtils.stringifyException(t)));
228    return parameterBuilder.build();
229  }
230
231// End utilities for Client
232// Start utilities for Admin
233
234  /**
235   * Get the list of region info from a GetOnlineRegionResponse
236   *
237   * @param proto the GetOnlineRegionResponse
238   * @return the list of region info
239   */
240  public static List<RegionInfo> getRegionInfos(final GetOnlineRegionResponse proto) {
241    if (proto == null || proto.getRegionInfoCount() == 0) return null;
242    return ProtobufUtil.getRegionInfos(proto);
243  }
244
245  /**
246   * Check if the region is closed from a CloseRegionResponse
247   *
248   * @param proto the CloseRegionResponse
249   * @return the region close state
250   */
251  public static boolean isClosed
252      (final CloseRegionResponse proto) {
253    if (proto == null || !proto.hasClosed()) return false;
254    return proto.getClosed();
255  }
256
257  /**
258   * A utility to build a GetServerInfoResponse.
259   *
260   * @param serverName
261   * @param webuiPort
262   * @return the response
263   */
264  public static GetServerInfoResponse buildGetServerInfoResponse(
265      final ServerName serverName, final int webuiPort) {
266    GetServerInfoResponse.Builder builder = GetServerInfoResponse.newBuilder();
267    ServerInfo.Builder serverInfoBuilder = ServerInfo.newBuilder();
268    serverInfoBuilder.setServerName(ProtobufUtil.toServerName(serverName));
269    if (webuiPort >= 0) {
270      serverInfoBuilder.setWebuiPort(webuiPort);
271    }
272    builder.setServerInfo(serverInfoBuilder.build());
273    return builder.build();
274  }
275
276  /**
277   * A utility to build a GetOnlineRegionResponse.
278   *
279   * @param regions
280   * @return the response
281   */
282  public static GetOnlineRegionResponse buildGetOnlineRegionResponse(
283      final List<RegionInfo> regions) {
284    GetOnlineRegionResponse.Builder builder = GetOnlineRegionResponse.newBuilder();
285    for (RegionInfo region: regions) {
286      builder.addRegionInfo(ProtobufUtil.toRegionInfo(region));
287    }
288    return builder.build();
289  }
290
291  /**
292   * Creates a response for the catalog scan request
293   * @return A RunCatalogScanResponse
294   */
295  public static RunCatalogScanResponse buildRunCatalogScanResponse(int numCleaned) {
296    return RunCatalogScanResponse.newBuilder().setScanResult(numCleaned).build();
297  }
298
299  /**
300   * Creates a response for the catalog scan request
301   * @return A EnableCatalogJanitorResponse
302   */
303  public static EnableCatalogJanitorResponse buildEnableCatalogJanitorResponse(boolean prevValue) {
304    return EnableCatalogJanitorResponse.newBuilder().setPrevValue(prevValue).build();
305  }
306
307  /**
308   * Creates a response for the cleaner chore request
309   * @return A RunCleanerChoreResponse
310   */
311  public static RunCleanerChoreResponse buildRunCleanerChoreResponse(boolean ran) {
312    return RunCleanerChoreResponse.newBuilder().setCleanerChoreRan(ran).build();
313  }
314
315// End utilities for Admin
316
317  /**
318   * Creates a response for the last flushed sequence Id request
319   * @return A GetLastFlushedSequenceIdResponse
320   */
321  public static GetLastFlushedSequenceIdResponse buildGetLastFlushedSequenceIdResponse(
322      RegionStoreSequenceIds ids) {
323    return GetLastFlushedSequenceIdResponse.newBuilder()
324        .setLastFlushedSequenceId(ids.getLastFlushedSequenceId())
325        .addAllStoreLastFlushedSequenceId(ids.getStoreSequenceIdList()).build();
326  }
327
328  /**
329   * Stores an exception encountered during RPC invocation so it can be passed back
330   * through to the client.
331   * @param controller the controller instance provided by the client when calling the service
332   * @param ioe the exception encountered
333   */
334  public static void setControllerException(com.google.protobuf.RpcController controller,
335      IOException ioe) {
336    if (controller != null) {
337      if (controller instanceof ServerRpcController) {
338        ((ServerRpcController)controller).setFailedOn(ioe);
339      } else {
340        controller.setFailed(StringUtils.stringifyException(ioe));
341      }
342    }
343  }
344
345  /**
346   * Retreivies exception stored during RPC invocation.
347   * @param controller the controller instance provided by the client when calling the service
348   * @return exception if any, or null; Will return DoNotRetryIOException for string represented
349   * failure causes in controller.
350   */
351  @Nullable
352  public static IOException getControllerException(RpcController controller) throws IOException {
353    if (controller != null && controller.failed()) {
354      if (controller instanceof ServerRpcController) {
355        return ((ServerRpcController)controller).getFailedOn();
356      } else {
357        return new DoNotRetryIOException(controller.errorText());
358      }
359    }
360    return null;
361  }
362
363
364  /**
365   * Create Results from the cells using the cells meta data.
366   * @param cellScanner
367   * @param response
368   * @return results
369   */
370  public static Result[] getResults(CellScanner cellScanner, ScanResponse response)
371      throws IOException {
372    if (response == null) return null;
373    // If cellscanner, then the number of Results to return is the count of elements in the
374    // cellsPerResult list.  Otherwise, it is how many results are embedded inside the response.
375    int noOfResults = cellScanner != null?
376      response.getCellsPerResultCount(): response.getResultsCount();
377    Result[] results = new Result[noOfResults];
378    for (int i = 0; i < noOfResults; i++) {
379      if (cellScanner != null) {
380        // Cells are out in cellblocks.  Group them up again as Results.  How many to read at a
381        // time will be found in getCellsLength -- length here is how many Cells in the i'th Result
382        int noOfCells = response.getCellsPerResult(i);
383        boolean isPartial =
384            response.getPartialFlagPerResultCount() > i ?
385                response.getPartialFlagPerResult(i) : false;
386        List<Cell> cells = new ArrayList<>(noOfCells);
387        for (int j = 0; j < noOfCells; j++) {
388          try {
389            if (cellScanner.advance() == false) {
390              // We are not able to retrieve the exact number of cells which ResultCellMeta says us.
391              // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
392              // same scanner will result in OutOfOrderScannerNextException
393              String msg = "Results sent from server=" + noOfResults + ". But only got " + i
394                + " results completely at client. Resetting the scanner to scan again.";
395              LOG.error(msg);
396              throw new DoNotRetryIOException(msg);
397            }
398          } catch (IOException ioe) {
399            // We are getting IOE while retrieving the cells for Results.
400            // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
401            // same scanner will result in OutOfOrderScannerNextException
402            LOG.error("Exception while reading cells from result."
403              + "Resetting the scanner to scan again.", ioe);
404            throw new DoNotRetryIOException("Resetting the scanner.", ioe);
405          }
406          cells.add(cellScanner.current());
407        }
408        results[i] = Result.create(cells, null, response.getStale(), isPartial);
409      } else {
410        // Result is pure pb.
411        results[i] = ProtobufUtil.toResult(response.getResults(i));
412      }
413    }
414    return results;
415  }
416
417  public static Map<String, Long> getScanMetrics(ScanResponse response) {
418    Map<String, Long> metricMap = new HashMap<>();
419    if (response == null || !response.hasScanMetrics()) {
420      return metricMap;
421    }
422
423    ScanMetrics metrics = response.getScanMetrics();
424    int numberOfMetrics = metrics.getMetricsCount();
425    for (int i = 0; i < numberOfMetrics; i++) {
426      NameInt64Pair metricPair = metrics.getMetrics(i);
427      if (metricPair != null) {
428        String name = metricPair.getName();
429        Long value = metricPair.getValue();
430        if (name != null && value != null) {
431          metricMap.put(name, value);
432        }
433      }
434    }
435
436    return metricMap;
437  }
438
439  /**
440   * Creates a protocol buffer ClearRegionBlockCacheResponse
441   *
442   * @return a ClearRegionBlockCacheResponse
443   */
444  public static AdminProtos.ClearRegionBlockCacheResponse buildClearRegionBlockCacheResponse(final HBaseProtos.CacheEvictionStats
445                                                                                   cacheEvictionStats) {
446    return AdminProtos.ClearRegionBlockCacheResponse.newBuilder().setStats(cacheEvictionStats).build();
447  }
448}