001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *    http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.hadoop.hbase.spark.example.hbasecontext;
018
019import org.apache.hadoop.conf.Configuration;
020import org.apache.hadoop.hbase.HBaseConfiguration;
021import org.apache.hadoop.hbase.TableName;
022import org.apache.hadoop.hbase.client.Put;
023import org.apache.hadoop.hbase.spark.JavaHBaseContext;
024import org.apache.hadoop.hbase.util.Bytes;
025import org.apache.spark.SparkConf;
026import org.apache.spark.api.java.JavaSparkContext;
027import org.apache.spark.api.java.function.Function;
028import org.apache.spark.streaming.Duration;
029import org.apache.spark.streaming.api.java.JavaReceiverInputDStream;
030import org.apache.spark.streaming.api.java.JavaStreamingContext;
031
032/**
033 * This is a simple example of BulkPut with Spark Streaming
034 */
035final public class JavaHBaseStreamingBulkPutExample {
036
037  private JavaHBaseStreamingBulkPutExample() {}
038
039  public static void main(String[] args) {
040    if (args.length < 4) {
041      System.out.println("JavaHBaseBulkPutExample  " +
042              "{host} {port} {tableName}");
043      return;
044    }
045
046    String host = args[0];
047    String port = args[1];
048    String tableName = args[2];
049
050    SparkConf sparkConf =
051            new SparkConf().setAppName("JavaHBaseStreamingBulkPutExample " +
052                    tableName + ":" + port + ":" + tableName);
053
054    JavaSparkContext jsc = new JavaSparkContext(sparkConf);
055
056    try {
057      JavaStreamingContext jssc =
058              new JavaStreamingContext(jsc, new Duration(1000));
059
060      JavaReceiverInputDStream<String> javaDstream =
061              jssc.socketTextStream(host, Integer.parseInt(port));
062
063      Configuration conf = HBaseConfiguration.create();
064
065      JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf);
066
067      hbaseContext.streamBulkPut(javaDstream,
068              TableName.valueOf(tableName),
069              new PutFunction());
070    } finally {
071      jsc.stop();
072    }
073  }
074
075  public static class PutFunction implements Function<String, Put> {
076
077    private static final long serialVersionUID = 1L;
078
079    public Put call(String v) throws Exception {
080      String[] part = v.split(",");
081      Put put = new Put(Bytes.toBytes(part[0]));
082
083      put.addColumn(Bytes.toBytes(part[1]),
084              Bytes.toBytes(part[2]),
085              Bytes.toBytes(part[3]));
086      return put;
087    }
088
089  }
090}