/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hive.hbase;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import junit.framework.TestCase;

import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
import org.apache.hadoop.hive.serde2.lazy.LazyPrimitive;
import org.apache.hadoop.hive.serde2.lazy.LazyString;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;

/**
 * TestLazyHBaseObject is a test for the LazyHBaseXXX classes.
 */
public class TestLazyHBaseObject extends TestCase {
  /**
   * Test the LazyMap class with Integer-to-String.
   * @throws SerDeException
   */
  public void testLazyHBaseCellMap1() throws SerDeException {
    // Map of Integer to String
    Text nullSequence = new Text("\\N");
    ObjectInspector oi = LazyFactory.createLazyObjectInspector(
      TypeInfoUtils.getTypeInfosFromTypeString("map<int,string>").get(0),
      new byte[]{(byte)1, (byte)2}, 0, nullSequence, false, (byte)0);

    LazyHBaseCellMap b = new LazyHBaseCellMap((LazyMapObjectInspector) oi);

    // Initialize a result
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfa"),
        Bytes.toBytes("col1"), Bytes.toBytes("cfacol1")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfa"),
        Bytes.toBytes("col2"), Bytes.toBytes("cfacol2")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfb"),
        Bytes.toBytes("2"), Bytes.toBytes("def")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfb"),
        Bytes.toBytes("-1"), Bytes.toBytes("")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfb"),
        Bytes.toBytes("0"), Bytes.toBytes("0")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfb"),
        Bytes.toBytes("8"), Bytes.toBytes("abc")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfc"),
        Bytes.toBytes("col3"), Bytes.toBytes("cfccol3")));

    Result r = new Result(kvs);

    List<Boolean> mapBinaryStorage = new ArrayList<Boolean>();
    mapBinaryStorage.add(false);
    mapBinaryStorage.add(false);

    b.init(r, "cfb".getBytes(), mapBinaryStorage);

    assertEquals(
      new Text("def"),
      ((LazyString)b.getMapValueElement(
        new IntWritable(2))).getWritableObject());

    assertNull(b.getMapValueElement(new IntWritable(-1)));

    assertEquals(
      new Text("0"),
      ((LazyString)b.getMapValueElement(
        new IntWritable(0))).getWritableObject());

    assertEquals(
      new Text("abc"),
      ((LazyString)b.getMapValueElement(
        new IntWritable(8))).getWritableObject());

    assertNull(b.getMapValueElement(new IntWritable(12345)));

    assertEquals("{0:'0',2:'def',8:'abc'}".replace('\'', '\"'),
      SerDeUtils.getJSONString(b, oi));
  }

  /**
   * Test the LazyMap class with String-to-String.
   * @throws SerDeException
   */
  public void testLazyHBaseCellMap2() throws SerDeException {
    // Map of String to String
    Text nullSequence = new Text("\\N");
    ObjectInspector oi = LazyFactory.createLazyObjectInspector(
      TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(0),
      new byte[]{(byte)'#', (byte)'\t'}, 0, nullSequence, false, (byte)0);

    LazyHBaseCellMap b = new LazyHBaseCellMap((LazyMapObjectInspector) oi);

    // Initialize a result
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfa"), Bytes.toBytes("col1"), Bytes.toBytes("cfacol1")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfa"), Bytes.toBytes("col2"), Bytes.toBytes("cfacol2")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("2"), Bytes.toBytes("d\tf")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("-1"), Bytes.toBytes("")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("0"), Bytes.toBytes("0")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("8"), Bytes.toBytes("abc")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfc"), Bytes.toBytes("col3"), Bytes.toBytes("cfccol3")));

    Result r = new Result(kvs);
    List<Boolean> mapBinaryStorage = new ArrayList<Boolean>();
    mapBinaryStorage.add(false);
    mapBinaryStorage.add(false);

    b.init(r, "cfb".getBytes(), mapBinaryStorage);

    assertEquals(
      new Text("d\tf"),
      ((LazyString)b.getMapValueElement(
        new Text("2"))).getWritableObject());

    assertNull(b.getMapValueElement(new Text("-1")));

    assertEquals(
      new Text("0"),
      ((LazyString)b.getMapValueElement(
        new Text("0"))).getWritableObject());

    assertEquals(
      new Text("abc"),
      ((LazyString)b.getMapValueElement(
        new Text("8"))).getWritableObject());

    assertNull(b.getMapValueElement(new Text("-")));

    assertEquals(
      "{'0':'0','2':'d\\tf','8':'abc'}".replace('\'', '\"'),
      SerDeUtils.getJSONString(b, oi));
  }

  /**
   * Test the LazyHBaseCellMap class for the case where both the key and the value in the family
   * map are stored in binary format using the appropriate LazyPrimitive objects.
   * @throws SerDeException
   */
  public void testLazyHBaseCellMap3() throws SerDeException {

    Text nullSequence = new Text("\\N");
    TypeInfo mapBinaryIntKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map<int,int>");
    ObjectInspector oi = LazyFactory.createLazyObjectInspector(
        mapBinaryIntKeyValue, new byte [] {(byte)1, (byte) 2}, 0, nullSequence, false, (byte) 0);
    LazyHBaseCellMap hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);

    List<KeyValue> kvs = new ArrayList<KeyValue>();
    byte [] rowKey = "row-key".getBytes();
    byte [] cfInt = "cf-int".getBytes();
    kvs.add(new KeyValue(rowKey, cfInt, Bytes.toBytes(1), Bytes.toBytes(1)));
    Result result = new Result(kvs);
    List<Boolean> mapBinaryStorage = new ArrayList<Boolean>();
    mapBinaryStorage.add(true);
    mapBinaryStorage.add(true);
    hbaseCellMap.init(result, cfInt, mapBinaryStorage);
    IntWritable expectedIntValue = new IntWritable(1);
    LazyPrimitive<?, ?> lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedIntValue);

    assertEquals(expectedIntValue, lazyPrimitive.getWritableObject());

    kvs.clear();
    kvs.add(new KeyValue(
        rowKey, cfInt, Bytes.toBytes(Integer.MIN_VALUE), Bytes.toBytes(Integer.MIN_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfInt, mapBinaryStorage);
    expectedIntValue = new IntWritable(Integer.MIN_VALUE);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedIntValue);

    assertEquals(expectedIntValue, lazyPrimitive.getWritableObject());

    kvs.clear();
    kvs.add(new KeyValue(
        rowKey, cfInt, Bytes.toBytes(Integer.MAX_VALUE), Bytes.toBytes(Integer.MAX_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfInt, mapBinaryStorage);
    expectedIntValue = new IntWritable(Integer.MAX_VALUE);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedIntValue);

    assertEquals(expectedIntValue, lazyPrimitive.getWritableObject());

    TypeInfo mapBinaryByteKeyValue =
      TypeInfoUtils.getTypeInfoFromTypeString("map<tinyint,tinyint>");
    oi = LazyFactory.createLazyObjectInspector(
        mapBinaryByteKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false, (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte [] cfByte = "cf-byte".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfByte, new byte [] {(byte) 1}, new byte [] {(byte) 1}));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfByte, mapBinaryStorage);
    ByteWritable expectedByteValue = new ByteWritable((byte) 1);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedByteValue);

    assertEquals(expectedByteValue, lazyPrimitive.getWritableObject());

    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfByte, new byte [] {Byte.MIN_VALUE},
      new byte [] {Byte.MIN_VALUE}));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfByte, mapBinaryStorage);
    expectedByteValue = new ByteWritable(Byte.MIN_VALUE);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedByteValue);

    assertEquals(expectedByteValue, lazyPrimitive.getWritableObject());

    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfByte, new byte [] {Byte.MAX_VALUE},
      new byte [] {Byte.MAX_VALUE}));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfByte, mapBinaryStorage);
    expectedByteValue = new ByteWritable(Byte.MAX_VALUE);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedByteValue);

    assertEquals(expectedByteValue, lazyPrimitive.getWritableObject());

    TypeInfo mapBinaryShortKeyValue =
      TypeInfoUtils.getTypeInfoFromTypeString("map<smallint,smallint>");
    oi = LazyFactory.createLazyObjectInspector(
      mapBinaryShortKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false, (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte [] cfShort = "cf-short".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfShort, Bytes.toBytes((short) 1), Bytes.toBytes((short) 1)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfShort, mapBinaryStorage);
    ShortWritable expectedShortValue = new ShortWritable((short) 1);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedShortValue);

    assertEquals(expectedShortValue, lazyPrimitive.getWritableObject());

    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfShort, Bytes.toBytes(Short.MIN_VALUE),
      Bytes.toBytes(Short.MIN_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfShort, mapBinaryStorage);
    expectedShortValue = new ShortWritable(Short.MIN_VALUE);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedShortValue);

    assertEquals(expectedShortValue, lazyPrimitive.getWritableObject());

    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfShort, Bytes.toBytes(Short.MAX_VALUE),
      Bytes.toBytes(Short.MAX_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfShort, mapBinaryStorage);
    expectedShortValue = new ShortWritable(Short.MAX_VALUE);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedShortValue);

    assertEquals(expectedShortValue, lazyPrimitive.getWritableObject());

    TypeInfo mapBinaryLongKeyValue =
      TypeInfoUtils.getTypeInfoFromTypeString("map<bigint,bigint>");
    oi = LazyFactory.createLazyObjectInspector(
        mapBinaryLongKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false, (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte [] cfLong = "cf-long".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfLong, Bytes.toBytes((long) 1), Bytes.toBytes((long) 1)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfLong, mapBinaryStorage);
    LongWritable expectedLongValue = new LongWritable(1);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedLongValue);

    assertEquals(expectedLongValue, lazyPrimitive.getWritableObject());

    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfLong, Bytes.toBytes(Long.MIN_VALUE),
      Bytes.toBytes(Long.MIN_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfLong, mapBinaryStorage);
    expectedLongValue = new LongWritable(Long.MIN_VALUE);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedLongValue);

    assertEquals(expectedLongValue, lazyPrimitive.getWritableObject());

    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfLong, Bytes.toBytes(Long.MAX_VALUE),
      Bytes.toBytes(Long.MAX_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfLong, mapBinaryStorage);
    expectedLongValue = new LongWritable(Long.MAX_VALUE);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedLongValue);

    assertEquals(expectedLongValue, lazyPrimitive.getWritableObject());

    TypeInfo mapBinaryFloatKeyValue =
      TypeInfoUtils.getTypeInfoFromTypeString("map<float,float>");
    oi = LazyFactory.createLazyObjectInspector(
        mapBinaryFloatKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false,
        (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte [] cfFloat = "cf-float".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfFloat, Bytes.toBytes((float) 1.0F),
      Bytes.toBytes((float) 1.0F)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfFloat, mapBinaryStorage);
    FloatWritable expectedFloatValue = new FloatWritable(1.0F);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedFloatValue);

    assertEquals(expectedFloatValue, lazyPrimitive.getWritableObject());

    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfFloat, Bytes.toBytes((float) Float.MIN_VALUE),
      Bytes.toBytes((float) Float.MIN_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfFloat, mapBinaryStorage);
    expectedFloatValue = new FloatWritable(Float.MIN_VALUE);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedFloatValue);

    assertEquals(expectedFloatValue, lazyPrimitive.getWritableObject());

    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfFloat, Bytes.toBytes((float) Float.MAX_VALUE),
      Bytes.toBytes((float) Float.MAX_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfFloat, mapBinaryStorage);
    expectedFloatValue = new FloatWritable(Float.MAX_VALUE);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedFloatValue);

    assertEquals(expectedFloatValue, lazyPrimitive.getWritableObject());

    TypeInfo mapBinaryDoubleKeyValue =
      TypeInfoUtils.getTypeInfoFromTypeString("map<double,double>");
    oi = LazyFactory.createLazyObjectInspector(
        mapBinaryDoubleKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false,
        (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte [] cfDouble = "cf-double".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfDouble, Bytes.toBytes(1.0), Bytes.toBytes(1.0)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfDouble, mapBinaryStorage);
    DoubleWritable expectedDoubleValue = new DoubleWritable(1.0);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedDoubleValue);

    assertEquals(expectedDoubleValue, lazyPrimitive.getWritableObject());

    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfDouble, Bytes.toBytes(Double.MIN_VALUE),
      Bytes.toBytes(Double.MIN_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfDouble, mapBinaryStorage);
    expectedDoubleValue = new DoubleWritable(Double.MIN_VALUE);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedDoubleValue);

    assertEquals(expectedDoubleValue, lazyPrimitive.getWritableObject());

    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfDouble, Bytes.toBytes(Double.MAX_VALUE),
      Bytes.toBytes(Double.MAX_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfDouble, mapBinaryStorage);
    expectedDoubleValue = new DoubleWritable(Double.MAX_VALUE);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedDoubleValue);

    assertEquals(expectedDoubleValue, lazyPrimitive.getWritableObject());

    TypeInfo mapBinaryBooleanKeyValue =
      TypeInfoUtils.getTypeInfoFromTypeString("map<boolean,boolean>");
    oi = LazyFactory.createLazyObjectInspector(
        mapBinaryBooleanKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false,
        (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte [] cfBoolean = "cf-boolean".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfBoolean, Bytes.toBytes(false), Bytes.toBytes(false)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfBoolean, mapBinaryStorage);
    BooleanWritable expectedBooleanValue = new BooleanWritable(false);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedBooleanValue);

    assertEquals(expectedBooleanValue, lazyPrimitive.getWritableObject());

    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfBoolean, Bytes.toBytes(true), Bytes.toBytes(true)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfBoolean, mapBinaryStorage);
    expectedBooleanValue = new BooleanWritable(true);
    lazyPrimitive =
      (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedBooleanValue);

    assertEquals(expectedBooleanValue, lazyPrimitive.getWritableObject());
  }

  /**
   * Test the LazyHBaseRow class with one-for-one mappings between
   * Hive fields and HBase columns.
   * @throws SerDeException
   */
  public void testLazyHBaseRow1() throws SerDeException {
    List<TypeInfo> fieldTypeInfos =
      TypeInfoUtils.getTypeInfosFromTypeString(
          "string,int,array<string>,map<string,string>,string");
    List<String> fieldNames = Arrays.asList("key", "a", "b", "c", "d");
    Text nullSequence = new Text("\\N");

    String hbaseColsMapping = ":key,cfa:a,cfa:b,cfb:c,cfb:d";
    ColumnMappings columnMappings = null;

    try {
      columnMappings = HBaseSerDe.parseColumnsMapping(hbaseColsMapping);
    } catch (SerDeException e) {
      fail(e.toString());
    }

    for (ColumnMapping colMap : columnMappings) {
      if (!colMap.hbaseRowKey && colMap.qualifierName == null) {
        colMap.binaryStorage.add(false);
        colMap.binaryStorage.add(false);
      } else {
        colMap.binaryStorage.add(false);
      }
    }

    ObjectInspector oi = LazyFactory.createLazyStructInspector(fieldNames,
      fieldTypeInfos, new byte[] {' ', ':', '='},
      nullSequence, false, false, (byte)0);
    LazyHBaseRow o = new LazyHBaseRow((LazySimpleStructObjectInspector) oi, columnMappings);

    List<KeyValue> kvs = new ArrayList<KeyValue>();

    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfa"), Bytes.toBytes("a"), Bytes.toBytes("123")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfa"), Bytes.toBytes("b"), Bytes.toBytes("a:b:c")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("c"), Bytes.toBytes("d=e:f=g")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("hi")));

    Result r = new Result(kvs);
    o.init(r);

    assertEquals(
      ("{'key':'test-row','a':123,'b':['a','b','c'],"
        + "'c':{'d':'e','f':'g'},'d':'hi'}").replace("'", "\""),
      SerDeUtils.getJSONString(o, oi));

    kvs.clear();
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfa"), Bytes.toBytes("a"), Bytes.toBytes("123")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("c"), Bytes.toBytes("d=e:f=g")));

    r = new Result(kvs);
    o.init(r);

    assertEquals(
        ("{'key':'test-row','a':123,'b':null,"
          + "'c':{'d':'e','f':'g'},'d':null}").replace("'", "\""),
        SerDeUtils.getJSONString(o, oi));

    kvs.clear();
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfa"), Bytes.toBytes("b"), Bytes.toBytes("a")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("c"), Bytes.toBytes("d=\\N:f=g:h")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("no")));

    r = new Result(kvs);
    o.init(r);

    assertEquals(
        ("{'key':'test-row','a':null,'b':['a'],"
          + "'c':{'d':null,'f':'g','h':null},'d':'no'}").replace("'", "\""),
        SerDeUtils.getJSONString(o, oi));

    kvs.clear();
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfa"), Bytes.toBytes("b"), Bytes.toBytes(":a::")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("no")));

    r = new Result(kvs);
    o.init(r);

    assertEquals(
      ("{'key':'test-row','a':null,'b':['','a','',''],"
        + "'c':null,'d':'no'}").replace("'", "\""),
      SerDeUtils.getJSONString(o, oi));

    // This is intentionally duplicated because of HIVE-3179
    assertEquals(
      ("{'key':'test-row','a':null,'b':['','a','',''],"
       + "'c':null,'d':'no'}").replace("'", "\""),
      SerDeUtils.getJSONString(o, oi));

    kvs.clear();
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfa"), Bytes.toBytes("a"), Bytes.toBytes("123")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfa"), Bytes.toBytes("b"), Bytes.toBytes("")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("c"), Bytes.toBytes("")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("")));

    r = new Result(kvs);
    o.init(r);

    assertEquals(
      "{'key':'test-row','a':123,'b':[],'c':{},'d':''}".replace("'", "\""),
      SerDeUtils.getJSONString(o, oi));
  }

  /**
   * Test the LazyHBaseRow class with a mapping from a Hive field to
   * an HBase column family.
   * @throws SerDeException
   */
  public void testLazyHBaseRow2() throws SerDeException {
    // column family is mapped to Map<string,string>
    List<TypeInfo> fieldTypeInfos =
      TypeInfoUtils.getTypeInfosFromTypeString(
        "string,int,array<string>,map<string,string>,string");
    List<String> fieldNames = Arrays.asList(
      new String[]{"key", "a", "b", "c", "d"});
    Text nullSequence = new Text("\\N");
    String hbaseColsMapping = ":key,cfa:a,cfa:b,cfb:,cfc:d";

    ColumnMappings columnMappings = null;

    try {
      columnMappings = HBaseSerDe.parseColumnsMapping(hbaseColsMapping);
    } catch (SerDeException e) {
      fail(e.toString());
    }

    for (ColumnMapping colMap : columnMappings) {
      if (!colMap.hbaseRowKey && colMap.qualifierName == null) {
        colMap.binaryStorage.add(false);
        colMap.binaryStorage.add(false);
      } else {
        colMap.binaryStorage.add(false);
      }
    }

    ObjectInspector oi = LazyFactory.createLazyStructInspector(
      fieldNames,
      fieldTypeInfos,
      new byte[] {' ', ':', '='},
      nullSequence, false, false, (byte) 0);
    LazyHBaseRow o = new LazyHBaseRow((LazySimpleStructObjectInspector) oi, columnMappings);

    List<KeyValue> kvs = new ArrayList<KeyValue>();
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfa"), Bytes.toBytes("a"), Bytes.toBytes("123")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfa"), Bytes.toBytes("b"), Bytes.toBytes("a:b:c")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("e")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("f"), Bytes.toBytes("g")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfc"), Bytes.toBytes("d"), Bytes.toBytes("hi")));

    Result r = new Result(kvs);
    o.init(r);

    assertEquals(
      ("{'key':'test-row','a':123,'b':['a','b','c'],"
        + "'c':{'d':'e','f':'g'},'d':'hi'}").replace("'", "\""),
      SerDeUtils.getJSONString(o, oi));

    kvs.clear();
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfa"), Bytes.toBytes("a"), Bytes.toBytes("123")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("e")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("f"), Bytes.toBytes("g")));

    r = new Result(kvs);
    o.init(r);

    assertEquals(
      ("{'key':'test-row','a':123,'b':null,"
        + "'c':{'d':'e','f':'g'},'d':null}").replace("'", "\""),
      SerDeUtils.getJSONString(o, oi));

    kvs.clear();
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfa"), Bytes.toBytes("b"), Bytes.toBytes("a")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfb"), Bytes.toBytes("f"), Bytes.toBytes("g")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfc"), Bytes.toBytes("d"), Bytes.toBytes("no")));

    r = new Result(kvs);
    o.init(r);

    assertEquals(
      ("{'key':'test-row','a':null,'b':['a'],"
        + "'c':{'f':'g'},'d':'no'}").replace("'", "\""),
      SerDeUtils.getJSONString(o, oi));

    kvs.clear();
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfa"), Bytes.toBytes("b"), Bytes.toBytes(":a::")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfc"), Bytes.toBytes("d"), Bytes.toBytes("no")));

    r = new Result(kvs);
    o.init(r);

    assertEquals(
      ("{'key':'test-row','a':null,'b':['','a','',''],"
        + "'c':{},'d':'no'}").replace("'", "\""),
      SerDeUtils.getJSONString(o, oi));

    kvs.clear();
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfa"), Bytes.toBytes("a"), Bytes.toBytes("123")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfa"), Bytes.toBytes("b"), Bytes.toBytes("")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"),
        Bytes.toBytes("cfc"), Bytes.toBytes("d"), Bytes.toBytes("")));

    r = new Result(kvs);
    o.init(r);

    assertEquals(
      "{'key':'test-row','a':123,'b':[],'c':{},'d':''}".replace("'", "\""),
      SerDeUtils.getJSONString(o, oi));
  }

  /**
   * Test the LazyHBaseRow class with a one-to-one/onto mapping between Hive columns and
   * HBase column family/column qualifier pairs. The column types are primitive and fields
   * are stored in binary format in HBase.
   * @throws SerDeException
   */
  public void testLazyHBaseRow3() throws SerDeException {

    List<TypeInfo> fieldTypeInfos = TypeInfoUtils.getTypeInfosFromTypeString(
        "string,int,tinyint,smallint,bigint,float,double,string,boolean");
    List<String> fieldNames = Arrays.asList(
        new String [] {"key", "c_int", "c_byte", "c_short", "c_long", "c_float", "c_double",
            "c_string", "c_bool"});
    Text nullSequence = new Text("\\N");
    String hbaseColumnsMapping = ":key#str,cf-int:cq-int#bin,cf-byte:cq-byte#bin,"
      + "cf-short:cq-short#bin,cf-long:cq-long#bin,cf-float:cq-float#bin,cf-double:cq-double#bin,"
      + "cf-string:cq-string#str,cf-bool:cq-bool#bin";
    ColumnMappings columnMappings = null;

    try {
      columnMappings = HBaseSerDe.parseColumnsMapping(hbaseColumnsMapping);
    } catch (SerDeException e) {
      fail(e.toString());
    }

    ColumnMapping[] columnsMapping = columnMappings.getColumnsMapping();
    for (int i = 0; i < columnsMapping.length; i++) {
      ColumnMapping colMap = columnsMapping[i];

      if (i == 0 || i == 7) {
        colMap.binaryStorage.add(false);
      } else {
        colMap.binaryStorage.add(true);
      }
    }

    ObjectInspector oi =
      LazyFactory.createLazyStructInspector(fieldNames, fieldTypeInfos,
          new byte [] {' ', ':', '='}, nullSequence, false, false, (byte) 0);

    LazyHBaseRow o = new LazyHBaseRow((LazySimpleStructObjectInspector) oi, columnMappings);

    byte [] rowKey = "row-key".getBytes();
    List<KeyValue> kvs = new ArrayList<KeyValue>();
    byte [] value;

    for (int i = 1; i < columnsMapping.length; i++) {

      switch (i) {

      case 1:
        value = Bytes.toBytes(1);
        break;

      case 2:
        value = new byte[]{(byte)1};
        break;

      case 3:
        value = Bytes.toBytes((short) 1);
        break;

      case 4:
        value = Bytes.toBytes((long) 1);
        break;

      case 5:
        value = Bytes.toBytes((float) 1.0F);
        break;

      case 6:
        value = Bytes.toBytes((double) 1.0);
        break;

      case 7:
        value = "Hadoop, Hive, with HBase storage handler.".getBytes();
        break;

      case 8:
        value = Bytes.toBytes(true);
        break;

      default:
        throw new RuntimeException("Not expected: " + i);
      }

      ColumnMapping colMap = columnsMapping[i];
      kvs.add(new KeyValue(rowKey, colMap.familyNameBytes, colMap.qualifierNameBytes, value));
    }

    Collections.sort(kvs, KeyValue.COMPARATOR);
    Result result = new Result(kvs);
    o.init(result);
    List<? extends StructField> fieldRefs = ((StructObjectInspector) oi).getAllStructFieldRefs();


    for (int i = 0; i < fieldRefs.size(); i++) {
      Object fieldData = ((StructObjectInspector) oi).getStructFieldData(o, fieldRefs.get(i));

      assert(fieldData != null);
      assert(fieldData instanceof LazyPrimitive<?, ?>);
      Writable writable = ((LazyPrimitive<?, ?>) fieldData).getWritableObject();

      switch (i) {
      case 0:
        Text text = new Text("row-key");
        assertEquals(text, writable);
        break;

      case 1:
        IntWritable iw = new IntWritable(1);
        assertEquals(iw, writable);
        break;

      case 2:
        ByteWritable bw = new ByteWritable((byte) 1);
        assertEquals(bw, writable);
        break;

      case 3:
        ShortWritable sw = new ShortWritable((short) 1);
        assertEquals(sw, writable);
        break;

      case 4:
        LongWritable lw = new LongWritable(1);
        assertEquals(lw, writable);
        break;

      case 5:
        FloatWritable fw = new FloatWritable(1.0F);
        assertEquals(fw, writable);
        break;

      case 6:
        DoubleWritable dw = new DoubleWritable(1.0);
        assertEquals(dw, writable);
        break;

      case 7:
        Text t = new Text("Hadoop, Hive, with HBase storage handler.");
        assertEquals(t, writable);
        break;

      case 8:
        BooleanWritable boolWritable = new BooleanWritable(true);
        assertEquals(boolWritable, writable);
        break;

      default:
        fail("Error: Unanticipated value in deserializing fields for HBaseSerDe.");
        break;
      }
    }
  }
}
