/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hive.hbase;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import junit.framework.Assert;
import junit.framework.TestCase;

import org.apache.avro.Schema;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.Encoder;
import org.apache.avro.io.EncoderFactory;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.hbase.avro.Address;
import org.apache.hadoop.hive.hbase.avro.ContactInfo;
import org.apache.hadoop.hive.hbase.avro.Employee;
import org.apache.hadoop.hive.hbase.avro.Gender;
import org.apache.hadoop.hive.hbase.avro.HomePhone;
import org.apache.hadoop.hive.hbase.avro.OfficePhone;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils.AvroTableProperties;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.lazy.LazyPrimitive;
import org.apache.hadoop.hive.serde2.lazy.LazyStruct;
import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.thrift.TException;

/**
 * Tests the HBaseSerDe class.
 */
public class TestHBaseSerDe extends TestCase {

  static final byte[] TEST_BYTE_ARRAY = Bytes.toBytes("test");

  private static final String RECORD_SCHEMA = "{\n" +
      "  \"namespace\": \"testing.test.mctesty\",\n" +
      "  \"name\": \"oneRecord\",\n" +
      "  \"type\": \"record\",\n" +
      "  \"fields\": [\n" +
      "    {\n" +
      "      \"name\":\"aRecord\",\n" +
      "      \"type\":{\"type\":\"record\",\n" +
      "              \"name\":\"recordWithinARecord\",\n" +
      "              \"fields\": [\n" +
      "                 {\n" +
      "                  \"name\":\"int1\",\n" +
      "                  \"type\":\"int\"\n" +
      "                },\n" +
      "                {\n" +
      "                  \"name\":\"boolean1\",\n" +
      "                  \"type\":\"boolean\"\n" +
      "                },\n" +
      "                {\n" +
      "                  \"name\":\"long1\",\n" +
      "                  \"type\":\"long\"\n" +
      "                }\n" +
      "      ]}\n" +
      "    }\n" +
      "  ]\n" +
      "}";

  private static final String RECORD_SCHEMA_EVOLVED = "{\n" +
      "  \"namespace\": \"testing.test.mctesty\",\n" +
      "  \"name\": \"oneRecord\",\n" +
      "  \"type\": \"record\",\n" +
      "  \"fields\": [\n" +
      "    {\n" +
      "      \"name\":\"aRecord\",\n" +
      "      \"type\":{\"type\":\"record\",\n" +
      "              \"name\":\"recordWithinARecord\",\n" +
      "              \"fields\": [\n" +
      "                 {\n" +
      "                  \"name\":\"int1\",\n" +
      "                  \"type\":\"int\"\n" +
      "                },\n" +
      "                {\n" +
      "                  \"name\":\"string1\",\n" +
      "                  \"type\":\"string\", \"default\": \"test\"\n" +
      "                },\n" +
      "                {\n" +
      "                  \"name\":\"boolean1\",\n" +
      "                  \"type\":\"boolean\"\n" +
      "                },\n" +
      "                {\n" +
      "                  \"name\":\"long1\",\n" +
      "                  \"type\":\"long\"\n" +
      "                }\n" +
      "      ]}\n" +
      "    }\n" +
      "  ]\n" +
      "}";

  private static final String EXPECTED_DESERIALIZED_AVRO_STRING =
      "{\"key\":\"test-row1\",\"cola_avro\":{\"arecord\":{\"int1\":42,\"boolean1\":true,"
          + "\"long1\":42432234234}}}";

  private static final String EXPECTED_DESERIALIZED_AVRO_STRING_2 =
 "{\"key\":\"test-row1\","
      + "\"cola_avro\":{\"employeename\":\"Avro Employee1\","
      + "\"employeeid\":11111,\"age\":25,\"gender\":\"FEMALE\","
      + "\"contactinfo\":{\"address\":[{\"address1\":\"Avro First Address1\",\"address2\":"
      + "\"Avro Second Address1\",\"city\":\"Avro City1\",\"zipcode\":123456,\"county\":"
      + "{0:{\"areacode\":999,\"number\":1234567890}},\"aliases\":null,\"metadata\":"
      + "{\"testkey\":\"testvalue\"}},{\"address1\":\"Avro First Address1\",\"address2\":"
      + "\"Avro Second Address1\",\"city\":\"Avro City1\",\"zipcode\":123456,\"county\":"
      + "{0:{\"areacode\":999,\"number\":1234567890}},\"aliases\":null,\"metadata\":"
      + "{\"testkey\":\"testvalue\"}}],\"homephone\":{\"areacode\":999,\"number\":1234567890},"
      + "\"officephone\":{\"areacode\":999,\"number\":1234455555}}}}";

  private static final String EXPECTED_DESERIALIZED_AVRO_STRING_3 =
      "{\"key\":\"test-row1\",\"cola_avro\":{\"arecord\":{\"int1\":42,\"string1\":\"test\","
          + "\"boolean1\":true,\"long1\":42432234234}}}";

  /**
   * Test the default behavior of the Lazy family of objects and object inspectors.
   */
  public void testHBaseSerDeI() throws SerDeException {

    byte [] cfa = "cola".getBytes();
    byte [] cfb = "colb".getBytes();
    byte [] cfc = "colc".getBytes();

    byte [] qualByte = "byte".getBytes();
    byte [] qualShort = "short".getBytes();
    byte [] qualInt = "int".getBytes();
    byte [] qualLong = "long".getBytes();
    byte [] qualFloat = "float".getBytes();
    byte [] qualDouble = "double".getBytes();
    byte [] qualString = "string".getBytes();
    byte [] qualBool = "boolean".getBytes();

    byte [] rowKey = Bytes.toBytes("test-row1");

    // Data
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    kvs.add(new KeyValue(rowKey, cfa, qualByte, Bytes.toBytes("123")));
    kvs.add(new KeyValue(rowKey, cfb, qualShort, Bytes.toBytes("456")));
    kvs.add(new KeyValue(rowKey, cfc, qualInt, Bytes.toBytes("789")));
    kvs.add(new KeyValue(rowKey, cfa, qualLong, Bytes.toBytes("1000")));
    kvs.add(new KeyValue(rowKey, cfb, qualFloat, Bytes.toBytes("-0.01")));
    kvs.add(new KeyValue(rowKey, cfc, qualDouble, Bytes.toBytes("5.3")));
    kvs.add(new KeyValue(rowKey, cfa, qualString, Bytes.toBytes("Hadoop, HBase, and Hive")));
    kvs.add(new KeyValue(rowKey, cfb, qualBool, Bytes.toBytes("true")));
    Collections.sort(kvs, KeyValue.COMPARATOR);

    Result r = new Result(kvs);

    Put p = new Put(rowKey);

    p.add(cfa, qualByte, Bytes.toBytes("123"));
    p.add(cfb, qualShort, Bytes.toBytes("456"));
    p.add(cfc, qualInt, Bytes.toBytes("789"));
    p.add(cfa, qualLong, Bytes.toBytes("1000"));
    p.add(cfb, qualFloat, Bytes.toBytes("-0.01"));
    p.add(cfc, qualDouble, Bytes.toBytes("5.3"));
    p.add(cfa, qualString, Bytes.toBytes("Hadoop, HBase, and Hive"));
    p.add(cfb, qualBool, Bytes.toBytes("true"));

    Object[] expectedFieldsData = {
      new Text("test-row1"),
      new ByteWritable((byte)123),
      new ShortWritable((short)456),
      new IntWritable(789),
      new LongWritable(1000),
      new FloatWritable(-0.01F),
      new DoubleWritable(5.3),
      new Text("Hadoop, HBase, and Hive"),
      new BooleanWritable(true)
    };

    // Create, initialize, and test the SerDe
    HBaseSerDe serDe = new HBaseSerDe();
    Configuration conf = new Configuration();
    Properties tbl = createPropertiesI_I();
    SerDeUtils.initializeSerDe(serDe, conf, tbl, null);

    deserializeAndSerialize(serDe, r, p, expectedFieldsData);

    serDe = new HBaseSerDe();
    conf = new Configuration();
    tbl = createPropertiesI_II();
    SerDeUtils.initializeSerDe(serDe, conf, tbl, null);

    deserializeAndSerialize(serDe, r, p, expectedFieldsData);

    serDe = new HBaseSerDe();
    conf = new Configuration();
    tbl = createPropertiesI_III();
    SerDeUtils.initializeSerDe(serDe, conf, tbl, null);

    deserializeAndSerialize(serDe, r, p, expectedFieldsData);

    serDe = new HBaseSerDe();
    conf = new Configuration();
    tbl = createPropertiesI_IV();
    SerDeUtils.initializeSerDe(serDe, conf, tbl, null);

    deserializeAndSerialize(serDe, r, p, expectedFieldsData);
  }

  public void testHBaseSerDeWithTimestamp() throws SerDeException {
    // Create the SerDe
    HBaseSerDe serDe = new HBaseSerDe();
    Configuration conf = new Configuration();
    Properties tbl = createPropertiesI_I();
    long putTimestamp = 1;
    tbl.setProperty(HBaseSerDe.HBASE_PUT_TIMESTAMP,
            Long.toString(putTimestamp));
    SerDeUtils.initializeSerDe(serDe, conf, tbl, null);


    byte [] cfa = "cola".getBytes();
    byte [] cfb = "colb".getBytes();
    byte [] cfc = "colc".getBytes();

    byte [] qualByte = "byte".getBytes();
    byte [] qualShort = "short".getBytes();
    byte [] qualInt = "int".getBytes();
    byte [] qualLong = "long".getBytes();
    byte [] qualFloat = "float".getBytes();
    byte [] qualDouble = "double".getBytes();
    byte [] qualString = "string".getBytes();
    byte [] qualBool = "boolean".getBytes();

    byte [] rowKey = Bytes.toBytes("test-row1");

    // Data
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    kvs.add(new KeyValue(rowKey, cfa, qualByte, Bytes.toBytes("123")));
    kvs.add(new KeyValue(rowKey, cfb, qualShort, Bytes.toBytes("456")));
    kvs.add(new KeyValue(rowKey, cfc, qualInt, Bytes.toBytes("789")));
    kvs.add(new KeyValue(rowKey, cfa, qualLong, Bytes.toBytes("1000")));
    kvs.add(new KeyValue(rowKey, cfb, qualFloat, Bytes.toBytes("-0.01")));
    kvs.add(new KeyValue(rowKey, cfc, qualDouble, Bytes.toBytes("5.3")));
    kvs.add(new KeyValue(rowKey, cfa, qualString, Bytes.toBytes("Hadoop, HBase, and Hive")));
    kvs.add(new KeyValue(rowKey, cfb, qualBool, Bytes.toBytes("true")));
    Collections.sort(kvs, KeyValue.COMPARATOR);

    Result r = new Result(kvs);

    Put p = new Put(rowKey,putTimestamp);

    p.add(cfa, qualByte, Bytes.toBytes("123"));
    p.add(cfb, qualShort, Bytes.toBytes("456"));
    p.add(cfc, qualInt, Bytes.toBytes("789"));
    p.add(cfa, qualLong, Bytes.toBytes("1000"));
    p.add(cfb, qualFloat, Bytes.toBytes("-0.01"));
    p.add(cfc, qualDouble, Bytes.toBytes("5.3"));
    p.add(cfa, qualString, Bytes.toBytes("Hadoop, HBase, and Hive"));
    p.add(cfb, qualBool, Bytes.toBytes("true"));

    Object[] expectedFieldsData = {
      new Text("test-row1"),
      new ByteWritable((byte)123),
      new ShortWritable((short)456),
      new IntWritable(789),
      new LongWritable(1000),
      new FloatWritable(-0.01F),
      new DoubleWritable(5.3),
      new Text("Hadoop, HBase, and Hive"),
      new BooleanWritable(true)
    };

    deserializeAndSerialize(serDe, r, p, expectedFieldsData);
  }

  private void deserializeAndSerialize(
      HBaseSerDe serDe, Result r, Put p,
      Object[] expectedFieldsData) throws SerDeException {

    // Get the row structure
    StructObjectInspector oi = (StructObjectInspector)
      serDe.getObjectInspector();
    List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs();
    assertEquals(9, fieldRefs.size());

    // Deserialize
    Object row = serDe.deserialize(new ResultWritable(r));
    for (int i = 0; i < fieldRefs.size(); i++) {
      Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i));
      if (fieldData != null) {
        fieldData = ((LazyPrimitive<?, ?>)fieldData).getWritableObject();
      }
      assertEquals("Field " + i, expectedFieldsData[i], fieldData);
    }
    // Serialize
    assertEquals(PutWritable.class, serDe.getSerializedClass());
    PutWritable serializedPut = (PutWritable) serDe.serialize(row, oi);
    assertEquals("Serialized data", p.toString(),String.valueOf(serializedPut.getPut()));
  }

  // No specifications default to UTF8 String storage for backwards compatibility
  private Properties createPropertiesI_I() {
    Properties tbl = new Properties();

    // Set the configuration parameters
    tbl.setProperty(serdeConstants.SERIALIZATION_FORMAT, "9");
    tbl.setProperty("columns", "key,abyte,ashort,aint,along,afloat,adouble,astring,abool");
    tbl.setProperty("columns.types",
        "string,tinyint:smallint:int:bigint:float:double:string:boolean");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING,
        "cola:byte,colb:short,colc:int,cola:long,colb:float,colc:double,cola:string,colb:boolean");
    return tbl;
  }

  // Default column storage specification inherits from table level default
  // (in this case a missing specification) of UTF String storage
  private Properties createPropertiesI_II() {
    Properties tbl = new Properties();

    // Set the configuration parameters
    tbl.setProperty(serdeConstants.SERIALIZATION_FORMAT, "9");
    tbl.setProperty("columns", "key,abyte,ashort,aint,along,afloat,adouble,astring,abool");
    tbl.setProperty("columns.types",
        "string,tinyint:smallint:int:bigint:float:double:string:boolean");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING,
        ":key#-,cola:byte#s,colb:short#-,colc:int#s,cola:long#s,colb:float#-,colc:double#-," +
        "cola:string#s,colb:boolean#s");
    return tbl;
  }

  // String storage type overrides table level default of binary storage
  private Properties createPropertiesI_III() {
    Properties tbl = new Properties();

    // Set the configuration parameters
    tbl.setProperty(serdeConstants.SERIALIZATION_FORMAT, "9");
    tbl.setProperty("columns", "key,abyte,ashort,aint,along,afloat,adouble,astring,abool");
    tbl.setProperty("columns.types",
        "string,tinyint:smallint:int:bigint:float:double:string:boolean");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING,
        ":key#s,cola:byte#s,colb:short#s,colc:int#s,cola:long#s,colb:float#s,colc:double#s," +
        "cola:string#s,colb:boolean#s");
    tbl.setProperty(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE, "binary");
    return tbl;
  }

  // String type is never stored as anything other than an escaped string
  // A specification of binary storage should not affect ser/de.
  private Properties createPropertiesI_IV() {
    Properties tbl = new Properties();

    // Set the configuration parameters
    tbl.setProperty(serdeConstants.SERIALIZATION_FORMAT, "9");
    tbl.setProperty("columns", "key,abyte,ashort,aint,along,afloat,adouble,astring,abool");
    tbl.setProperty("columns.types",
        "string,tinyint:smallint:int:bigint:float:double:string:boolean");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING,
        ":key#-,cola:byte#s,colb:short#s,colc:int#s,cola:long#s,colb:float#s,colc:double#s," +
        "cola:string#b,colb:boolean#s");
    tbl.setProperty(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE, "binary");
    return tbl;
  }

  public void testHBaseSerDeII() throws SerDeException {

    byte [] cfa = "cfa".getBytes();
    byte [] cfb = "cfb".getBytes();
    byte [] cfc = "cfc".getBytes();

    byte [] qualByte = "byte".getBytes();
    byte [] qualShort = "short".getBytes();
    byte [] qualInt = "int".getBytes();
    byte [] qualLong = "long".getBytes();
    byte [] qualFloat = "float".getBytes();
    byte [] qualDouble = "double".getBytes();
    byte [] qualString = "string".getBytes();
    byte [] qualBool = "boolean".getBytes();

    byte [] rowKey = Bytes.toBytes("test-row-2");

    // Data
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    kvs.add(new KeyValue(rowKey, cfa, qualByte, new byte [] { Byte.MIN_VALUE }));
    kvs.add(new KeyValue(rowKey, cfb, qualShort, Bytes.toBytes(Short.MIN_VALUE)));
    kvs.add(new KeyValue(rowKey, cfc, qualInt, Bytes.toBytes(Integer.MIN_VALUE)));
    kvs.add(new KeyValue(rowKey, cfa, qualLong, Bytes.toBytes(Long.MIN_VALUE)));
    kvs.add(new KeyValue(rowKey, cfb, qualFloat, Bytes.toBytes(Float.MIN_VALUE)));
    kvs.add(new KeyValue(rowKey, cfc, qualDouble, Bytes.toBytes(Double.MAX_VALUE)));
    kvs.add(new KeyValue(rowKey, cfa, qualString, Bytes.toBytes(
      "Hadoop, HBase, and Hive Again!")));
    kvs.add(new KeyValue(rowKey, cfb, qualBool, Bytes.toBytes(false)));

    Collections.sort(kvs, KeyValue.COMPARATOR);
    Result r = new Result(kvs);

    Put p = new Put(rowKey);

    p.add(cfa, qualByte, new byte [] { Byte.MIN_VALUE });
    p.add(cfb, qualShort, Bytes.toBytes(Short.MIN_VALUE));
    p.add(cfc, qualInt, Bytes.toBytes(Integer.MIN_VALUE));
    p.add(cfa, qualLong, Bytes.toBytes(Long.MIN_VALUE));
    p.add(cfb, qualFloat, Bytes.toBytes(Float.MIN_VALUE));
    p.add(cfc, qualDouble, Bytes.toBytes(Double.MAX_VALUE));
    p.add(cfa, qualString, Bytes.toBytes("Hadoop, HBase, and Hive Again!"));
    p.add(cfb, qualBool, Bytes.toBytes(false));

    Object[] expectedFieldsData = {
      new Text("test-row-2"),
      new ByteWritable(Byte.MIN_VALUE),
      new ShortWritable(Short.MIN_VALUE),
      new IntWritable(Integer.MIN_VALUE),
      new LongWritable(Long.MIN_VALUE),
      new FloatWritable(Float.MIN_VALUE),
      new DoubleWritable(Double.MAX_VALUE),
      new Text("Hadoop, HBase, and Hive Again!"),
      new BooleanWritable(false)
    };

    // Create, initialize, and test the SerDe
    HBaseSerDe serDe = new HBaseSerDe();
    Configuration conf = new Configuration();
    Properties tbl = createPropertiesII_I();
    SerDeUtils.initializeSerDe(serDe, conf, tbl, null);

    deserializeAndSerialize(serDe, r, p, expectedFieldsData);

    serDe = new HBaseSerDe();
    conf = new Configuration();
    tbl = createPropertiesII_II();
    SerDeUtils.initializeSerDe(serDe, conf, tbl, null);

    deserializeAndSerialize(serDe, r, p, expectedFieldsData);

    serDe = new HBaseSerDe();
    conf = new Configuration();
    tbl = createPropertiesII_III();
    SerDeUtils.initializeSerDe(serDe, conf, tbl, null);

    deserializeAndSerialize(serDe, r, p, expectedFieldsData);
  }

  private Properties createPropertiesII_I() {
    Properties tbl = new Properties();

    // Set the configuration parameters
    tbl.setProperty(serdeConstants.SERIALIZATION_FORMAT, "9");
    tbl.setProperty("columns", "key,abyte,ashort,aint,along,afloat,adouble,astring,abool");
    tbl.setProperty("columns.types",
        "string,tinyint:smallint:int:bigint:float:double:string:boolean");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING,
        ":key#-,cfa:byte#b,cfb:short#b,cfc:int#-,cfa:long#b,cfb:float#-,cfc:double#b," +
        "cfa:string#b,cfb:boolean#-");
    tbl.setProperty(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE, "binary");
    return tbl;
  }

  private Properties createPropertiesII_II() {
    Properties tbl = new Properties();

    // Set the configuration parameters
    tbl.setProperty(serdeConstants.SERIALIZATION_FORMAT, "9");
    tbl.setProperty("columns", "key,abyte,ashort,aint,along,afloat,adouble,astring,abool");
    tbl.setProperty("columns.types",
        "string,tinyint:smallint:int:bigint:float:double:string:boolean");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING,
        ":key#b,cfa:byte#b,cfb:short#b,cfc:int#b,cfa:long#b,cfb:float#b,cfc:double#b," +
        "cfa:string#b,cfb:boolean#b");
    tbl.setProperty(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE, "string");
    return tbl;
  }

  private Properties createPropertiesII_III() {
    Properties tbl = new Properties();

    // Set the configuration parameters
    tbl.setProperty(serdeConstants.SERIALIZATION_FORMAT, "9");
    tbl.setProperty("columns", "key,abyte,ashort,aint,along,afloat,adouble,astring,abool");
    tbl.setProperty("columns.types",
        "string,tinyint:smallint:int:bigint:float:double:string:boolean");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING,
        ":key#-,cfa:byte#b,cfb:short#b,cfc:int#b,cfa:long#b,cfb:float#b,cfc:double#b," +
        "cfa:string#-,cfb:boolean#b");
    return tbl;
  }

  public void testHBaseSerDeWithHiveMapToHBaseColumnFamily() throws SerDeException {

    byte [] cfint = "cf-int".getBytes();
    byte [] cfbyte = "cf-byte".getBytes();
    byte [] cfshort = "cf-short".getBytes();
    byte [] cflong = "cf-long".getBytes();
    byte [] cffloat = "cf-float".getBytes();
    byte [] cfdouble = "cf-double".getBytes();
    byte [] cfbool = "cf-bool".getBytes();

    byte [][] columnFamilies =
      new byte [][] {cfint, cfbyte, cfshort, cflong, cffloat, cfdouble, cfbool};

    byte [][] rowKeys = new byte [][] {
        Integer.toString(1).getBytes(),
        Integer.toString(Integer.MIN_VALUE).getBytes(),
        Integer.toString(Integer.MAX_VALUE).getBytes()
    };

    byte [][][] columnQualifiersAndValues = new byte [][][] {
        {Bytes.toBytes(1), new byte [] {1}, Bytes.toBytes((short) 1),
         Bytes.toBytes((long) 1), Bytes.toBytes((float) 1.0F), Bytes.toBytes(1.0),
         Bytes.toBytes(true)},
        {Bytes.toBytes(Integer.MIN_VALUE), new byte [] {Byte.MIN_VALUE},
         Bytes.toBytes((short) Short.MIN_VALUE), Bytes.toBytes((long) Long.MIN_VALUE),
         Bytes.toBytes((float) Float.MIN_VALUE), Bytes.toBytes(Double.MIN_VALUE),
         Bytes.toBytes(false)},
        {Bytes.toBytes(Integer.MAX_VALUE), new byte [] {Byte.MAX_VALUE},
         Bytes.toBytes((short) Short.MAX_VALUE), Bytes.toBytes((long) Long.MAX_VALUE),
         Bytes.toBytes((float) Float.MAX_VALUE), Bytes.toBytes(Double.MAX_VALUE),
         Bytes.toBytes(true)}
    };

    List<KeyValue> kvs = new ArrayList<KeyValue>();
    Result [] r = new Result [] {null, null, null};
    Put [] p = new Put [] {null, null, null};

    for (int i = 0; i < r.length; i++) {
      kvs.clear();
      p[i] = new Put(rowKeys[i]);

      for (int j = 0; j < columnQualifiersAndValues[i].length; j++) {
        kvs.add(new KeyValue(rowKeys[i], columnFamilies[j], columnQualifiersAndValues[i][j],
            columnQualifiersAndValues[i][j]));
        p[i].add(columnFamilies[j], columnQualifiersAndValues[i][j],
            columnQualifiersAndValues[i][j]);
      }

      r[i] = new Result(kvs);
    }

    Object [][] expectedData = {
        {new Text(Integer.toString(1)), new IntWritable(1), new ByteWritable((byte) 1),
         new ShortWritable((short) 1), new LongWritable(1), new FloatWritable(1.0F),
         new DoubleWritable(1.0), new BooleanWritable(true)},
        {new Text(Integer.toString(Integer.MIN_VALUE)), new IntWritable(Integer.MIN_VALUE),
         new ByteWritable(Byte.MIN_VALUE), new ShortWritable(Short.MIN_VALUE),
         new LongWritable(Long.MIN_VALUE), new FloatWritable(Float.MIN_VALUE),
         new DoubleWritable(Double.MIN_VALUE), new BooleanWritable(false)},
        {new Text(Integer.toString(Integer.MAX_VALUE)), new IntWritable(Integer.MAX_VALUE),
         new ByteWritable(Byte.MAX_VALUE), new ShortWritable(Short.MAX_VALUE),
         new LongWritable(Long.MAX_VALUE), new FloatWritable(Float.MAX_VALUE),
         new DoubleWritable(Double.MAX_VALUE), new BooleanWritable(true)}};

    HBaseSerDe hbaseSerDe = new HBaseSerDe();
    Configuration conf = new Configuration();
    Properties tbl = createPropertiesForHiveMapHBaseColumnFamily();
    SerDeUtils.initializeSerDe(hbaseSerDe, conf, tbl, null);

    deserializeAndSerializeHiveMapHBaseColumnFamily(hbaseSerDe, r, p, expectedData, rowKeys,
        columnFamilies, columnQualifiersAndValues);

    hbaseSerDe = new HBaseSerDe();
    conf = new Configuration();
    tbl = createPropertiesForHiveMapHBaseColumnFamilyII();
    SerDeUtils.initializeSerDe(hbaseSerDe, conf, tbl, null);

    deserializeAndSerializeHiveMapHBaseColumnFamily(hbaseSerDe, r, p, expectedData, rowKeys,
        columnFamilies, columnQualifiersAndValues);
  }

  private void deserializeAndSerializeHiveMapHBaseColumnFamily(
      HBaseSerDe hbaseSerDe,
      Result [] r,
      Put [] p,
      Object [][] expectedData,
      byte [][] rowKeys,
      byte [][] columnFamilies,
      byte [][][] columnQualifiersAndValues) throws SerDeException {

    StructObjectInspector soi = (StructObjectInspector) hbaseSerDe.getObjectInspector();
    List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();
    assertEquals(8, fieldRefs.size());

    // Deserialize
    for (int i = 0; i < r.length; i++) {
      Object row = hbaseSerDe.deserialize(new ResultWritable(r[i]));
      Put serializedPut = ((PutWritable) hbaseSerDe.serialize(row, soi)).getPut();
      byte [] rowKey = serializedPut.getRow();

      for (int k = 0; k < rowKey.length; k++) {
        assertEquals(rowKey[k], rowKeys[i][k]);
      }

      assertEquals(columnFamilies.length, serializedPut.numFamilies());

      for (int j = 0; j < fieldRefs.size(); j++) {
        Object fieldData = soi.getStructFieldData(row, fieldRefs.get(j));

        assertNotNull(fieldData);

        if (fieldData instanceof LazyPrimitive<?, ?>) {
          assertEquals(expectedData[i][j],
              ((LazyPrimitive<?, ?>) fieldData).getWritableObject());
        } else if (fieldData instanceof LazyHBaseCellMap) {
          LazyPrimitive<?, ?> lazyPrimitive = (LazyPrimitive<?, ?>)
              ((LazyHBaseCellMap) fieldData).getMapValueElement(expectedData[i][j]);
          assertEquals(expectedData[i][j], lazyPrimitive.getWritableObject());
        } else {
          fail("Error: field data not an instance of LazyPrimitive<?,?> or LazyMap");
        }
      }
    }
  }

  private Properties createPropertiesForHiveMapHBaseColumnFamily() {
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.SERIALIZATION_FORMAT, "9");
    tbl.setProperty(serdeConstants.LIST_COLUMNS,
        "key,valint,valbyte,valshort,vallong,valfloat,valdouble,valbool");
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES,
        "string:map<int,int>:map<tinyint,tinyint>:map<smallint,smallint>:map<bigint,bigint>:"
            + "map<float,float>:map<double,double>:map<boolean,boolean>");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING,
        ":key#-,cf-int:#b:b,cf-byte:#b:b,cf-short:#b:b,cf-long:#b:b,cf-float:#b:b,cf-double:#b:b," +
        "cf-bool:#b:b");
    return tbl;
  }

  private Properties createPropertiesForHiveMapHBaseColumnFamilyII() {
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.SERIALIZATION_FORMAT, "9");
    tbl.setProperty(serdeConstants.LIST_COLUMNS,
        "key,valint,valbyte,valshort,vallong,valfloat,valdouble,valbool");
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES,
        "string:map<int,int>:map<tinyint,tinyint>:map<smallint,smallint>:map<bigint,bigint>:"
            + "map<float,float>:map<double,double>:map<boolean,boolean>");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING,
        ":key#-,cf-int:#-:-,cf-byte:#-:-,cf-short:#-:-,cf-long:#-:-,cf-float:#-:-,cf-double:#-:-," +
        "cf-bool:#-:-");
    tbl.setProperty(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE, "binary");
    return tbl;
  }

  public void testHBaseSerDeWithHiveMapToHBaseColumnFamilyII() throws SerDeException {

    byte [] cfbyte = "cf-byte".getBytes();
    byte [] cfshort = "cf-short".getBytes();
    byte [] cfint = "cf-int".getBytes();
    byte [] cflong = "cf-long".getBytes();
    byte [] cffloat = "cf-float".getBytes();
    byte [] cfdouble = "cf-double".getBytes();
    byte [] cfstring = "cf-string".getBytes();
    byte [] cfbool = "cf-bool".getBytes();

    byte [][] columnFamilies =
      new byte [][] {cfbyte, cfshort, cfint, cflong, cffloat, cfdouble, cfstring, cfbool};

    byte [] rowKey = Bytes.toBytes("row-key");

    byte [][] columnQualifiersAndValues = new byte [][] {
        Bytes.toBytes("123"), Bytes.toBytes("456"), Bytes.toBytes("789"), Bytes.toBytes("1000"),
        Bytes.toBytes("-0.01"), Bytes.toBytes("5.3"), Bytes.toBytes("Hive"),
        Bytes.toBytes("true")
    };

    Put p = new Put(rowKey);
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    for (int j = 0; j < columnQualifiersAndValues.length; j++) {
      kvs.add(new KeyValue(rowKey,
          columnFamilies[j], columnQualifiersAndValues[j], columnQualifiersAndValues[j]));
      p.add(columnFamilies[j], columnQualifiersAndValues[j], columnQualifiersAndValues[j]);
    }

    Result r = new Result(kvs);

    Object [] expectedData = {
        new Text("row-key"), new ByteWritable((byte) 123), new ShortWritable((short) 456),
        new IntWritable(789), new LongWritable(1000), new FloatWritable(-0.01F),
        new DoubleWritable(5.3), new Text("Hive"), new BooleanWritable(true)
    };

    HBaseSerDe hbaseSerDe = new HBaseSerDe();
    Configuration conf = new Configuration();
    Properties tbl = createPropertiesForHiveMapHBaseColumnFamilyII_I();
    SerDeUtils.initializeSerDe(hbaseSerDe, conf, tbl, null);

    deserializeAndSerializeHiveMapHBaseColumnFamilyII(hbaseSerDe, r, p, expectedData,
        columnFamilies, columnQualifiersAndValues);

    hbaseSerDe = new HBaseSerDe();
    conf = new Configuration();
    tbl = createPropertiesForHiveMapHBaseColumnFamilyII_II();
    SerDeUtils.initializeSerDe(hbaseSerDe, conf, tbl, null);

    deserializeAndSerializeHiveMapHBaseColumnFamilyII(hbaseSerDe, r, p, expectedData,
        columnFamilies, columnQualifiersAndValues);
  }

  private Properties createPropertiesForHiveMapHBaseColumnFamilyII_I() {
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.SERIALIZATION_FORMAT, "9");
    tbl.setProperty(serdeConstants.LIST_COLUMNS,
        "key,valbyte,valshort,valint,vallong,valfloat,valdouble,valstring,valbool");
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES,
        "string:map<tinyint,tinyint>:map<smallint,smallint>:map<int,int>:map<bigint,bigint>:"
            + "map<float,float>:map<double,double>:map<string,string>:map<boolean,boolean>");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING,
        ":key#s,cf-byte:#-:s,cf-short:#s:-,cf-int:#s:s,cf-long:#-:-,cf-float:#s:-,cf-double:#-:s," +
        "cf-string:#s:s,cf-bool:#-:-");
    return tbl;
  }

  private Properties createPropertiesForHiveMapHBaseColumnFamilyII_II() {
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.SERIALIZATION_FORMAT, "9");
    tbl.setProperty(serdeConstants.LIST_COLUMNS,
        "key,valbyte,valshort,valint,vallong,valfloat,valdouble,valstring,valbool");
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES,
        "string:map<tinyint,tinyint>:map<smallint,smallint>:map<int,int>:map<bigint,bigint>:"
            + "map<float,float>:map<double,double>:map<string,string>:map<boolean,boolean>");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING,
        ":key#s,cf-byte:#s:s,cf-short:#s:s,cf-int:#s:s,cf-long:#s:s,cf-float:#s:s,cf-double:#s:s," +
        "cf-string:#s:s,cf-bool:#s:s");
    tbl.setProperty(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE, "binary");
    return tbl;
  }

  private void deserializeAndSerializeHiveMapHBaseColumnFamilyII(
      HBaseSerDe hbaseSerDe,
      Result r,
      Put p,
      Object [] expectedData,
      byte [][] columnFamilies,
      byte [][] columnQualifiersAndValues) throws SerDeException {

    StructObjectInspector soi = (StructObjectInspector) hbaseSerDe.getObjectInspector();
    List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();
    assertEquals(9, fieldRefs.size());

    // Deserialize
    Object row = hbaseSerDe.deserialize(new ResultWritable(r));

    for (int j = 0; j < fieldRefs.size(); j++) {
      Object fieldData = soi.getStructFieldData(row, fieldRefs.get(j));
      assertNotNull(fieldData);

      if (fieldData instanceof LazyPrimitive<?, ?>) {
        assertEquals(expectedData[j], ((LazyPrimitive<?, ?>) fieldData).getWritableObject());
      } else if (fieldData instanceof LazyHBaseCellMap) {
        LazyPrimitive<?, ?> lazyPrimitive = (LazyPrimitive<?, ?>)
          ((LazyHBaseCellMap) fieldData).getMapValueElement(expectedData[j]);
        assertEquals(expectedData[j], lazyPrimitive.getWritableObject());
      } else {
        fail("Error: field data not an instance of LazyPrimitive<?, ?> or LazyHBaseCellMap");
      }
    }

    // Serialize
    Put serializedPut = ((PutWritable) hbaseSerDe.serialize(row, soi)).getPut();
    assertEquals("Serialized data: ", p.toString(), serializedPut.toString());
  }

  public void testHBaseSerDeWithColumnPrefixes()
      throws Exception {
    byte[] cfa = "cola".getBytes();

    byte[] qualA = "prefixA_col1".getBytes();
    byte[] qualB = "prefixB_col2".getBytes();
    byte[] qualC = "prefixB_col3".getBytes();
    byte[] qualD = "unwanted_col".getBytes();

    List<Object> qualifiers = new ArrayList<Object>();
    qualifiers.add(new Text("prefixA_col1"));
    qualifiers.add(new Text("prefixB_col2"));
    qualifiers.add(new Text("prefixB_col3"));
    qualifiers.add(new Text("unwanted_col"));

    List<Object> expectedQualifiers = new ArrayList<Object>();
    expectedQualifiers.add(new Text("prefixA_col1"));
    expectedQualifiers.add(new Text("prefixB_col2"));
    expectedQualifiers.add(new Text("prefixB_col3"));

    byte[] rowKey = Bytes.toBytes("test-row1");

    // Data
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    byte[] dataA = "This is first test data".getBytes();
    byte[] dataB = "This is second test data".getBytes();
    byte[] dataC = "This is third test data".getBytes();
    byte[] dataD = "Unwanted data".getBytes();

    kvs.add(new KeyValue(rowKey, cfa, qualA, dataA));
    kvs.add(new KeyValue(rowKey, cfa, qualB, dataB));
    kvs.add(new KeyValue(rowKey, cfa, qualC, dataC));
    kvs.add(new KeyValue(rowKey, cfa, qualD, dataD));

    Result r = new Result(kvs);

    Put p = new Put(rowKey);

    p.add(new KeyValue(rowKey, cfa, qualA, dataA));
    p.add(new KeyValue(rowKey, cfa, qualB, dataB));
    p.add(new KeyValue(rowKey, cfa, qualC, dataC));

    Object[] expectedFieldsData = {
        new Text("test-row1"),
        new String("This is first test data"),
        new String("This is second test data"),
        new String("This is third test data")};

    int[] expectedMapSize = new int[] {1, 2};

    // Create, initialize, and test the SerDe
    HBaseSerDe serDe = new HBaseSerDe();
    Configuration conf = new Configuration();
    Properties tbl = createPropertiesForColumnPrefixes();
    SerDeUtils.initializeSerDe(serDe, conf, tbl, null);

    Object notPresentKey = new Text("unwanted_col");

    deserializeAndSerializeHivePrefixColumnFamily(serDe, r, p, expectedFieldsData, expectedMapSize,
        expectedQualifiers,
        notPresentKey);
  }

  private Properties createPropertiesForColumnPrefixes() {
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.LIST_COLUMNS,
        "key,astring,along");
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES,
        "string:map<string,string>:map<string,string>");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING,
        ":key,cola:prefixA_.*,cola:prefixB_.*");

    return tbl;
  }

  private void deserializeAndSerializeHivePrefixColumnFamily(HBaseSerDe serDe, Result r, Put p,
      Object[] expectedFieldsData, int[] expectedMapSize, List<Object> expectedQualifiers,
      Object notPresentKey)
      throws SerDeException, IOException {
    StructObjectInspector soi = (StructObjectInspector) serDe.getObjectInspector();

    List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();

    Object row = serDe.deserialize(new ResultWritable(r));

    int j = 0;

    for (int i = 0; i < fieldRefs.size(); i++) {
      Object fieldData = soi.getStructFieldData(row, fieldRefs.get(i));
      assertNotNull(fieldData);

      if (fieldData instanceof LazyPrimitive<?, ?>) {
        assertEquals(expectedFieldsData[i], ((LazyPrimitive<?, ?>) fieldData).getWritableObject());
      } else if (fieldData instanceof LazyHBaseCellMap) {
        assertEquals(expectedFieldsData[i], ((LazyHBaseCellMap) fieldData)
            .getMapValueElement(expectedQualifiers.get(j)).toString().trim());

        assertEquals(expectedMapSize[j], ((LazyHBaseCellMap) fieldData).getMapSize());
        // Make sure that the unwanted key is not present in the map
        assertNull(((LazyHBaseCellMap) fieldData).getMapValueElement(notPresentKey));

        j++;

      } else {
        fail("Error: field data not an instance of LazyPrimitive<?, ?> or LazyHBaseCellMap");
      }
    }

    SerDeUtils.getJSONString(row, soi);

    // Now serialize
    Put put = ((PutWritable) serDe.serialize(row, soi)).getPut();

    if (p != null) {
      assertEquals("Serialized put:", p.toString(), put.toString());
    }
  }

  public void testHBaseSerDeCompositeKeyWithSeparator() throws SerDeException, TException,
      IOException {
    byte[] cfa = "cola".getBytes();

    byte[] qualStruct = "struct".getBytes();

    TestStruct testStruct = new TestStruct("A", "B", "C", true, (byte) 45);

    byte[] rowKey = testStruct.getBytes();

    // Data
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    byte[] testData = "This is a test data".getBytes();

    kvs.add(new KeyValue(rowKey, cfa, qualStruct, testData));

    Result r = new Result(kvs);

    Put p = new Put(rowKey);

    // Post serialization, separators are automatically inserted between different fields in the
    // struct. Currently there is not way to disable that. So the work around here is to pad the
    // data with the separator bytes before creating a "Put" object
    p.add(new KeyValue(rowKey, cfa, qualStruct, testData));

    // Create, initialize, and test the SerDe
    HBaseSerDe serDe = new HBaseSerDe();
    Configuration conf = new Configuration();
    Properties tbl = createPropertiesForCompositeKeyWithSeparator();
    SerDeUtils.initializeSerDe(serDe, conf, tbl, null);

    deserializeAndSerializeHBaseCompositeKey(serDe, r, p);
  }

  private Properties createPropertiesForCompositeKeyWithSeparator() {
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.LIST_COLUMNS,
        "key,astring");
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES,
        "struct<col1:string,col2:string,col3:string>,string");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING,
        ":key,cola:struct");
    tbl.setProperty(serdeConstants.COLLECTION_DELIM, "-");

    return tbl;
  }

  public void testHBaseSerDeCompositeKeyWithoutSeparator() throws SerDeException, TException,
      IOException {
    byte[] cfa = "cola".getBytes();

    byte[] qualStruct = "struct".getBytes();

    TestStruct testStruct = new TestStruct("A", "B", "C", false, (byte) 0);

    byte[] rowKey = testStruct.getBytes();

    // Data
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    byte[] testData = "This is a test data".getBytes();

    kvs.add(new KeyValue(rowKey, cfa, qualStruct, testData));

    Result r = new Result(kvs);

    byte[] putRowKey = testStruct.getBytesWithDelimiters();

    Put p = new Put(putRowKey);

    // Post serialization, separators are automatically inserted between different fields in the
    // struct. Currently there is not way to disable that. So the work around here is to pad the
    // data with the separator bytes before creating a "Put" object
    p.add(new KeyValue(putRowKey, cfa, qualStruct, testData));

    // Create, initialize, and test the SerDe
    HBaseSerDe serDe = new HBaseSerDe();
    Configuration conf = new Configuration();
    Properties tbl = createPropertiesForCompositeKeyWithoutSeparator();
    SerDeUtils.initializeSerDe(serDe, conf, tbl, null);

    deserializeAndSerializeHBaseCompositeKey(serDe, r, p);
  }

  private Properties createPropertiesForCompositeKeyWithoutSeparator() {
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.LIST_COLUMNS,
        "key,astring");
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES,
        "struct<col1:string,col2:string,col3:string>,string");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING,
        ":key,cola:struct");
    tbl.setProperty(HBaseSerDe.HBASE_COMPOSITE_KEY_CLASS,
        "org.apache.hadoop.hive.hbase.HBaseTestCompositeKey");

    return tbl;
  }

  private void deserializeAndSerializeHBaseCompositeKey(HBaseSerDe serDe, Result r, Put p)
      throws SerDeException, IOException {
    StructObjectInspector soi = (StructObjectInspector) serDe.getObjectInspector();

    List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();

    Object row = serDe.deserialize(new ResultWritable(r));

    for (int j = 0; j < fieldRefs.size(); j++) {
      Object fieldData = soi.getStructFieldData(row, fieldRefs.get(j));
      assertNotNull(fieldData);
    }

    assertEquals(
        "{\"key\":{\"col1\":\"A\",\"col2\":\"B\",\"col3\":\"C\"},\"astring\":\"This is a test data\"}",
        SerDeUtils.getJSONString(row, soi));

    // Now serialize
    Put put = ((PutWritable) serDe.serialize(row, soi)).getPut();

    assertEquals("Serialized put:", p.toString(), put.toString());
  }

  public void testHBaseSerDeWithAvroSchemaInline() throws SerDeException, IOException {
    byte[] cfa = "cola".getBytes();

    byte[] qualAvro = "avro".getBytes();

    byte[] rowKey = Bytes.toBytes("test-row1");

    // Data
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    byte[] avroData = getTestAvroBytesFromSchema(RECORD_SCHEMA);

    kvs.add(new KeyValue(rowKey, cfa, qualAvro, avroData));

    Result r = new Result(kvs);

    Put p = new Put(rowKey);

    // Post serialization, separators are automatically inserted between different fields in the
    // struct. Currently there is not way to disable that. So the work around here is to pad the
    // data with the separator bytes before creating a "Put" object
    p.add(new KeyValue(rowKey, cfa, qualAvro, avroData));

    Object[] expectedFieldsData = {new String("test-row1"), new String("[[42, true, 42432234234]]")};

    // Create, initialize, and test the SerDe
    HBaseSerDe serDe = new HBaseSerDe();
    Configuration conf = new Configuration();
    Properties tbl = createPropertiesForHiveAvroSchemaInline();
    serDe.initialize(conf, tbl);

    deserializeAndSerializeHiveAvro(serDe, r, p, expectedFieldsData,
        EXPECTED_DESERIALIZED_AVRO_STRING);
  }

  private Properties createPropertiesForHiveAvroSchemaInline() {
    Properties tbl = new Properties();
    tbl.setProperty("cola.avro.serialization.type", "avro");
    tbl.setProperty("cola.avro." + AvroTableProperties.SCHEMA_LITERAL.getPropName(), RECORD_SCHEMA);
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING, ":key,cola:avro");
    tbl.setProperty(HBaseSerDe.HBASE_AUTOGENERATE_STRUCT, "true");

    return tbl;
  }

  public void testHBaseSerDeWithForwardEvolvedSchema() throws SerDeException, IOException {
    byte[] cfa = "cola".getBytes();

    byte[] qualAvro = "avro".getBytes();

    byte[] rowKey = Bytes.toBytes("test-row1");

    // Data
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    byte[] avroData = getTestAvroBytesFromSchema(RECORD_SCHEMA);

    kvs.add(new KeyValue(rowKey, cfa, qualAvro, avroData));

    Result r = new Result(kvs);

    Put p = new Put(rowKey);

    // Post serialization, separators are automatically inserted between different fields in the
    // struct. Currently there is not way to disable that. So the work around here is to pad the
    // data with the separator bytes before creating a "Put" object
    p.add(new KeyValue(rowKey, cfa, qualAvro, avroData));

    Object[] expectedFieldsData = {new String("test-row1"),
        new String("[[42, test, true, 42432234234]]")};

    // Create, initialize, and test the SerDe
    HBaseSerDe serDe = new HBaseSerDe();
    Configuration conf = new Configuration();
    Properties tbl = createPropertiesForHiveAvroForwardEvolvedSchema();
    serDe.initialize(conf, tbl);

    deserializeAndSerializeHiveAvro(serDe, r, p, expectedFieldsData,
        EXPECTED_DESERIALIZED_AVRO_STRING_3);
  }

  private Properties createPropertiesForHiveAvroForwardEvolvedSchema() {
    Properties tbl = new Properties();
    tbl.setProperty("cola.avro.serialization.type", "avro");
    tbl.setProperty("cola.avro." + AvroTableProperties.SCHEMA_LITERAL.getPropName(), RECORD_SCHEMA_EVOLVED);
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING, ":key,cola:avro");
    tbl.setProperty(HBaseSerDe.HBASE_AUTOGENERATE_STRUCT, "true");

    return tbl;
  }

  public void testHBaseSerDeWithBackwardEvolvedSchema() throws SerDeException, IOException {
    byte[] cfa = "cola".getBytes();

    byte[] qualAvro = "avro".getBytes();

    byte[] rowKey = Bytes.toBytes("test-row1");

    // Data
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    byte[] avroData = getTestAvroBytesFromSchema(RECORD_SCHEMA_EVOLVED);

    kvs.add(new KeyValue(rowKey, cfa, qualAvro, avroData));

    Result r = new Result(kvs);

    Put p = new Put(rowKey);

    // Post serialization, separators are automatically inserted between different fields in the
    // struct. Currently there is not way to disable that. So the work around here is to pad the
    // data with the separator bytes before creating a "Put" object
    p.add(new KeyValue(rowKey, cfa, qualAvro, avroData));

    Object[] expectedFieldsData = {new String("test-row1"), new String("[[42, true, 42432234234]]")};

    // Create, initialize, and test the SerDe
    HBaseSerDe serDe = new HBaseSerDe();
    Configuration conf = new Configuration();
    Properties tbl = createPropertiesForHiveAvroBackwardEvolvedSchema();
    serDe.initialize(conf, tbl);

    deserializeAndSerializeHiveAvro(serDe, r, p, expectedFieldsData,
        EXPECTED_DESERIALIZED_AVRO_STRING);
  }

  private Properties createPropertiesForHiveAvroBackwardEvolvedSchema() {
    Properties tbl = new Properties();
    tbl.setProperty("cola.avro.serialization.type", "avro");
    tbl.setProperty("cola.avro." + AvroTableProperties.SCHEMA_LITERAL.getPropName(), RECORD_SCHEMA);
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING, ":key,cola:avro");
    tbl.setProperty(HBaseSerDe.HBASE_AUTOGENERATE_STRUCT, "true");

    return tbl;
  }

  public void testHBaseSerDeWithAvroSerClass() throws SerDeException, IOException {
    byte[] cfa = "cola".getBytes();

    byte[] qualAvro = "avro".getBytes();

    byte[] rowKey = Bytes.toBytes("test-row1");

    // Data
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    byte[] avroData = getTestAvroBytesFromClass1(1);

    kvs.add(new KeyValue(rowKey, cfa, qualAvro, avroData));

    Result r = new Result(kvs);

    Put p = new Put(rowKey);

    // Post serialization, separators are automatically inserted between different fields in the
    // struct. Currently there is not way to disable that. So the work around here is to pad the
    // data with the separator bytes before creating a "Put" object
    p.add(new KeyValue(rowKey, cfa, qualAvro, avroData));

    Object[] expectedFieldsData = {
        new String("test-row1"),
        new String(
            "[Avro Employee1, 11111, 25, FEMALE, [[[Avro First Address1, Avro Second Address1, Avro City1, 123456, 0:[999, 1234567890], null, {testkey=testvalue}], "
                + "[Avro First Address1, Avro Second Address1, Avro City1, 123456, 0:[999, 1234567890], null, {testkey=testvalue}]], "
                + "[999, 1234567890], [999, 1234455555]]]")};

    // Create, initialize, and test the SerDe
    HBaseSerDe serDe = new HBaseSerDe();
    Configuration conf = new Configuration();
    Properties tbl = createPropertiesForHiveAvroSerClass();
    serDe.initialize(conf, tbl);

    deserializeAndSerializeHiveAvro(serDe, r, p, expectedFieldsData,
        EXPECTED_DESERIALIZED_AVRO_STRING_2);
  }

  private Properties createPropertiesForHiveAvroSerClass() {
    Properties tbl = new Properties();
    tbl.setProperty("cola.avro.serialization.type", "avro");
    tbl.setProperty("cola.avro." + serdeConstants.SERIALIZATION_CLASS,
        "org.apache.hadoop.hive.hbase.avro.Employee");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING, ":key,cola:avro");
    tbl.setProperty(HBaseSerDe.HBASE_AUTOGENERATE_STRUCT, "true");

    return tbl;
  }

  public void testHBaseSerDeWithAvroSchemaUrl() throws SerDeException, IOException {
    byte[] cfa = "cola".getBytes();

    byte[] qualAvro = "avro".getBytes();

    byte[] rowKey = Bytes.toBytes("test-row1");

    // Data
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    byte[] avroData = getTestAvroBytesFromSchema(RECORD_SCHEMA);

    kvs.add(new KeyValue(rowKey, cfa, qualAvro, avroData));

    Result r = new Result(kvs);

    Put p = new Put(rowKey);

    // Post serialization, separators are automatically inserted between different fields in the
    // struct. Currently there is not way to disable that. So the work around here is to pad the
    // data with the separator bytes before creating a "Put" object
    p.add(new KeyValue(rowKey, cfa, qualAvro, avroData));

    Object[] expectedFieldsData = {new String("test-row1"), new String("[[42, true, 42432234234]]")};

    MiniDFSCluster miniDfs = null;

    try {
      // MiniDFSCluster litters files and folders all over the place.
      miniDfs = new MiniDFSCluster(new Configuration(), 1, true, null);

      miniDfs.getFileSystem().mkdirs(new Path("/path/to/schema"));
      FSDataOutputStream out = miniDfs.getFileSystem().create(
          new Path("/path/to/schema/schema.avsc"));
      out.writeBytes(RECORD_SCHEMA);
      out.close();
      String onHDFS = miniDfs.getFileSystem().getUri() + "/path/to/schema/schema.avsc";

      // Create, initialize, and test the SerDe
      HBaseSerDe serDe = new HBaseSerDe();
      Configuration conf = new Configuration();
      Properties tbl = createPropertiesForHiveAvroSchemaUrl(onHDFS);
      serDe.initialize(conf, tbl);

      deserializeAndSerializeHiveAvro(serDe, r, p, expectedFieldsData,
          EXPECTED_DESERIALIZED_AVRO_STRING);
    } finally {
      // Teardown the cluster
      if (miniDfs != null) {
        miniDfs.shutdown();
      }
    }
  }

  private Properties createPropertiesForHiveAvroSchemaUrl(String schemaUrl) {
    Properties tbl = new Properties();
    tbl.setProperty("cola.avro.serialization.type", "avro");
    tbl.setProperty("cola.avro." + AvroTableProperties.SCHEMA_URL.getPropName(), schemaUrl);
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING, ":key,cola:avro");
    tbl.setProperty(HBaseSerDe.HBASE_AUTOGENERATE_STRUCT, "true");

    return tbl;
  }

  public void testHBaseSerDeWithAvroExternalSchema() throws SerDeException, IOException {
    byte[] cfa = "cola".getBytes();

    byte[] qualAvro = "avro".getBytes();

    byte[] rowKey = Bytes.toBytes("test-row1");

    // Data
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    byte[] avroData = getTestAvroBytesFromClass2(1);

    kvs.add(new KeyValue(rowKey, cfa, qualAvro, avroData));

    Result r = new Result(kvs);

    Put p = new Put(rowKey);

    // Post serialization, separators are automatically inserted between different fields in the
    // struct. Currently there is not way to disable that. So the work around here is to pad the
    // data with the separator bytes before creating a "Put" object
    p.add(new KeyValue(rowKey, cfa, qualAvro, avroData));

    Object[] expectedFieldsData = {
        new String("test-row1"),
        new String(
            "[Avro Employee1, 11111, 25, FEMALE, [[[Avro First Address1, Avro Second Address1, Avro City1, 123456, 0:[999, 1234567890], null, {testkey=testvalue}], [Avro First Address1, Avro Second Address1, Avro City1, 123456, 0:[999, 1234567890], null, {testkey=testvalue}]], "
                + "[999, 1234567890], [999, 1234455555]]]")};

    // Create, initialize, and test the SerDe
    HBaseSerDe serDe = new HBaseSerDe();
    Configuration conf = new Configuration();

    Properties tbl = createPropertiesForHiveAvroExternalSchema();
    serDe.initialize(conf, tbl);

    deserializeAndSerializeHiveAvro(serDe, r, p, expectedFieldsData,
        EXPECTED_DESERIALIZED_AVRO_STRING_2);
  }

  private Properties createPropertiesForHiveAvroExternalSchema() {
    Properties tbl = new Properties();
    tbl.setProperty("cola.avro.serialization.type", "avro");
    tbl.setProperty(AvroTableProperties.SCHEMA_RETRIEVER.getPropName(),
        "org.apache.hadoop.hive.hbase.HBaseTestAvroSchemaRetriever");
    tbl.setProperty("cola.avro." + serdeConstants.SERIALIZATION_CLASS,
        "org.apache.hadoop.hive.hbase.avro.Employee");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING, ":key,cola:avro");
    tbl.setProperty(HBaseSerDe.HBASE_AUTOGENERATE_STRUCT, "true");

    return tbl;
  }

  public void testHBaseSerDeWithHiveMapToHBaseAvroColumnFamily() throws Exception {
    byte[] cfa = "cola".getBytes();

    byte[] qualAvroA = "prefixA_avro1".getBytes();
    byte[] qualAvroB = "prefixB_avro2".getBytes();
    byte[] qualAvroC = "prefixB_avro3".getBytes();

    List<Object> qualifiers = new ArrayList<Object>();
    qualifiers.add(new Text("prefixA_avro1"));
    qualifiers.add(new Text("prefixB_avro2"));
    qualifiers.add(new Text("prefixB_avro3"));

    List<Object> expectedQualifiers = new ArrayList<Object>();
    expectedQualifiers.add(new Text("prefixB_avro2"));
    expectedQualifiers.add(new Text("prefixB_avro3"));

    byte[] rowKey = Bytes.toBytes("test-row1");

    // Data
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    byte[] avroDataA = getTestAvroBytesFromSchema(RECORD_SCHEMA);
    byte[] avroDataB = getTestAvroBytesFromClass1(1);
    byte[] avroDataC = getTestAvroBytesFromClass1(2);

    kvs.add(new KeyValue(rowKey, cfa, qualAvroA, avroDataA));
    kvs.add(new KeyValue(rowKey, cfa, qualAvroB, avroDataB));
    kvs.add(new KeyValue(rowKey, cfa, qualAvroC, avroDataC));

    Result r = new Result(kvs);

    Put p = new Put(rowKey);

    // Post serialization, separators are automatically inserted between different fields in the
    // struct. Currently there is not way to disable that. So the work around here is to pad the
    // data with the separator bytes before creating a "Put" object
    p.add(new KeyValue(rowKey, cfa, qualAvroB, Bytes.padTail(avroDataB, 11)));
    p.add(new KeyValue(rowKey, cfa, qualAvroC, Bytes.padTail(avroDataC, 11)));

    Object[] expectedFieldsData = {
        new Text("test-row1"),
        new String(
            "[Avro Employee1, 11111, 25, FEMALE, [[[Avro First Address1, Avro Second Address1, Avro City1, 123456, 0:[999, 1234567890], null, {testkey=testvalue}], [Avro First Address1, Avro Second Address1, Avro City1, 123456, 0:[999, 1234567890], null, {testkey=testvalue}]], "
                + "[999, 1234567890], [999, 1234455555]]]"),
        new String(
            "[Avro Employee2, 11111, 25, FEMALE, [[[Avro First Address2, Avro Second Address2, Avro City2, 123456, 0:[999, 1234567890], null, {testkey=testvalue}], [Avro First Address2, Avro Second Address2, Avro City2, 123456, 0:[999, 1234567890], null, {testkey=testvalue}]], "
                + "[999, 1234567890], [999, 1234455555]]]")};

    int[] expectedMapSize = new int[] {2};

    // Create, initialize, and test the SerDe
    HBaseSerDe serDe = new HBaseSerDe();
    Configuration conf = new Configuration();
    Properties tbl = createPropertiesForHiveAvroColumnFamilyMap();
    serDe.initialize(conf, tbl);

    Object notPresentKey = new Text("prefixA_avro1");

    deserializeAndSerializeHiveStructColumnFamily(serDe, r, p, expectedFieldsData, expectedMapSize,
        expectedQualifiers,
        notPresentKey);
  }

  private Properties createPropertiesForHiveAvroColumnFamilyMap() {
    Properties tbl = new Properties();
    tbl.setProperty("cola.prefixB_.serialization.type", "avro");
    tbl.setProperty("cola.prefixB_." + serdeConstants.SERIALIZATION_CLASS,
        "org.apache.hadoop.hive.hbase.avro.Employee");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING, "cola:prefixB_.*");
    tbl.setProperty(HBaseSerDe.HBASE_AUTOGENERATE_STRUCT, "true");
    tbl.setProperty(LazySerDeParameters.SERIALIZATION_EXTEND_NESTING_LEVELS, "true");

    return tbl;
  }

  public void testHBaseSerDeCustomStructValue() throws IOException, SerDeException {

    byte[] cfa = "cola".getBytes();
    byte[] qualStruct = "struct".getBytes();

    TestStruct testStruct = new TestStruct("A", "B", "C", false, (byte) 0);
    byte[] key = testStruct.getBytes();
    // Data
    List<KeyValue> kvs = new ArrayList<KeyValue>();

    byte[] testData = testStruct.getBytes();
    kvs.add(new KeyValue(key, cfa, qualStruct, testData));

    Result r = new Result(kvs);
    byte[] putKey = testStruct.getBytesWithDelimiters();

    Put p = new Put(putKey);

    // Post serialization, separators are automatically inserted between different fields in the
    // struct. Currently there is not way to disable that. So the work around here is to pad the
    // data with the separator bytes before creating a "Put" object
    p.add(new KeyValue(putKey, cfa, qualStruct, Bytes.padTail(testData, 2)));

    // Create, initialize, and test the SerDe
    HBaseSerDe serDe = new HBaseSerDe();
    Configuration conf = new Configuration();
    Properties tbl = createPropertiesForValueStruct();
    serDe.initialize(conf, tbl);

    deserializeAndSerializeHBaseValueStruct(serDe, r, p);

  }

  private Properties createPropertiesForValueStruct() {
    Properties tbl = new Properties();
    tbl.setProperty("cola.struct.serialization.type", "struct");
    tbl.setProperty("cola.struct.test.value", "test value");
    tbl.setProperty(HBaseSerDe.HBASE_STRUCT_SERIALIZER_CLASS,
        "org.apache.hadoop.hive.hbase.HBaseTestStructSerializer");
    tbl.setProperty(serdeConstants.LIST_COLUMNS, "key,astring");
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES,
        "struct<col1:string,col2:string,col3:string>,struct<col1:string,col2:string,col3:string>");
    tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING, ":key,cola:struct");
    tbl.setProperty(HBaseSerDe.HBASE_COMPOSITE_KEY_CLASS,
        "org.apache.hadoop.hive.hbase.HBaseTestCompositeKey");
    return tbl;
  }

  private void deserializeAndSerializeHBaseValueStruct(HBaseSerDe serDe, Result r, Put p)
      throws SerDeException, IOException {
    StructObjectInspector soi = (StructObjectInspector) serDe.getObjectInspector();

    List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();

    Object row = serDe.deserialize(new ResultWritable(r));

    Object fieldData = null;
    for (int j = 0; j < fieldRefs.size(); j++) {
      fieldData = soi.getStructFieldData(row, fieldRefs.get(j));
      assertNotNull(fieldData);
      if (fieldData instanceof LazyStruct) {
        assertEquals(((LazyStruct) fieldData).getField(0).toString(), "A");
        assertEquals(((LazyStruct) fieldData).getField(1).toString(), "B");
        assertEquals(((LazyStruct) fieldData).getField(2).toString(), "C");
      } else {
        Assert.fail("fieldData should be an instance of LazyStruct");
      }
    }

    assertEquals(
        "{\"key\":{\"col1\":\"A\",\"col2\":\"B\",\"col3\":\"C\"},\"astring\":{\"col1\":\"A\",\"col2\":\"B\",\"col3\":\"C\"}}",
        SerDeUtils.getJSONString(row, soi));

    // Now serialize
    Put put = ((PutWritable) serDe.serialize(row, soi)).getPut();

    assertEquals("Serialized put:", p.toString(), put.toString());
  }

  private void deserializeAndSerializeHiveAvro(HBaseSerDe serDe, Result r, Put p,
      Object[] expectedFieldsData, String expectedDeserializedAvroString)
      throws SerDeException, IOException {
    StructObjectInspector soi = (StructObjectInspector) serDe.getObjectInspector();

    List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();

    Object row = serDe.deserialize(new ResultWritable(r));

    for (int j = 0; j < fieldRefs.size(); j++) {
      Object fieldData = soi.getStructFieldData(row, fieldRefs.get(j));
      assertNotNull(fieldData);
      assertEquals(expectedFieldsData[j], fieldData.toString().trim());
    }
    
    assertEquals(expectedDeserializedAvroString, SerDeUtils.getJSONString(row, soi));

    // Now serialize
    Put put = ((PutWritable) serDe.serialize(row, soi)).getPut();

    assertNotNull(put);
    assertEquals(p.getFamilyCellMap(), put.getFamilyCellMap());
  }

  private void deserializeAndSerializeHiveStructColumnFamily(HBaseSerDe serDe, Result r, Put p,
      Object[] expectedFieldsData,
      int[] expectedMapSize, List<Object> expectedQualifiers, Object notPresentKey)
      throws SerDeException, IOException {
    StructObjectInspector soi = (StructObjectInspector) serDe.getObjectInspector();

    List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();

    Object row = serDe.deserialize(new ResultWritable(r));

    int k = 0;

    for (int i = 0; i < fieldRefs.size(); i++) {
      Object fieldData = soi.getStructFieldData(row, fieldRefs.get(i));
      assertNotNull(fieldData);

      if (fieldData instanceof LazyPrimitive<?, ?>) {
        assertEquals(expectedFieldsData[i], ((LazyPrimitive<?, ?>) fieldData).getWritableObject());
      } else if (fieldData instanceof LazyHBaseCellMap) {

        for (int j = 0; j < ((LazyHBaseCellMap) fieldData).getMapSize(); j++) {
          assertEquals(expectedFieldsData[k + 1],
              ((LazyHBaseCellMap) fieldData).getMapValueElement(expectedQualifiers.get(k))
                  .toString().trim());
          k++;
        }

        assertEquals(expectedMapSize[i - 1], ((LazyHBaseCellMap) fieldData).getMapSize());

        // Make sure that the unwanted key is not present in the map
        assertNull(((LazyHBaseCellMap) fieldData).getMapValueElement(notPresentKey));

      } else {
        fail("Error: field data not an instance of LazyPrimitive<?, ?> or LazyHBaseCellMap");
      }
    }

    SerDeUtils.getJSONString(row, soi);

    // Now serialize
    Put put = ((PutWritable) serDe.serialize(row, soi)).getPut();

    assertNotNull(put);
  }

  private byte[] getTestAvroBytesFromSchema(String schemaToUse) throws IOException {
    Schema s = Schema.parse(schemaToUse);
    GenericData.Record record = new GenericData.Record(s);
    GenericData.Record innerRecord = new GenericData.Record(s.getField("aRecord").schema());
    innerRecord.put("int1", 42);
    innerRecord.put("boolean1", true);
    innerRecord.put("long1", 42432234234l);

    if (schemaToUse.equals(RECORD_SCHEMA_EVOLVED)) {
      innerRecord.put("string1", "new value");
    }

    record.put("aRecord", innerRecord);

    DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(s);
    ByteArrayOutputStream out = new ByteArrayOutputStream();

    DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(datumWriter);
    dataFileWriter.create(s, out);
    dataFileWriter.append(record);
    dataFileWriter.close();

    byte[] data = out.toByteArray();

    out.close();
    return data;
  }

  private byte[] getTestAvroBytesFromClass1(int i) throws IOException {
    Employee employee = new Employee();

    employee.setEmployeeName("Avro Employee" + i);
    employee.setEmployeeID(11111L);
    employee.setGender(Gender.FEMALE);
    employee.setAge(25L);

    Address address = new Address();

    address.setAddress1("Avro First Address" + i);
    address.setAddress2("Avro Second Address" + i);
    address.setCity("Avro City" + i);
    address.setZipcode(123456L);

    Map<CharSequence, CharSequence> metadata = new HashMap<CharSequence, CharSequence>();

    metadata.put("testkey", "testvalue");

    address.setMetadata(metadata);

    HomePhone hPhone = new HomePhone();

    hPhone.setAreaCode(999L);
    hPhone.setNumber(1234567890L);

    OfficePhone oPhone = new OfficePhone();

    oPhone.setAreaCode(999L);
    oPhone.setNumber(1234455555L);

    ContactInfo contact = new ContactInfo();

    List<Address> addresses = new ArrayList<Address>();
    address.setCounty(hPhone); // set value for the union type
    addresses.add(address);
    addresses.add(address);

    contact.setAddress(addresses);

    contact.setHomePhone(hPhone);
    contact.setOfficePhone(oPhone);

    employee.setContactInfo(contact);

    DatumWriter<Employee> datumWriter = new SpecificDatumWriter<Employee>(Employee.class);
    DataFileWriter<Employee> dataFileWriter = new DataFileWriter<Employee>(datumWriter);

    ByteArrayOutputStream out = new ByteArrayOutputStream();

    dataFileWriter.create(employee.getSchema(), out);
    dataFileWriter.append(employee);
    dataFileWriter.close();

    return out.toByteArray();
  }

  private byte[] getTestAvroBytesFromClass2(int i) throws IOException {
    Employee employee = new Employee();

    employee.setEmployeeName("Avro Employee" + i);
    employee.setEmployeeID(11111L);
    employee.setGender(Gender.FEMALE);
    employee.setAge(25L);

    Address address = new Address();

    address.setAddress1("Avro First Address" + i);
    address.setAddress2("Avro Second Address" + i);
    address.setCity("Avro City" + i);
    address.setZipcode(123456L);

    Map<CharSequence, CharSequence> metadata = new HashMap<CharSequence, CharSequence>();

    metadata.put("testkey", "testvalue");

    address.setMetadata(metadata);

    HomePhone hPhone = new HomePhone();

    hPhone.setAreaCode(999L);
    hPhone.setNumber(1234567890L);

    OfficePhone oPhone = new OfficePhone();

    oPhone.setAreaCode(999L);
    oPhone.setNumber(1234455555L);

    ContactInfo contact = new ContactInfo();

    List<Address> addresses = new ArrayList<Address>();
    address.setCounty(hPhone); // set value for the union type
    addresses.add(address);
    addresses.add(address);

    contact.setAddress(addresses);

    contact.setHomePhone(hPhone);
    contact.setOfficePhone(oPhone);

    employee.setContactInfo(contact);

    DatumWriter<Employee> employeeWriter = new SpecificDatumWriter<Employee>(Employee.class);

    ByteArrayOutputStream out = new ByteArrayOutputStream();

    Encoder encoder = EncoderFactory.get().binaryEncoder(out, null);

    // write out a header for the payload
    out.write(TEST_BYTE_ARRAY);

    employeeWriter.write(employee, encoder);

    encoder.flush();

    return out.toByteArray();
  }

  class TestStruct {
    String f1;
    String f2;
    String f3;
    boolean hasSeparator;
    byte separator;

    TestStruct(String f1, String f2, String f3, boolean hasSeparator, byte separator) {
      this.f1 = f1;
      this.f2 = f2;
      this.f3 = f3;
      this.hasSeparator = hasSeparator;
      this.separator = separator;
    }

    public byte[] getBytes() throws IOException {
      ByteArrayOutputStream bos = new ByteArrayOutputStream();

      bos.write(f1.getBytes());
      if (hasSeparator) {
        bos.write(separator); // Add field separator
      }
      bos.write(f2.getBytes());
      if (hasSeparator) {
        bos.write(separator);
      }
      bos.write(f3.getBytes());

      return bos.toByteArray();
    }

    public byte[] getBytesWithDelimiters() throws IOException {
      // Add Ctrl-B delimiter between the fields. This is necessary because for structs in case no
      // delimiter is provided, hive automatically adds Ctrl-B as a default delimiter between fields
      ByteArrayOutputStream bos = new ByteArrayOutputStream();

      bos.write(f1.getBytes());
      bos.write("\002".getBytes("UTF8"));
      bos.write(f2.getBytes());
      bos.write("\002".getBytes("UTF8"));
      bos.write(f3.getBytes());

      return bos.toByteArray();
    }
  }
}
