/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hive.serde2.lazybinary;

import java.sql.Date;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Random;

import junit.framework.TestCase;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.binarysortable.MyTestClass;
import org.apache.hadoop.hive.serde2.binarysortable.MyTestInnerStruct;
import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass;
import org.apache.hadoop.hive.serde2.binarysortable.TestBinarySortableSerDe;
import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
import org.apache.hadoop.hive.serde2.lazy.LazyBinary;
import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.AbstractPrimitiveLazyObjectInspector;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryMapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaBinaryObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.BytesWritable;

/**
 * TestLazyBinarySerDe.
 *
 */
public class TestLazyBinarySerDe extends TestCase {

  /**
   * Generate a random struct array.
   *
   * @param r
   *          random number generator
   * @return an struct array
   */
  static List<MyTestInnerStruct> getRandStructArray(Random r) {
    int length = r.nextInt(10);
    ArrayList<MyTestInnerStruct> result = new ArrayList<MyTestInnerStruct>(
        length);
    for (int i = 0; i < length; i++) {
      MyTestInnerStruct ti = new MyTestInnerStruct(r.nextInt(), r.nextInt());
      result.add(ti);
    }
    return result;
  }

  /**
   * Initialize the LazyBinarySerDe.
   *
   * @param fieldNames
   *          table field names
   * @param fieldTypes
   *          table field types
   * @return the initialized LazyBinarySerDe
   * @throws Throwable
   */
  protected static AbstractSerDe getSerDe(String fieldNames, String fieldTypes) throws Throwable {
    Properties schema = new Properties();
    schema.setProperty(serdeConstants.LIST_COLUMNS, fieldNames);
    schema.setProperty(serdeConstants.LIST_COLUMN_TYPES, fieldTypes);

    LazyBinarySerDe serde = new LazyBinarySerDe();
    SerDeUtils.initializeSerDe(serde, new Configuration(), schema, null);
    return serde;
  }

  /**
   * Test the LazyBinarySerDe.
   *
   * @param rows
   *          array of structs to be serialized
   * @param rowOI
   *          array of struct object inspectors
   * @param serde
   *          the serde
   * @throws Throwable
   */
  private void testLazyBinarySerDe(Object[] rows, ObjectInspector rowOI,
      AbstractSerDe serde) throws Throwable {

    ObjectInspector serdeOI = serde.getObjectInspector();

    // Try to serialize
    BytesWritable bytes[] = new BytesWritable[rows.length];
    for (int i = 0; i < rows.length; i++) {
      BytesWritable s = (BytesWritable) serde.serialize(rows[i], rowOI);
      bytes[i] = new BytesWritable();
      bytes[i].set(s);
    }

    // Try to deserialize
    Object[] deserialized = new Object[rows.length];
    for (int i = 0; i < rows.length; i++) {
      deserialized[i] = serde.deserialize(bytes[i]);
      if (0 != ObjectInspectorUtils.compare(rows[i], rowOI, deserialized[i],
          serdeOI)) {
        System.out.println("structs[" + i + "] = "
            + SerDeUtils.getJSONString(rows[i], rowOI));
        System.out.println("deserialized[" + i + "] = "
            + SerDeUtils.getJSONString(deserialized[i], serdeOI));
        System.out.println("serialized[" + i + "] = "
            + TestBinarySortableSerDe.hexString(bytes[i]));
        assertEquals(rows[i], deserialized[i]);
      }
    }
  }

  /**
   * Compare two structs that have different number of fields. We just compare
   * the first few common fields, ignoring the fields existing in one struct but
   * not the other.
   *
   * @see ObjectInspectorUtils#compare(Object, ObjectInspector, Object,
   *      ObjectInspector)
   */
  int compareDiffSizedStructs(Object o1, ObjectInspector oi1, Object o2,
      ObjectInspector oi2) {
    StructObjectInspector soi1 = (StructObjectInspector) oi1;
    StructObjectInspector soi2 = (StructObjectInspector) oi2;
    List<? extends StructField> fields1 = soi1.getAllStructFieldRefs();
    List<? extends StructField> fields2 = soi2.getAllStructFieldRefs();
    int minimum = Math.min(fields1.size(), fields2.size());
    for (int i = 0; i < minimum; i++) {
      int result = ObjectInspectorUtils.compare(soi1.getStructFieldData(o1,
          fields1.get(i)), fields1.get(i).getFieldObjectInspector(), soi2
          .getStructFieldData(o2, fields2.get(i)), fields2.get(i)
          .getFieldObjectInspector());
      if (result != 0) {
        return result;
      }
    }
    return 0;
  }

  /**
   * Test shorter schema deserialization where a bigger struct is serialized and
   * it is then deserialized with a smaller struct. Here the serialized struct
   * has 10 fields and we deserialized to a struct of 9 fields.
   */
  private void testShorterSchemaDeserialization(Random r) throws Throwable {

    StructObjectInspector rowOI1 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClassBigger.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames1 = ObjectInspectorUtils.getFieldNames(rowOI1);
    String fieldTypes1 = ObjectInspectorUtils.getFieldTypes(rowOI1);
    AbstractSerDe serde1 = getSerDe(fieldNames1, fieldTypes1);
    serde1.getObjectInspector();

    StructObjectInspector rowOI2 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClass.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames2 = ObjectInspectorUtils.getFieldNames(rowOI2);
    String fieldTypes2 = ObjectInspectorUtils.getFieldTypes(rowOI2);
    AbstractSerDe serde2 = getSerDe(fieldNames2, fieldTypes2);
    ObjectInspector serdeOI2 = serde2.getObjectInspector();

    int num = 100;
    for (int itest = 0; itest < num; itest++) {
      MyTestClassBigger t = new MyTestClassBigger();
      ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
      t.randomFill(r, extraTypeInfo);
      BytesWritable bw = (BytesWritable) serde1.serialize(t, rowOI1);
      Object output = serde2.deserialize(bw);

      if (0 != compareDiffSizedStructs(t, rowOI1, output, serdeOI2)) {
        System.out.println("structs      = "
            + SerDeUtils.getJSONString(t, rowOI1));
        System.out.println("deserialized = "
            + SerDeUtils.getJSONString(output, serdeOI2));
        System.out.println("serialized   = "
            + TestBinarySortableSerDe.hexString(bw));
        assertEquals(t, output);
      }
    }
  }

  /**
   * Test shorter schema deserialization where a bigger struct is serialized and
   * it is then deserialized with a smaller struct. Here the serialized struct
   * has 9 fields and we deserialized to a struct of 8 fields.
   */
  private void testShorterSchemaDeserialization1(Random r) throws Throwable {

    StructObjectInspector rowOI1 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClass.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames1 = ObjectInspectorUtils.getFieldNames(rowOI1);
    String fieldTypes1 = ObjectInspectorUtils.getFieldTypes(rowOI1);
    AbstractSerDe serde1 = getSerDe(fieldNames1, fieldTypes1);
    serde1.getObjectInspector();

    StructObjectInspector rowOI2 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClassSmaller.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames2 = ObjectInspectorUtils.getFieldNames(rowOI2);
    String fieldTypes2 = ObjectInspectorUtils.getFieldTypes(rowOI2);
    AbstractSerDe serde2 = getSerDe(fieldNames2, fieldTypes2);
    ObjectInspector serdeOI2 = serde2.getObjectInspector();

    int num = 100;
    for (int itest = 0; itest < num; itest++) {
      MyTestClass t = new MyTestClass();
      ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
      t.randomFill(r, extraTypeInfo);
      BytesWritable bw = (BytesWritable) serde1.serialize(t, rowOI1);
      Object output = serde2.deserialize(bw);

      if (0 != compareDiffSizedStructs(t, rowOI1, output, serdeOI2)) {
        System.out.println("structs      = "
            + SerDeUtils.getJSONString(t, rowOI1));
        System.out.println("deserialized = "
            + SerDeUtils.getJSONString(output, serdeOI2));
        System.out.println("serialized   = "
            + TestBinarySortableSerDe.hexString(bw));
        assertEquals(t, output);
      }
    }
  }

  /**
   * Test longer schema deserialization where a smaller struct is serialized and
   * it is then deserialized with a bigger struct Here the serialized struct has
   * 9 fields and we deserialized to a struct of 10 fields.
   */
  void testLongerSchemaDeserialization(Random r) throws Throwable {

    StructObjectInspector rowOI1 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClass.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames1 = ObjectInspectorUtils.getFieldNames(rowOI1);
    String fieldTypes1 = ObjectInspectorUtils.getFieldTypes(rowOI1);
    AbstractSerDe serde1 = getSerDe(fieldNames1, fieldTypes1);
    serde1.getObjectInspector();

    StructObjectInspector rowOI2 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClassBigger.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames2 = ObjectInspectorUtils.getFieldNames(rowOI2);
    String fieldTypes2 = ObjectInspectorUtils.getFieldTypes(rowOI2);
    AbstractSerDe serde2 = getSerDe(fieldNames2, fieldTypes2);
    ObjectInspector serdeOI2 = serde2.getObjectInspector();

    int num = 100;
    for (int itest = 0; itest < num; itest++) {
      MyTestClass t = new MyTestClass();
      ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
      t.randomFill(r, extraTypeInfo);

      BytesWritable bw = (BytesWritable) serde1.serialize(t, rowOI1);
      Object output = serde2.deserialize(bw);

      if (0 != compareDiffSizedStructs(t, rowOI1, output, serdeOI2)) {
        System.out.println("structs      = "
            + SerDeUtils.getJSONString(t, rowOI1));
        System.out.println("deserialized = "
            + SerDeUtils.getJSONString(output, serdeOI2));
        System.out.println("serialized   = "
            + TestBinarySortableSerDe.hexString(bw));
        assertEquals(t, output);
      }
    }
  }

  /**
   * Test longer schema deserialization where a smaller struct is serialized and
   * it is then deserialized with a bigger struct Here the serialized struct has
   * 8 fields and we deserialized to a struct of 9 fields.
   */
  void testLongerSchemaDeserialization1(Random r) throws Throwable {

    StructObjectInspector rowOI1 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClassSmaller.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames1 = ObjectInspectorUtils.getFieldNames(rowOI1);
    String fieldTypes1 = ObjectInspectorUtils.getFieldTypes(rowOI1);
    AbstractSerDe serde1 = getSerDe(fieldNames1, fieldTypes1);
    serde1.getObjectInspector();

    StructObjectInspector rowOI2 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClass.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames2 = ObjectInspectorUtils.getFieldNames(rowOI2);
    String fieldTypes2 = ObjectInspectorUtils.getFieldTypes(rowOI2);
    AbstractSerDe serde2 = getSerDe(fieldNames2, fieldTypes2);
    ObjectInspector serdeOI2 = serde2.getObjectInspector();

    int num = 100;
    for (int itest = 0; itest < num; itest++) {
      MyTestClassSmaller t = new MyTestClassSmaller();
      ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
      t.randomFill(r, extraTypeInfo);
      BytesWritable bw = (BytesWritable) serde1.serialize(t, rowOI1);
      Object output = serde2.deserialize(bw);

      if (0 != compareDiffSizedStructs(t, rowOI1, output, serdeOI2)) {
        System.out.println("structs      = "
            + SerDeUtils.getJSONString(t, rowOI1));
        System.out.println("deserialized = "
            + SerDeUtils.getJSONString(output, serdeOI2));
        System.out.println("serialized   = "
            + TestBinarySortableSerDe.hexString(bw));
        assertEquals(t, output);
      }
    }
  }

  void testLazyBinaryMap(Random r) throws Throwable {

    StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClassBigger.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
    String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
    AbstractSerDe serde = getSerDe(fieldNames, fieldTypes);
    ObjectInspector serdeOI = serde.getObjectInspector();

    StructObjectInspector soi1 = (StructObjectInspector) serdeOI;
    List<? extends StructField> fields1 = soi1.getAllStructFieldRefs();
    LazyBinaryMapObjectInspector lazympoi = (LazyBinaryMapObjectInspector) fields1
        .get(MyTestClassBigger.mapPos).getFieldObjectInspector();
    ObjectInspector lazympkeyoi = lazympoi.getMapKeyObjectInspector();
    ObjectInspector lazympvalueoi = lazympoi.getMapValueObjectInspector();

    StructObjectInspector soi2 = rowOI;
    List<? extends StructField> fields2 = soi2.getAllStructFieldRefs();
    MapObjectInspector inputmpoi = (MapObjectInspector) fields2.get(MyTestClassBigger.mapPos)
        .getFieldObjectInspector();
    ObjectInspector inputmpkeyoi = inputmpoi.getMapKeyObjectInspector();
    ObjectInspector inputmpvalueoi = inputmpoi.getMapValueObjectInspector();

    int num = 100;
    for (int testi = 0; testi < num; testi++) {

      Map<String, List<MyTestInnerStruct>> mp = new LinkedHashMap<String, List<MyTestInnerStruct>>();

      int randFields = r.nextInt(10);
      for (int i = 0; i < randFields; i++) {
        String key = MyTestPrimitiveClass.getRandString(r);
        int randField = r.nextInt(10);
        List<MyTestInnerStruct> value = randField > 4 ? null
            : getRandStructArray(r);
        mp.put(key, value);

      }

      MyTestClassBigger t = new MyTestClassBigger();
      t.myMap = mp;
      BytesWritable bw = (BytesWritable) serde.serialize(t, rowOI);
      Object output = serde.deserialize(bw);
      Object lazyobj = soi1.getStructFieldData(output, fields1.get(MyTestClassBigger.mapPos));
      Map<?, ?> outputmp = lazympoi.getMap(lazyobj);

      if (outputmp.size() != mp.size()) {
        throw new RuntimeException("Map size changed from " + mp.size()
            + " to " + outputmp.size() + " after serialization!");
      }

      for (Map.Entry<?, ?> entryinput : mp.entrySet()) {
        boolean bEqual = false;
        for (Map.Entry<?, ?> entryoutput : outputmp.entrySet()) {
          // find the same key
          if (0 == ObjectInspectorUtils.compare(entryoutput.getKey(),
              lazympkeyoi, entryinput.getKey(), inputmpkeyoi)) {
            if (0 != ObjectInspectorUtils.compare(entryoutput.getValue(),
                lazympvalueoi, entryinput.getValue(), inputmpvalueoi)) {
              assertEquals(entryoutput.getValue(), entryinput.getValue());
            } else {
              bEqual = true;
            }
            break;
          }
        }
        if (!bEqual) {
          throw new RuntimeException(
              "Could not find matched key in deserialized map : "
              + entryinput.getKey());
        }
      }
    }
  }

  /**
   * The test entrance function.
   *
   * @throws Throwable
   */
  public void testLazyBinarySerDe() throws Throwable {
    try {

      System.out.println("Beginning Test TestLazyBinarySerDe:");

      // generate the data
      int num = 1000;
      Random r = new Random(1234);
      MyTestClass rows[] = new MyTestClass[num];
      for (int i = 0; i < num; i++) {
        MyTestClass t = new MyTestClass();
        ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
        t.randomFill(r, extraTypeInfo);
        rows[i] = t;
      }

      StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory
          .getReflectionObjectInspector(MyTestClass.class,
          ObjectInspectorOptions.JAVA);

      String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
      String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);

      // call the tests
      // 1/ test LazyBinarySerDe
      testLazyBinarySerDe(rows, rowOI, getSerDe(fieldNames, fieldTypes));
      // 2/ test LazyBinaryMap
      testLazyBinaryMap(r);
      // 3/ test serialization and deserialization with different schemas
      testShorterSchemaDeserialization(r);
      // 4/ test serialization and deserialization with different schemas
      testLongerSchemaDeserialization(r);
      // 5/ test serialization and deserialization with different schemas
      testShorterSchemaDeserialization1(r);
      // 6/ test serialization and deserialization with different schemas
      testLongerSchemaDeserialization1(r);

      System.out.println("Test TestLazyBinarySerDe passed!");
    } catch (Throwable e) {
      e.printStackTrace();
      throw e;
    }
  }

  private final  byte[] inpBArray = {'1','\u0001','3','4'};
  private BytesWritable getInputBytesWritable() {
    //create input BytesWritable. This would have capacity greater than length)
    BytesWritable bW = new BytesWritable();
    bW.set(inpBArray, 0, inpBArray.length);
    return bW;
  }

  /**
   * Test to see if byte[] with correct contents is generated by
   * JavaBinaryObjectInspector from input BytesWritable
   * @throws Throwable
   */
  public void testJavaBinaryObjectInspector() throws Throwable {
    BytesWritable bW = getInputBytesWritable();

    //create JavaBinaryObjectInspector
    JavaBinaryObjectInspector binInspector =
        PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector;

    //convert BytesWritable to byte][
    byte[] outBARef = binInspector.set(null, bW);

    assertTrue("compare input and output BAs",
        Arrays.equals(inpBArray, outBARef));
  }


  /**
   * Test to see if byte[] with correct contents is generated by
   * WritableBinaryObjectInspector from input BytesWritable
   * @throws Throwable
   */
  public void testWritableBinaryObjectInspector() throws Throwable {
    BytesWritable bW = getInputBytesWritable();

    //test WritableBinaryObjectInspector
    WritableBinaryObjectInspector writableBinInsp =
        PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;

    //convert BytesWritable to byte[]
    byte[] outBARef = writableBinInsp.getPrimitiveJavaObject(bW);

    assertTrue("compare input and output BAs",
        Arrays.equals(inpBArray, outBARef));
  }

  /**
   * Test to see if byte[] with correct contents is generated by
   * LazyBinaryObjectInspector from input BytesWritable
   * @throws Throwable
   */
  public void testLazyBinaryObjectInspector() throws Throwable {

    //create input ByteArrayRef
    ByteArrayRef inpBARef = new ByteArrayRef();
    inpBARef.setData(inpBArray);

    AbstractPrimitiveLazyObjectInspector<?> binInspector = LazyPrimitiveObjectInspectorFactory
    .getLazyObjectInspector(TypeInfoFactory.binaryTypeInfo, false, (byte)0);

    //create LazyBinary initialed with inputBA
    LazyBinary lazyBin = (LazyBinary) LazyFactory.createLazyObject(binInspector);
    lazyBin.init(inpBARef, 0, inpBArray.length);

    //use inspector to get a byte[] out of LazyBinary
    byte[] outBARef = (byte[]) binInspector.getPrimitiveJavaObject(lazyBin);

    assertTrue("compare input and output BAs",
        Arrays.equals(inpBArray, outBARef));

  }

}
