/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.hive.hcatalog.pig;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.io.IOConstants;
import org.apache.hadoop.hive.ql.io.StorageFormats;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;

import org.apache.hadoop.util.Shell;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.executionengine.ExecJob;
import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;

import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assume.assumeTrue;

@RunWith(Parameterized.class)
public class TestHCatLoaderComplexSchema {

  //private static MiniCluster cluster = MiniCluster.buildCluster();
  private static Driver driver;
  //private static Properties props;
  private static final Logger LOG = LoggerFactory.getLogger(TestHCatLoaderComplexSchema.class);

  private static final Map<String, Set<String>> DISABLED_STORAGE_FORMATS =
      new HashMap<String, Set<String>>() {{
        put(IOConstants.AVRO, new HashSet<String>() {{
          add("testMapNullKey");
        }});
        put(IOConstants.PARQUETFILE, new HashSet<String>() {{
          add("testSyntheticComplexSchema");
          add("testTupleInBagInTupleInBag");
          add("testMapWithComplexData");
          add("testMapNullKey");
        }});
      }};

  private String storageFormat;

  @Parameterized.Parameters
  public static Collection<Object[]> generateParameters() {
    return StorageFormats.names();
  }

  public TestHCatLoaderComplexSchema(String storageFormat) {
    this.storageFormat = storageFormat;
  }

  private void dropTable(String tablename) throws IOException, CommandNeedRetryException {
    driver.run("drop table " + tablename);
  }

  private void createTable(String tablename, String schema, String partitionedBy) throws IOException, CommandNeedRetryException {
    String createTable;
    createTable = "create table " + tablename + "(" + schema + ") ";
    if ((partitionedBy != null) && (!partitionedBy.trim().isEmpty())) {
      createTable = createTable + "partitioned by (" + partitionedBy + ") ";
    }
    createTable = createTable + "stored as " + storageFormat;
    LOG.info("Creating table:\n {}", createTable);
    CommandProcessorResponse result = driver.run(createTable);
    int retCode = result.getResponseCode();
    if (retCode != 0) {
      throw new IOException("Failed to create table. [" + createTable + "], return code from hive driver : [" + retCode + " " + result.getErrorMessage() + "]");
    }
  }

  private void createTable(String tablename, String schema) throws IOException, CommandNeedRetryException {
    createTable(tablename, schema, null);
  }

  @BeforeClass
  public static void setUpBeforeClass() throws Exception {
    HiveConf hiveConf = new HiveConf(TestHCatLoaderComplexSchema.class);
    hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
    hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
    hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
    hiveConf
    .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
        "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");

    driver = new Driver(hiveConf);
    SessionState.start(new CliSessionState(hiveConf));
    //props = new Properties();
    //props.setProperty("fs.default.name", cluster.getProperties().getProperty("fs.default.name"));
  }

  private static final TupleFactory tf = TupleFactory.getInstance();
  private static final BagFactory bf = BagFactory.getInstance();

  private Tuple t(Object... objects) {
    return tf.newTuple(Arrays.asList(objects));
  }

  private DataBag b(Tuple... objects) {
    return bf.newDefaultBag(Arrays.asList(objects));
  }

  /**
   * artificially complex nested schema to test nested schema conversion
   * @throws Exception
   */
  @Test
  public void testSyntheticComplexSchema() throws Exception {
    assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS));
    String pigSchema =
        "a: " +
        "(" +
        "aa: chararray, " +
        "ab: long, " +
        "ac: map[], " +
        "ad: { t: (ada: long) }, " +
        "ae: { t: (aea:long, aeb: ( aeba: chararray, aebb: long)) }," +
        "af: (afa: chararray, afb: long) " +
        ")," +
        "b: chararray, " +
        "c: long, " +
        "d:  { t: (da:long, db: ( dba: chararray, dbb: long), dc: { t: (dca: long) } ) } ";

    // with extra structs
    String tableSchema =
      "a struct<" +
        "aa: string, " +
        "ab: bigint, " +
        "ac: map<string, string>, " +
        "ad: array<struct<ada:bigint>>, " +
        "ae: array<struct<aea:bigint, aeb: struct<aeba: string, aebb: bigint>>>," +
        "af: struct<afa: string, afb: bigint> " +
        ">, " +
        "b string, " +
        "c bigint, " +
        "d array<struct<da: bigint, db: struct<dba:string, dbb:bigint>, dc: array<struct<dca: bigint>>>>";

    // without extra structs
    String tableSchema2 =
      "a struct<" +
        "aa: string, " +
        "ab: bigint, " +
        "ac: map<string, string>, " +
        "ad: array<bigint>, " +
        "ae: array<struct<aea:bigint, aeb: struct<aeba: string, aebb: bigint>>>," +
        "af: struct<afa: string, afb: bigint> " +
        ">, " +
        "b string, " +
        "c bigint, " +
        "d array<struct<da: bigint, db: struct<dba:string, dbb:bigint>, dc: array<bigint>>>";

    List<Tuple> data = new ArrayList<Tuple>();
    for (int i = 0; i < 10; i++) {
      Tuple t = t(
        t(
          "aa test",
          2l,
          new HashMap<String, String>() {
            {
              put("ac test1", "test 1");
              put("ac test2", "test 2");
            }
          },
          b(t(3l), t(4l)),
          b(t(5l, t("aeba test", 6l))),
          t("afa test", 7l)
        ),
        "b test",
        (long) i,
        b(t(8l, t("dba test", 9l), b(t(10l)))));

      data.add(t);
    }
    verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, true);
    verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, false);
    verifyWriteRead("testSyntheticComplexSchema2", pigSchema, tableSchema2, data, true);
    verifyWriteRead("testSyntheticComplexSchema2", pigSchema, tableSchema2, data, false);
  }

  private void verifyWriteRead(String tablename, String pigSchema, String tableSchema, List<Tuple> data, boolean provideSchemaToStorer)
    throws IOException, CommandNeedRetryException, ExecException, FrontendException {
    verifyWriteRead(tablename, pigSchema, tableSchema, data, data, provideSchemaToStorer);
  }
  private void verifyWriteRead(String tablename, String pigSchema, String tableSchema, List<Tuple> data, List<Tuple> result, boolean provideSchemaToStorer)
    throws IOException, CommandNeedRetryException, ExecException, FrontendException {
    MockLoader.setData(tablename + "Input", data);
    try {
      createTable(tablename, tableSchema);
      PigServer server = new PigServer(ExecType.LOCAL);
      server.setBatchOn();
      server.registerQuery("A = load '" + tablename + "Input' using org.apache.hive.hcatalog.pig.MockLoader() AS (" + pigSchema + ");");
      Schema dumpedASchema = server.dumpSchema("A");
      server.registerQuery("STORE A into '" + tablename + "' using org.apache.hive.hcatalog.pig.HCatStorer("
        + (provideSchemaToStorer ? "'', '" + pigSchema + "'" : "")
        + ");");

      ExecJob execJob = server.executeBatch().get(0);
      if (!execJob.getStatistics().isSuccessful()) {
        throw new RuntimeException("Import failed", execJob.getException());
      }
      // test that schema was loaded correctly
      server.registerQuery("X = load '" + tablename + "' using org.apache.hive.hcatalog.pig.HCatLoader();");
      server.dumpSchema("X");
      Iterator<Tuple> it = server.openIterator("X");
      int i = 0;
      while (it.hasNext()) {
        Tuple input = result.get(i++);
        Tuple output = it.next();
        compareTuples(input, output);
        LOG.info("tuple : {} ", output);
      }
      Schema dumpedXSchema = server.dumpSchema("X");

      assertEquals(
        "expected " + dumpedASchema + " but was " + dumpedXSchema + " (ignoring field names)",
        "",
        compareIgnoreFiledNames(dumpedASchema, dumpedXSchema));

    } finally {
      dropTable(tablename);
    }
  }

  private void compareTuples(Tuple t1, Tuple t2) throws ExecException {
    assertEquals("Tuple Sizes don't match", t1.size(), t2.size());
    for (int i = 0; i < t1.size(); i++) {
      Object f1 = t1.get(i);
      Object f2 = t2.get(i);
      assertNotNull("left", f1);
      assertNotNull("right", f2);
      String msg = "right: " + f1 + ", left: " + f2;
      assertEquals(msg, noOrder(f1.toString()), noOrder(f2.toString()));
    }
  }

  private String noOrder(String s) {
    char[] chars = s.toCharArray();
    Arrays.sort(chars);
    return new String(chars);
  }

  private String compareIgnoreFiledNames(Schema expected, Schema got) throws FrontendException {
    if (expected == null || got == null) {
      if (expected == got) {
        return "";
      } else {
        return "\nexpected " + expected + " got " + got;
      }
    }
    if (expected.size() != got.size()) {
      return "\nsize expected " + expected.size() + " (" + expected + ") got " + got.size() + " (" + got + ")";
    }
    String message = "";
    for (int i = 0; i < expected.size(); i++) {
      FieldSchema expectedField = expected.getField(i);
      FieldSchema gotField = got.getField(i);
      if (expectedField.type != gotField.type) {
        message += "\ntype expected " + expectedField.type + " (" + expectedField + ") got " + gotField.type + " (" + gotField + ")";
      } else {
        message += compareIgnoreFiledNames(expectedField.schema, gotField.schema);
      }
    }
    return message;
  }

  /**
   * tests that unnecessary tuples are drop while converting schema
   * (Pig requires Tuples in Bags)
   * @throws Exception
   */
  @Test
  public void testTupleInBagInTupleInBag() throws Exception {
    assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS));
    String pigSchema = "a: { b : ( c: { d: (i : long) } ) }";

    String tableSchema = "a array< array< bigint > >";

    List<Tuple> data = new ArrayList<Tuple>();
    data.add(t(b(t(b(t(100l), t(101l))), t(b(t(110l))))));
    data.add(t(b(t(b(t(200l))), t(b(t(210l))), t(b(t(220l))))));
    data.add(t(b(t(b(t(300l), t(301l))))));
    data.add(t(b(t(b(t(400l))), t(b(t(410l), t(411l), t(412l))))));


    verifyWriteRead("TupleInBagInTupleInBag1", pigSchema, tableSchema, data, true);
    verifyWriteRead("TupleInBagInTupleInBag2", pigSchema, tableSchema, data, false);

    // test that we don't drop the unnecessary tuple if the table has the corresponding Struct
    String tableSchema2 = "a array< struct< c: array< struct< i: bigint > > > >";

    verifyWriteRead("TupleInBagInTupleInBag3", pigSchema, tableSchema2, data, true);
    verifyWriteRead("TupleInBagInTupleInBag4", pigSchema, tableSchema2, data, false);
  }

  @Test
  public void testMapWithComplexData() throws Exception {
    assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS));
    String pigSchema = "a: long, b: map[]";
    String tableSchema = "a bigint, b map<string, struct<aa:bigint, ab:string>>";

    List<Tuple> data = new ArrayList<Tuple>();
    for (int i = 0; i < 10; i++) {
      Tuple t = t(
        (long) i,
        new HashMap<String, Object>() {
          {
            put("b test 1", t(1l, "test 1"));
            put("b test 2", t(2l, "test 2"));
          }
        });

      data.add(t);
    }
    verifyWriteRead("testMapWithComplexData", pigSchema, tableSchema, data, true);
    verifyWriteRead("testMapWithComplexData2", pigSchema, tableSchema, data, false);
  }

  /**
   * artificially complex nested schema to test nested schema conversion
   * @throws Exception
   */
  @Test
  public void testMapNullKey() throws Exception {
    assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS));
    String pigSchema = "m:map[]";

    String tableSchema = "m map<string, string>";

    List<Tuple> data = new ArrayList<Tuple>();
    Tuple t = t(
      new HashMap<String, String>() {
      {
        put("ac test1", "test 1");
        put("ac test2", "test 2");
        put(null, "test 3");
      };
    });
    data.add(t);

    List<Tuple> result = new ArrayList<Tuple>();
    t = t(
      new HashMap<String, String>() {
      {
        put("ac test1", "test 1");
        put("ac test2", "test 2");
      };
    });
    result.add(t);

    verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, result, true);
    verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, result, false);
  }
}
