/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hive.ql.optimizer.listbucketingpruner;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.optimizer.PrunerUtils;
import org.apache.hadoop.hive.ql.optimizer.Transform;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;

/**
 * The transformation step that does list bucketing pruning.
 *
 */
public class ListBucketingPruner extends Transform {
  static final Logger LOG = LoggerFactory.getLogger(ListBucketingPruner.class.getName());

  /*
   * (non-Javadoc)
   *
   * @see org.apache.hadoop.hive.ql.optimizer.Transform#transform(org.apache.hadoop.hive.ql.parse.
   * ParseContext)
   */
  @Override
  public ParseContext transform(ParseContext pctx) throws SemanticException {
    // create a the context for walking operators
    NodeProcessorCtx opPartWalkerCtx = new LBOpPartitionWalkerCtx(pctx);

    // Retrieve all partitions generated from partition pruner and partition column pruner
    PrunerUtils.walkOperatorTree(pctx, opPartWalkerCtx, LBPartitionProcFactory.getFilterProc(),
        LBPartitionProcFactory.getDefaultProc());

    PrunedPartitionList partsList = ((LBOpPartitionWalkerCtx) opPartWalkerCtx).getPartitions();
    if (partsList != null) {
      Set<Partition> parts = partsList.getPartitions();
      if ((parts != null) && (parts.size() > 0)) {
        for (Partition part : parts) {
          // only process partition which is skewed and list bucketed
          if (ListBucketingPrunerUtils.isListBucketingPart(part)) {
            // create a the context for walking operators
            NodeProcessorCtx opWalkerCtx = new LBOpWalkerCtx(pctx.getOpToPartToSkewedPruner(),
                part);

            // walk operator tree to create expression tree for list bucketing
            PrunerUtils.walkOperatorTree(pctx, opWalkerCtx, LBProcFactory.getFilterProc(),
                LBProcFactory.getDefaultProc());
          }
        }
      }
    }

    return pctx;
  }

  /**
   * Prunes to the directories which match the skewed keys in where clause.
   *
   *
   * Algorithm
   *
   * =========
   *
   * For each possible skewed element combination:
   * 1. walk through ExprNode tree
   * 2. decide Boolean (True/False/unknown(null))
   *
   * Go through each skewed element combination again:
   * 1. if it is skewed value, skip the directory only if it is false, otherwise keep it
   * 2. skip the default directory only if all skewed elements,non-skewed value, are false.
   *
   * Example
   * =======
   * For example:
   * 1. skewed column (list): C1, C2
   * 2. skewed value (list of list): (1,a), (2,b), (1,c)
   *
   * Unique skewed elements for each skewed column (list of list):
   * (1,2,other), (a,b,c,other)
   *
   * Index: (0,1,2) (0,1,2,3)
   * Output matches order of skewed column. Output can be read as:
   *
   * C1 has unique element list (1,2,other)
   * C2 has unique element list (a,b,c,other)
   *
   * C1\C2 | a | b | c |Other
   * 1 | (1,a) | X | (1,c) |X
   * 2 | X |(2,b) | X |X
   * other | X | X | X |X
   *
   * Complete dynamic-multi-dimension collection
   *
   * (0,0) (1,a) * -> T
   * (0,1) (1,b) -> T
   * (0,2) (1,c) *-> F
   * (0,3) (1,other)-> F
   * (1,0) (2,a)-> F
   * (1,1) (2,b) * -> T
   * (1,2) (2,c)-> F
   * (1,3) (2,other)-> F
   * (2,0) (other,a) -> T
   * (2,1) (other,b) -> T
   * (2,2) (other,c) -> T
   * (2,3) (other,other) -> T
   * * is skewed value entry
   *
   * Expression Tree : ((c1=1) and (c2=a)) or ( (c1=3) or (c2=b))
   *
   * or
   * / \
   * and or
   * / \ / \
   * c1=1 c2=a c1=3 c2=b
   *
   *
   * For each entry in dynamic-multi-dimension container
   *
   * 1. walk through the tree to decide value (please see map's value above)
   * 2. if it is skewed value
   * 2.1 remove the entry from the map
   * 2.2 add directory to path unless value is false
   * 3. otherwise, add value to map
   *
   * Once it is done, go through the rest entries in map to decide default directory
   * 1. we know all is not skewed value
   * 2. we skip default directory only if all value is false
   *
   * What we choose at the end?
   *
   * 1. directory for (1,a) because it 's skewed value and match returns true
   * 2. directory for (2,b) because it 's skewed value and match returns true
   * 3. default directory because not all non-skewed value returns false
   *
   * we skip directory for (1,c) since match returns false
   *
   * Note: unknown is marked in {@link #transform(ParseContext)} <blockquote>
   * <pre>
   * newcd = new ExprNodeConstantDesc(cd.getTypeInfo(), null)
   * </pre>
   *
   * </blockquote> can be checked via <blockquote>
   *
   * <pre>
   *     child_nd instanceof ExprNodeConstantDesc
   *               && ((ExprNodeConstantDesc) child_nd).getValue() == null)
   * </pre>
   *
   * </blockquote>
   *
   * @param ctx
   *          parse context
   * @param part
   *          partition
   * @param pruner
   *          expression node tree
   * @return
   */
  public static Path[] prune(ParseContext ctx, Partition part, ExprNodeDesc pruner) {
    Path[] finalPaths = null;

    try {
      finalPaths = execute(ctx, part, pruner);
    } catch (SemanticException e) {
      // Use full partition path for error case.
      LOG.warn("Using full partition scan :" + Arrays.toString(part.getPath()) + ".", e);
      finalPaths = part.getPath();
    }

    return finalPaths;
  }

  /**
   * Main skeleton for list bucketing pruning.
   *
   * @param ctx
   * @param part
   * @param pruner
   * @return
   * @throws SemanticException
   */
  private static Path[] execute(ParseContext ctx, Partition part, ExprNodeDesc pruner)
      throws SemanticException {
    Path[] finalPaths;

    List<Path> selectedPaths = new ArrayList<Path>();

    if (ListBucketingPrunerUtils.isUnknownState(pruner)) {
      // Use full partition path for error case.
      LOG.warn("List bucketing pruner is either null or in unknown state "
          + " so that it uses full partition scan :" + Arrays.toString(part.getPath()));
      finalPaths = part.getPath();
    } else {
      // Retrieve skewed columns.
      List<List<String>> sVals = part.getSkewedColValues();

      assert ((sVals != null) && (sVals.size() > 0)) :
        part.getName() + " skewed metadata is corrupted. No skewed value information.";
      // Calculate collection.
      List<List<String>> indexCollection = DynamicMultiDimensionalCollection
          .generateCollection(sVals);

      assert (indexCollection != null) : "Collection is null.";

      // Calculate unique skewed elements for each skewed column.
      List<List<String>> uniqSkewValues = DynamicMultiDimensionalCollection.uniqueSkewedValueList(
          sVals);
      // Decide skewed value directory selection.
      List<Boolean> nonSkewedValueMatchResult = decideSkewedValueDirSelection(part, pruner,
          selectedPaths, indexCollection, uniqSkewValues);

      // Decide default directory selection.
      decideDefaultDirSelection(part, selectedPaths, nonSkewedValueMatchResult);

      // Finalize paths.
      finalPaths = generateFinalPath(part, selectedPaths);

    }
    return finalPaths;
  }

  /**
   * Walk through every entry in complete collection
   * 1. calculate if it matches expression tree
   * 2. decide if select skewed value directory
   * 3. store match result for non-skewed value for later handle on default directory
   * C1\C2 | a | b | c |Other
   * 1 | (1,a) | X | (1,c) |X
   * 2 | X |(2,b) | X |X
   * other | X | X | X |X
   * Final result
   * Complete dynamic-multi-dimension collection
   * (0,0) (1,a) * -> T
   * (0,1) (1,b) -> T
   * (0,2) (1,c) *-> F
   * (0,3) (1,other)-> F
   * (1,0) (2,a)-> F
   * (1,1) (2,b) * -> T
   * (1,2) (2,c)-> F
   * (1,3) (2,other)-> F
   * (2,0) (other,a) -> T
   * (2,1) (other,b) -> T
   * (2,2) (other,c) -> T
   * (2,3) (other,other) -> T
   *
   * * is skewed value entry
   *
   * 1. directory for (1,a) is chosen because it 's skewed value and match returns true
   * 2. directory for (2,b) is chosen because it 's skewed value and match returns true
   *
   * @param part
   * @param pruner
   * @param selectedPaths
   * @param collections
   * @param uniqSkewedValues
   * @return
   * @throws SemanticException
   */
  private static List<Boolean> decideSkewedValueDirSelection(Partition part, ExprNodeDesc pruner,
      List<Path> selectedPaths, List<List<String>> collections,
      List<List<String>> uniqSkewedValues) throws SemanticException {
    // For each entry in dynamic-multi-dimension collection.
    List<String> skewedCols = part.getSkewedColNames(); // Retrieve skewed column.
    Map<List<String>, String> mappings = part.getSkewedColValueLocationMaps(); // Retrieve skewed
                                                                               // map.
    assert ListBucketingPrunerUtils.isListBucketingPart(part) : part.getName()
        + " skewed metadata is corrupted. No skewed column and/or location mappings information.";
    List<List<String>> skewedValues = part.getSkewedColValues();
    List<Boolean> nonSkewedValueMatchResult = new ArrayList<Boolean>();
    for (List<String> cell : collections) {
      // Walk through the tree to decide value.
      // Example: skewed column: C1, C2 ;
      // index: (1,a) ;
      // expression tree: ((c1=1) and (c2=a)) or ((c1=3) or (c2=b))
      Boolean matchResult = ListBucketingPrunerUtils.evaluateExprOnCell(skewedCols, cell, pruner,
          uniqSkewedValues);
      // Handle skewed value.
      if (skewedValues.contains(cell)) { // if it is skewed value
        if ((matchResult == null) || matchResult) { // add directory to path unless value is false
          /* It's valid case if a partition: */
          /* 1. is defined with skewed columns and skewed values in metadata */
          /* 2. doesn't have all skewed values within its data */
          if (mappings.get(cell) != null) {
            selectedPaths.add(new Path(mappings.get(cell)));
          }
        }
      } else {
        // Non-skewed value, add it to list for later handle on default directory.
        nonSkewedValueMatchResult.add(matchResult);
      }
    }
    return nonSkewedValueMatchResult;
  }

  /**
   * Decide whether should select the default directory.
   *
   * @param part
   * @param selectedPaths
   * @param nonSkewedValueMatchResult
   */
  private static void decideDefaultDirSelection(Partition part, List<Path> selectedPaths,
      List<Boolean> nonSkewedValueMatchResult) {
    boolean skipDefDir = true;
    for (Boolean v : nonSkewedValueMatchResult) {
      if ((v == null) || v) {
        skipDefDir = false; // we skip default directory only if all value is false
        break;
      }
    }
    if (!skipDefDir) {
      StringBuilder builder = new StringBuilder();
      builder.append(part.getLocation());
      builder.append(Path.SEPARATOR);
      builder
          .append((FileUtils.makeDefaultListBucketingDirName(
              part.getSkewedColNames(),
              ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME)));
      selectedPaths.add(new Path(builder.toString()));
    }
  }

  /**
   * Decide the final path.
   *
   * @param part
   * @param selectedPaths
   * @return
   */
  private static Path[] generateFinalPath(Partition part, List<Path> selectedPaths) {
    Path[] finalPaths;
    if (selectedPaths.size() == 0) {
      LOG.warn("Using full partition scan :" + Arrays.toString(part.getPath()) + ".");
      finalPaths = part.getPath();
    } else {
      finalPaths = selectedPaths.toArray(new Path[0]);
    }
    return finalPaths;
  }


  /**
   * Note: this class is not designed to be used in general but for list bucketing pruner only.
   * The structure addresses the following requirements:
   * 1. multiple dimension collection
   * 2. length of each dimension is dynamic. It's decided at runtime.
   * The first user is list bucketing pruner and used in pruning phase:
   * 1. Each skewed column has a batch of skewed elements.
   * 2. One skewed column represents one dimension.
   * 3. Length of dimension is size of skewed elements.
   * 4. no. of skewed columns and length of dimension are dynamic and configured by user.
   * use case:
   * ========
   * Use case #1:
   * Multiple dimension collection represents if to select a directory representing by the cell.
   * skewed column: C1, C2, C3
   * skewed value: (1,a,x), (2,b,x), (1,c,x), (2,a,y)
   * Other: represent value for the column which is not part of skewed value.
   * C3 = x
   * C1\C2 | a | b | c |Other
   * 1 | Boolean(1,a,x) | X | Boolean(1,c,x) |X
   * 2 | X |Boolean(2,b,x) | X |X
   * other | X | X | X |X
   * C3 = y
   * C1\C2 | a | b | c |Other
   * 1 | X | X | X |X
   * 2 | Boolean(2,a,y) | X | X |X
   * other | X | X | X |X
   * Boolean is cell type which can be False/True/Null(Unknown).
   * (1,a,x) is just for information purpose to explain which skewed value it represents.
   * 1. value of Boolean(1,a,x) represents if we select the directory for list bucketing
   * 2. value of Boolean(2,b,x) represents if we select the directory for list bucketing
   * ...
   * 3. All the rest, marked as "X", will decide if to pickup the default directory.
   * 4. Not only "other" columns/rows but also the rest as long as it doesn't represent skewed
   * value.
   * For cell representing skewed value:
   * 1. False, skip the directory
   * 2. True/Unknown, select the directory
   * For cells representing default directory:
   * 1. only if all cells are false, skip the directory
   * 2. all other cases, select the directory
   * Use case #2:
   * Multiple dimension collection represents skewed elements so that walk through tree one by one.
   * Cell is a List<String> representing the value mapping from index path and skewed value.
   * skewed column: C1, C2, C3
   * skewed value: (1,a,x), (2,b,x), (1,c,x), (2,a,y)
   * Other: represent value for the column which is not part of skewed value.
   * C3 = x
   * C1\C2 | a | b | c |Other
   * 1 | (1,a,x) | X | (1,c,x) |X
   * 2 | X |(2,b,x) | X |X
   * other | X | X | X |X
   * C3 = y
   * C1\C2 | a | b | c |Other
   * 1 | X | X | X |X
   * 2 | (2,a,y) | X | X |X
   * other | X | X | X |X
   * Implementation:
   * ==============
   * please see another example in {@link ListBucketingPruner#prune}
   * We will use a HasMap to represent the Dynamic-Multiple-Dimension collection:
   * 1. Key is List<Integer> representing the index path to the cell
   * 2. value represents the cell (Boolean for use case #1, List<String> for case #2)
   * For example:
   * 1. skewed column (list): C1, C2, C3
   * 2. skewed value (list of list): (1,a,x), (2,b,x), (1,c,x), (2,a,y)
   * From skewed value, we calculate the unique skewed element for each skewed column:
   * C1: (1,2)
   * C2: (a,b,c)
   * C3: (x,y)
   * We store them in list of list. We don't need to store skewed column name since we use order to
   * match:
   * 1. Skewed column (list): C1, C2, C3
   * 2. Unique skewed elements for each skewed column (list of list):
   * (1,2,other), (a,b,c,other), (x,y,other)
   * 3. index (0,1,2) (0,1,2,3) (0,1,2)
   *
   * We use the index,starting at 0. to construct hashmap representing dynamic-multi-dimension
   * collection:
   * key (what skewed value key represents) -> value (Boolean for use case #1, List<String> for case
   * #2).
   * (0,0,0) (1,a,x)
   * (0,0,1) (1,a,y)
   * (0,1,0) (1,b,x)
   * (0,1,1) (1,b,y)
   * (0,2,0) (1,c,x)
   * (0,2,1) (1,c,y)
   * (1,0,0) (2,a,x)
   * (1,0,1) (2,a,y)
   * (1,1,0) (2,b,x)
   * (1,1,1) (2,b,y)
   * (1,2,0) (2,c,x)
   * (1,2,1) (2,c,y)
   * ...
   */
  public static class DynamicMultiDimensionalCollection {

    /**
     * Find out complete skewed-element collection
     * For example:
     * 1. skewed column (list): C1, C2
     * 2. skewed value (list of list): (1,a), (2,b), (1,c)
     * It returns the complete collection
     * (1,a) , (1,b) , (1,c) , (1,other), (2,a), (2,b) , (2,c), (2,other), (other,a), (other,b),
     * (other,c), (other,other)
     * @throws SemanticException
     */
    public static List<List<String>> generateCollection(List<List<String>> values)
        throws SemanticException {
      // Calculate unique skewed elements for each skewed column.
      List<List<String>> uniqSkewedElements = DynamicMultiDimensionalCollection.uniqueElementsList(
          values, ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_KEY);
      // Calculate complete dynamic-multi-dimension collection.
      return DynamicMultiDimensionalCollection.flat(uniqSkewedElements);
    }

    /**
     * Convert value to unique element list. This is specific for skew value use case:
     * For example:
     * 1. skewed column (list): C1, C2, C3
     * 2. skewed value (list of list): (1,a,x), (2,b,x), (1,c,x), (2,a,y)
     * Input: skewed value (list of list): (1,a,x), (2,b,x), (1,c,x), (2,a,y)
     * Output: Unique skewed elements for each skewed column (list of list):
     * (1,2,other), (a,b,c,other), (x,y,other)
     * Output matches order of skewed column. Output can be read as:
     * C1 has unique element list (1,2,other)
     * C2 has unique element list (a,b,c,other)
     * C3 has unique element list (x,y,other)
     * Other represents any value which is not part skewed-value combination.
     * @param values
     *          skewed value list
     * @return a list of unique element lists
     */
    public static List<List<String>> uniqueElementsList(List<List<String>> values,
        String defaultDirName) {
      // Get unique skewed value list.
      List<List<String>> result = uniqueSkewedValueList(values);

      // Add default dir at the end of each list
      for (List<String> list : result) {
        list.add(defaultDirName);
      }

      return result;
    }

    /**
     * Convert value to unique skewed value list. It is used in
     * {@link ListBucketingPrunerUtils#evaluateExprOnCell}
     *
     * For example:
     *
     * 1. skewed column (list): C1, C2, C3
     * 2. skewed value (list of list): (1,a,x), (2,b,x), (1,c,x), (2,a,y)
     *
     * Input: skewed value (list of list): (1,a,x), (2,b,x), (1,c,x), (2,a,y)
     * Output: Unique skewed value for each skewed column (list of list):
     * (1,2), (a,b,c), (x,y)
     *
     * Output matches order of skewed column. Output can be read as:
     * C1 has unique skewed value list (1,2,)
     * C2 has unique skewed value list (a,b,c)
     * C3 has unique skewed value list (x,y)
     *
     * @param values
     *          skewed value list
     * @return a list of unique skewed value lists
     */
    public static List<List<String>> uniqueSkewedValueList(List<List<String>> values) {
      if ((values == null) || (values.size() == 0)) {
        return null;
      }

      // skewed value has the same length.
      List<List<String>> result = new ArrayList<List<String>>();
      for (int i = 0; i < values.get(0).size(); i++) {
        result.add(new ArrayList<String>());
      }

      // add unique element to list per occurrence order in skewed value.
      // occurrence order in skewed value doesn't matter.
      // as long as we add them to a list, order is preserved from now on.
      for (List<String> value : values) {
        for (int i = 0; i < value.size(); i++) {
          if (!result.get(i).contains(value.get(i))) {
            result.get(i).add(value.get(i));
          }
        }
      }

      return result;
    }

    /**
     * Flat a dynamic-multi-dimension collection.
     *
     * For example:
     * 1. skewed column (list): C1, C2, C3
     * 2. skewed value (list of list): (1,a,x), (2,b,x), (1,c,x), (2,a,y)
     *
     * Unique skewed elements for each skewed column (list of list):
     * (1,2,other), (a,b,c,other)
     * Index: (0,1,2) (0,1,2,3)
     *
     * Complete dynamic-multi-dimension collection
     * (0,0) (1,a) * -> T
     * (0,1) (1,b) -> T
     * (0,2) (1,c) *-> F
     * (0,3) (1,other)-> F
     * (1,0) (2,a)-> F
     * (1,1) (2,b) * -> T
     * (1,2) (2,c)-> F
     * (1,3) (2,other)-> F
     * (2,0) (other,a) -> T
     * (2,1) (other,b) -> T
     * (2,2) (other,c) -> T
     * (2,3) (other,other) -> T
     * * is skewed value entry
     *
     * @param uniqSkewedElements
     *
     * @return
     */
    public static List<List<String>> flat(List<List<String>> uniqSkewedElements)
        throws SemanticException {
      if (uniqSkewedElements == null) {
        return null;
      }
      List<List<String>> collection = new ArrayList<List<String>>();
      walker(collection, uniqSkewedElements, new ArrayList<String>(), 0);
      return collection;
    }

    /**
     * Flat the collection recursively.
     *
     * @param finalResult
     * @param input
     * @param listSoFar
     * @param level
     * @throws SemanticException
     */
    private static void walker(List<List<String>> finalResult, final List<List<String>> input,
        List<String> listSoFar, final int level) throws SemanticException {
      // Base case.
      if (level == (input.size() - 1)) {
        assert (input.get(level) != null) : "Unique skewed element list has null list in " + level
            + "th position.";
        for (String v : input.get(level)) {
          List<String> oneCompleteIndex = new ArrayList<String>(listSoFar);
          oneCompleteIndex.add(v);
          finalResult.add(oneCompleteIndex);
        }
        return;
      }

      // Recursive.
      for (String v : input.get(level)) {
        List<String> clonedListSoFar = new ArrayList<String>(listSoFar);
        clonedListSoFar.add(v);
        int nextLevel = level + 1;
        walker(finalResult, input, clonedListSoFar, nextLevel);
      }
    }

  }
}
