/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hive.ql.optimizer.correlation;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Stack;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.PTFOperator;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.RowSchema;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.UnionOperator;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
import org.apache.hadoop.hive.ql.lib.Dispatcher;
import org.apache.hadoop.hive.ql.lib.GraphWalker;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.lib.RuleRegExp;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.MapJoinProcessor;
import org.apache.hadoop.hive.ql.optimizer.Transform;
import org.apache.hadoop.hive.ql.optimizer.physical.CommonJoinTaskDispatcher;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
import org.apache.hadoop.hive.ql.plan.JoinDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;

/**
 * Implementation of  Correlation Optimizer. This optimizer is based on
 * the paper "YSmart: Yet Another SQL-to-MapReduce Translator"
 * (Rubao Lee, Tian Luo, Yin Huai, Fusheng Wang, Yongqiang He, and Xiaodong Zhang)
 * (http://www.cse.ohio-state.edu/hpcs/WWW/HTML/publications/papers/TR-11-7.pdf).
 * Correlation Optimizer detects if ReduceSinkOperators share same keys.
 * Then, it will transform the query plan tree (operator tree) by exploiting
 * detected correlations. For details, see the original paper of YSmart.
 *
 * Test queries associated with this optimizer are correlationoptimizer1.q to
 * correlationoptimizer14.q
 */
public class CorrelationOptimizer extends Transform {

  private static final Logger LOG = LoggerFactory.getLogger(CorrelationOptimizer.class.getName());

  private boolean abort; // if correlation optimizer will not try to optimize this query

  private ParseContext pCtx;

  //Join operators which may be converted by CommonJoinResolver;
  private final Set<Operator<? extends OperatorDesc>> skipedJoinOperators;

  public CorrelationOptimizer() {
    super();
    pCtx = null;
    skipedJoinOperators = new HashSet<Operator<? extends OperatorDesc>>();
    abort = false;
  }

  private void findPossibleAutoConvertedJoinOperators() throws SemanticException {
    // Guess if CommonJoinResolver will work. If CommonJoinResolver may
    // convert a join operation, correlation optimizer will not merge that join.
    // TODO: If hive.auto.convert.join.noconditionaltask=true, for a JoinOperator
    // that has both intermediate tables and query input tables as input tables,
    // we should be able to guess if this JoinOperator will be converted to a MapJoin
    // based on hive.auto.convert.join.noconditionaltask.size.
    for (JoinOperator joinOp: pCtx.getJoinOps()) {
      boolean isAbleToGuess = true;
      boolean mayConvert = false;
      // Get total size and individual alias's size
      long aliasTotalKnownInputSize = 0;
      Map<String, Long> aliasToSize = new HashMap<String, Long>();
      Map<Integer, Set<String>> posToAliases = new HashMap<Integer, Set<String>>();
      for (int pos = 0; pos < joinOp.getNumParent(); pos++) {
        Operator<? extends OperatorDesc> op = joinOp.getParentOperators().get(pos);
        Set<TableScanOperator> topOps = CorrelationUtilities.findTableScanOperators(op);
        if (topOps.isEmpty()) {
          isAbleToGuess = false;
          break;
        }

        Set<String> aliases = new LinkedHashSet<String>();
        for (TableScanOperator tsop : topOps) {
          Table table = tsop.getConf().getTableMetadata();
          if (table == null) {
            // table should not be null.
            throw new SemanticException("The table of " +
                tsop.getName() + " " + tsop.getIdentifier() +
                " is null, which is not expected.");
          }
          String alias = tsop.getConf().getAlias();
          aliases.add(alias);

          Path p = table.getPath();
          ContentSummary resultCs = null;
          try {
            FileSystem fs = table.getPath().getFileSystem(pCtx.getConf());
            resultCs = fs.getContentSummary(p);
          } catch (IOException e) {
            LOG.warn("Encounter a error while querying content summary of table " +
                table.getCompleteName() + " from FileSystem. " +
                "Cannot guess if CommonJoinOperator will optimize " +
                joinOp.getName() + " " + joinOp.getIdentifier());
          }
          if (resultCs == null) {
            isAbleToGuess = false;
            break;
          }

          long size = resultCs.getLength();
          aliasTotalKnownInputSize += size;
          Long es = aliasToSize.get(alias);
          if(es == null) {
            es = new Long(0);
          }
          es += size;
          aliasToSize.put(alias, es);
        }
        posToAliases.put(pos, aliases);
      }

      if (!isAbleToGuess) {
        LOG.info("Cannot guess if CommonJoinOperator will optimize " +
            joinOp.getName() + " " + joinOp.getIdentifier());
        continue;
      }

      JoinDesc joinDesc = joinOp.getConf();
      Byte[] order = joinDesc.getTagOrder();
      int numAliases = order.length;
      Set<Integer> bigTableCandidates =
          MapJoinProcessor.getBigTableCandidates(joinDesc.getConds());
      if (bigTableCandidates.isEmpty()) {
        continue;
      }

      long ThresholdOfSmallTblSizeSum = HiveConf.getLongVar(pCtx.getConf(),
          HiveConf.ConfVars.HIVESMALLTABLESFILESIZE);
      for (int i = 0; i < numAliases; i++) {
        // this table cannot be big table
        if (!bigTableCandidates.contains(i)) {
          continue;
        }
        Set<String> aliases = posToAliases.get(i);
        long aliasKnownSize = Utilities.sumOf(aliasToSize, aliases);
        if (!CommonJoinTaskDispatcher.cannotConvert(aliasKnownSize,
            aliasTotalKnownInputSize, ThresholdOfSmallTblSizeSum)) {
          mayConvert = true;
        }
      }

      if (mayConvert) {
        LOG.info(joinOp.getName() + " " + joinOp.getIdentifier() +
            " may be converted to MapJoin by CommonJoinResolver");
        skipedJoinOperators.add(joinOp);
      }
    }
  }

  /**
   * Detect correlations and transform the query tree.
   *
   * @param pactx
   *          current parse context
   * @throws SemanticException
   */
  public ParseContext transform(ParseContext pctx) throws SemanticException {

    pCtx = pctx;

    if (HiveConf.getBoolVar(pCtx.getConf(),HiveConf.ConfVars.HIVECONVERTJOIN)) {
      findPossibleAutoConvertedJoinOperators();
    }

    // detect correlations
    CorrelationNodeProcCtx corrCtx = new CorrelationNodeProcCtx(pCtx);
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("R1", ReduceSinkOperator.getOperatorName() + "%"),
        new CorrelationNodeProc());

    Dispatcher disp = new DefaultRuleDispatcher(getDefaultProc(), opRules, corrCtx);
    GraphWalker ogw = new DefaultGraphWalker(disp);

    // Create a list of topOp nodes
    List<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pCtx.getTopOps().values());
    ogw.startWalking(topNodes, null);
    // We have finished tree walking (correlation detection).
    // We will first see if we need to abort (the operator tree has not been changed).
    // If not, we will start to transform the operator tree.
    abort = corrCtx.isAbort();
    if (abort) {
      LOG.info("Abort. Reasons are ...");
      for (String reason : corrCtx.getAbortReasons()) {
        LOG.info("-- " + reason);
      }
    } else {
      // transform the operator tree
      LOG.info("Begain query plan transformation based on intra-query correlations. " +
          corrCtx.getCorrelations().size() + " correlation(s) to be applied");
      for (IntraQueryCorrelation correlation : corrCtx.getCorrelations()) {
        QueryPlanTreeTransformation.applyCorrelation(pCtx, corrCtx, correlation);
      }
    }
    return pCtx;
  }

  private class CorrelationNodeProc implements NodeProcessor {

    private void analyzeReduceSinkOperatorsOfJoinOperator(JoinCondDesc[] joinConds,
        List<Operator<? extends OperatorDesc>> rsOps, Operator<? extends OperatorDesc> curentRsOp,
        Set<ReduceSinkOperator> correlatedRsOps) {
      if (correlatedRsOps.contains((ReduceSinkOperator) curentRsOp)) {
        return;
      }
      correlatedRsOps.add((ReduceSinkOperator) curentRsOp);

      int pos = rsOps.indexOf(curentRsOp);
      for (int i = 0; i < joinConds.length; i++) {
        JoinCondDesc joinCond = joinConds[i];
        int type = joinCond.getType();
        if (pos == joinCond.getLeft()) {
          if (type == JoinDesc.INNER_JOIN ||
              type == JoinDesc.LEFT_OUTER_JOIN ||
              type == JoinDesc.LEFT_SEMI_JOIN) {
            Operator<? extends OperatorDesc> newCurrentRsOps = rsOps.get(joinCond.getRight());
            analyzeReduceSinkOperatorsOfJoinOperator(joinConds, rsOps, newCurrentRsOps,
                correlatedRsOps);
          }
        } else if (pos == joinCond.getRight()) {
          if (type == JoinDesc.INNER_JOIN || type == JoinDesc.RIGHT_OUTER_JOIN) {
            Operator<? extends OperatorDesc> newCurrentRsOps = rsOps.get(joinCond.getLeft());
            analyzeReduceSinkOperatorsOfJoinOperator(joinConds, rsOps, newCurrentRsOps,
                correlatedRsOps);
          }
        }
      }
    }

    private boolean sameKeys(List<ExprNodeDesc> k1, List<ExprNodeDesc> k2) {
      if (k1.size() != k2.size()) {
        return false;
      }
      for (int i = 0; i < k1.size(); i++) {
        ExprNodeDesc expr1 = k1.get(i);
        ExprNodeDesc expr2 = k2.get(i);
        if (expr1 == null) {
          if (expr2 == null) {
            continue;
          } else {
            return false;
          }
        } else {
          if (!expr1.isSame(expr2)) {
            return false;
          }
        }
      }
      return true;
    }

    private boolean sameOrder(String order1, String order2) {
      if (order1 == null || order1.trim().equals("")) {
        if (order2 == null || order2.trim().equals("")) {
          return true;
        }
        return false;
      }
      if (order2 == null || order2.trim().equals("")) {
        return false;
      }
      order1 = order1.trim();
      order2 = order2.trim();
      if (!order1.equals(order2)) {
        return false;
      }
      return true;
    }
    /**
     * This method is used to recursively traverse the tree to find
     * ReduceSinkOperators which share the same key columns and partitioning
     * columns. Those ReduceSinkOperators are called correlated ReduceSinkOperaotrs.
     *
     * @param child The child of the current operator
     * @param childKeyCols The key columns from the child operator
     * @param childPartitionCols The partitioning columns from the child operator
     * @param childRSOrder The sorting order of key columns from the child operator
     * @param current The current operator we are visiting
     * @param correlation The object keeps tracking the correlation
     * @return
     * @throws SemanticException
     */
    private LinkedHashSet<ReduceSinkOperator> findCorrelatedReduceSinkOperators(
        Operator<? extends OperatorDesc> child,
        List<ExprNodeDesc> childKeyCols, List<ExprNodeDesc> childPartitionCols,
        String childRSOrder,
        Operator<? extends OperatorDesc> current,
        IntraQueryCorrelation correlation) throws SemanticException {

      LOG.info("now detecting operator " + current.getIdentifier() + " " + current.getName());
      LinkedHashSet<ReduceSinkOperator> correlatedReduceSinkOperators =
          new LinkedHashSet<ReduceSinkOperator>();
      if (skipedJoinOperators.contains(current)) {
        LOG.info(current.getName() + " " + current.getIdentifier() +
            " may be converted to MapJoin by " +
            "CommonJoinResolver. Correlation optimizer will not detect correlations" +
            "involved in this operator");
        return correlatedReduceSinkOperators;
      }
      if ((current.getParentOperators() == null) || (current.getParentOperators().isEmpty())) {
        return correlatedReduceSinkOperators;
      }
      if (current instanceof PTFOperator) {
        // Currently, we do not support PTF operator.
        LOG.info("Currently, correlation optimizer does not support PTF operator.");
        return correlatedReduceSinkOperators;
      }
      if (current instanceof UnionOperator) {
        // If we get a UnionOperator, right now, we only handle it when
        // we can find correlated ReduceSinkOperators from all inputs.
        LinkedHashSet<ReduceSinkOperator> corrRSs = new LinkedHashSet<ReduceSinkOperator>();
        for (Operator<? extends OperatorDesc> parent : current.getParentOperators()) {
          LinkedHashSet<ReduceSinkOperator> tmp =
              findCorrelatedReduceSinkOperators(
                  current, childKeyCols, childPartitionCols, childRSOrder, parent, correlation);
          if (tmp != null && tmp.size() > 0) {
            corrRSs.addAll(tmp);
          } else {
            return correlatedReduceSinkOperators;
          }
        }
        correlatedReduceSinkOperators.addAll(corrRSs);
        UnionOperator union = (UnionOperator)current;
        union.getConf().setAllInputsInSameReducer(true);
      } else if (current.getColumnExprMap() == null && !(current instanceof ReduceSinkOperator)) {
        for (Operator<? extends OperatorDesc> parent : current.getParentOperators()) {
          correlatedReduceSinkOperators.addAll(
              findCorrelatedReduceSinkOperators(
                  current, childKeyCols, childPartitionCols, childRSOrder, parent, correlation));
        }
      } else if (current.getColumnExprMap() != null && !(current instanceof ReduceSinkOperator)) {
        List<ExprNodeDesc> backtrackedKeyCols =
            ExprNodeDescUtils.backtrack(childKeyCols, child, current);
        List<ExprNodeDesc> backtrackedPartitionCols =
            ExprNodeDescUtils.backtrack(childPartitionCols, child, current);

        RowSchema rowSchema = current.getSchema();
        Set<String> tableNeedToCheck = new HashSet<String>();
        for (ExprNodeDesc expr: childKeyCols) {
          if (!(expr instanceof ExprNodeColumnDesc)) {
            return correlatedReduceSinkOperators;
          }
          String colName = ((ExprNodeColumnDesc)expr).getColumn();
          ColumnInfo columnInfo = rowSchema.getColumnInfo(colName);
          if (columnInfo != null) {
            tableNeedToCheck.add(columnInfo.getTabAlias());
          }
        }
        if (current instanceof JoinOperator) {
          boolean isCorrelated = true;
          int expectedNumCorrelatedRsops = current.getParentOperators().size();
          LinkedHashSet<ReduceSinkOperator> correlatedRsops = null;
          for (Operator<? extends OperatorDesc> parent : current.getParentOperators()) {
            Set<String> tableNames = parent.getSchema().getTableNames();
            for (String tbl : tableNames) {
              if (tableNeedToCheck.contains(tbl)) {
                correlatedRsops = findCorrelatedReduceSinkOperators(current,
                    backtrackedKeyCols, backtrackedPartitionCols,
                    childRSOrder, parent, correlation);
                if (correlatedRsops.size() != expectedNumCorrelatedRsops) {
                  isCorrelated = false;
                }
              }
            }
            if (!isCorrelated) {
              break;
            }
          }
          // If current is JoinOperaotr, we will stop to traverse the tree
          // when any of parent ReduceSinkOperaotr of this JoinOperator is
          // not considered as a correlated ReduceSinkOperator.
          if (isCorrelated && correlatedRsops != null) {
            correlatedReduceSinkOperators.addAll(correlatedRsops);
          } else {
            correlatedReduceSinkOperators.clear();
          }
        } else {
          for (Operator<? extends OperatorDesc> parent : current.getParentOperators()) {
            correlatedReduceSinkOperators.addAll(findCorrelatedReduceSinkOperators(
                current, backtrackedKeyCols, backtrackedPartitionCols, childRSOrder,
                parent, correlation));
          }
        }
      } else if (current.getColumnExprMap() != null && current instanceof ReduceSinkOperator) {
        ReduceSinkOperator rsop = (ReduceSinkOperator) current;
        List<ExprNodeDesc> backtrackedKeyCols =
            ExprNodeDescUtils.backtrack(childKeyCols, child, current);
        List<ExprNodeDesc> backtrackedPartitionCols =
            ExprNodeDescUtils.backtrack(childPartitionCols, child, current);
        List<ExprNodeDesc> rsKeyCols = rsop.getConf().getKeyCols();
        List<ExprNodeDesc> rsPartitionCols = rsop.getConf().getPartitionCols();

        // Two ReduceSinkOperators are correlated means that
        // they have same sorting columns (key columns), same partitioning columns,
        // same sorting orders, and no conflict on the numbers of reducers.
        // TODO: we should relax this condition
        // TODO: we need to handle aggregation functions with distinct keyword. In this case,
        // distinct columns will be added to the key columns.
        boolean isCorrelated = sameKeys(rsKeyCols, backtrackedKeyCols) &&
            sameOrder(rsop.getConf().getOrder(), childRSOrder) &&
            sameKeys(backtrackedPartitionCols, rsPartitionCols) &&
            correlation.adjustNumReducers(rsop.getConf().getNumReducers());
        GroupByOperator cGBY =
            CorrelationUtilities.getSingleChild(rsop, GroupByOperator.class);
        if (cGBY != null) {
          if (CorrelationUtilities.hasGroupingSet(rsop) ||
              cGBY.getConf().isGroupingSetsPresent()) {
            // Do not support grouping set right now
            isCorrelated = false;
          }
        }

        if (isCorrelated) {
          LOG.info("Operator " + current.getIdentifier() + " " +
              current.getName() + " is correlated");
          Operator<? extends OperatorDesc> childOperator =
              CorrelationUtilities.getSingleChild(current, true);
          if (childOperator instanceof JoinOperator) {
            JoinOperator joinOp = (JoinOperator) childOperator;
            JoinCondDesc[] joinConds = joinOp.getConf().getConds();
            List<Operator<? extends OperatorDesc>> rsOps = joinOp.getParentOperators();
            LinkedHashSet<ReduceSinkOperator> correlatedRsOps =
                new LinkedHashSet<ReduceSinkOperator>();
            analyzeReduceSinkOperatorsOfJoinOperator(joinConds, rsOps, current, correlatedRsOps);
            correlatedReduceSinkOperators.addAll(correlatedRsOps);
          } else {
            correlatedReduceSinkOperators.add(rsop);
          }
        } else {
          LOG.info("Operator " + current.getIdentifier() + " " +
              current.getName() + " is not correlated");
          correlatedReduceSinkOperators.clear();
        }
      } else {
        LOG.error("ReduceSinkOperator " + current.getIdentifier() + " does not have ColumnExprMap");
        throw new SemanticException("CorrelationOptimizer cannot optimize this plan. " +
            "ReduceSinkOperator " + current.getIdentifier()
            + " does not have ColumnExprMap");
      }
      return correlatedReduceSinkOperators;
    }

    /** Start to exploit Job Flow Correlation from op.
     * Example: here is the operator tree we have ...
     *       JOIN2
     *      /    \
     *     RS4   RS5
     *    /        \
     *   GBY1     JOIN1
     *    |       /    \
     *   RS1     RS2   RS3
     * The op will be RS4. If we can execute GBY1, JOIN1, and JOIN2 in
     * the same reducer. This method will return [RS1, RS2, RS3].
     * @param op
     * @param correlationCtx
     * @param correlation
     * @return
     * @throws SemanticException
     */
    private LinkedHashSet<ReduceSinkOperator> exploitJobFlowCorrelation(ReduceSinkOperator op,
        CorrelationNodeProcCtx correlationCtx, IntraQueryCorrelation correlation)
        throws SemanticException {
      correlationCtx.addWalked(op);
      correlation.addToAllReduceSinkOperators(op);
      boolean shouldDetect = true;
      LinkedHashSet<ReduceSinkOperator> reduceSinkOperators =
          new LinkedHashSet<ReduceSinkOperator>();
      List<ExprNodeDesc> keyCols = op.getConf().getKeyCols();
      List<ExprNodeDesc> partitionCols = op.getConf().getPartitionCols();
      for (ExprNodeDesc key : keyCols) {
        if (!(key instanceof ExprNodeColumnDesc)) {
          shouldDetect = false;
        }
      }
      for (ExprNodeDesc key : partitionCols) {
        if (!(key instanceof ExprNodeColumnDesc)) {
          shouldDetect = false;
        }
      }
      GroupByOperator cGBY =
          CorrelationUtilities.getSingleChild(op, GroupByOperator.class);
      if (cGBY != null) {
        if (CorrelationUtilities.hasGroupingSet(op) ||
            cGBY.getConf().isGroupingSetsPresent()) {
          // Do not support grouping set right now
          shouldDetect = false;
        }
      }

      if (shouldDetect) {
        LinkedHashSet<ReduceSinkOperator> newReduceSinkOperators =
            new LinkedHashSet<ReduceSinkOperator>();
        String sortOrder = op.getConf().getOrder();
        for (Operator<? extends OperatorDesc> parent : op.getParentOperators()) {
          LOG.info("Operator " + op.getIdentifier()
              + ": start detecting correlation from this operator");
          LinkedHashSet<ReduceSinkOperator> correlatedReduceSinkOperators =
              findCorrelatedReduceSinkOperators(op, keyCols, partitionCols,
                  sortOrder, parent, correlation);
          if (correlatedReduceSinkOperators.size() == 0) {
            newReduceSinkOperators.add(op);
          } else {
            for (ReduceSinkOperator rsop : correlatedReduceSinkOperators) {
              LinkedHashSet<ReduceSinkOperator> exploited =
                  exploitJobFlowCorrelation(rsop, correlationCtx, correlation);
              if (exploited.size() == 0) {
                newReduceSinkOperators.add(rsop);
              } else {
                newReduceSinkOperators.addAll(exploited);
              }
            }
          }
        }
        reduceSinkOperators.addAll(newReduceSinkOperators);
      }
      return reduceSinkOperators;
    }

    public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
        Object... nodeOutputs) throws SemanticException {
      CorrelationNodeProcCtx corrCtx = (CorrelationNodeProcCtx) ctx;
      ReduceSinkOperator op = (ReduceSinkOperator) nd;
      // Check if we have visited this operator
      if (corrCtx.isWalked(op)) {
        return null;
      }

      LOG.info("Walk to operator " + op.getIdentifier() + " " + op.getName());

      Operator<? extends OperatorDesc> child = CorrelationUtilities.getSingleChild(op, true);
      if (!(child instanceof JoinOperator) && !(child instanceof GroupByOperator)) {
        corrCtx.addWalked(op);
        return null;
      }

      // detect correlations
      IntraQueryCorrelation correlation = new IntraQueryCorrelation(corrCtx.minReducer());
      List<ReduceSinkOperator> topReduceSinkOperators =
          CorrelationUtilities.findSiblingReduceSinkOperators(op);
      List<ReduceSinkOperator> bottomReduceSinkOperators = new ArrayList<ReduceSinkOperator>();
      // Adjust the number of reducers of this correlation based on
      // those top layer ReduceSinkOperators.
      for (ReduceSinkOperator rsop : topReduceSinkOperators) {
        if (!correlation.adjustNumReducers(rsop.getConf().getNumReducers())) {
          // If we have a conflict on the number of reducers, we will not optimize
          // this plan from here.
          corrCtx.addWalked(op);
          return null;
        }
      }
      for (ReduceSinkOperator rsop : topReduceSinkOperators) {
        LinkedHashSet<ReduceSinkOperator> thisBottomReduceSinkOperators =
            exploitJobFlowCorrelation(rsop, corrCtx, correlation);
        if (thisBottomReduceSinkOperators.size() == 0) {
          thisBottomReduceSinkOperators.add(rsop);
        }
        bottomReduceSinkOperators.addAll(thisBottomReduceSinkOperators);
      }

      if (!topReduceSinkOperators.containsAll(bottomReduceSinkOperators)) {
        LOG.info("has job flow correlation");
        correlation.setJobFlowCorrelation(true, bottomReduceSinkOperators);
      }

      if (correlation.hasJobFlowCorrelation()) {
        corrCtx.addCorrelation(correlation);
      } else {
        // Since we cannot merge operators into a single MR job from here,
        // we should remove ReduceSinkOperators added into walked in exploitJFC
        corrCtx.removeWalkedAll(correlation.getAllReduceSinkOperators());
      }

      corrCtx.addWalked(op);
      return null;
    }
  }

  private NodeProcessor getDefaultProc() {
    return new NodeProcessor() {
      @Override
      public Object process(Node nd, Stack<Node> stack,
          NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException {
        Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) nd;
        LOG.info("Walk to operator " + op.getIdentifier() + " "
            + op.getName() + ". No actual work to do");
        CorrelationNodeProcCtx correlationCtx = (CorrelationNodeProcCtx) ctx;
        if (op.getName().equals(MapJoinOperator.getOperatorName())) {
          correlationCtx.setAbort(true);
          correlationCtx.getAbortReasons().add("Found MAPJOIN");
        }
        if (op.getName().equals(FileSinkOperator.getOperatorName())) {
          correlationCtx.incrementFileSinkOperatorCount();
        }
        return null;
      }
    };
  }

  protected class CorrelationNodeProcCtx extends AbstractCorrelationProcCtx {

    private boolean abort;
    private final List<String> abortReasons;

    private final Set<ReduceSinkOperator> walked;

    private final List<IntraQueryCorrelation> correlations;

    private int fileSinkOperatorCount;

    public CorrelationNodeProcCtx(ParseContext pctx) {
      super(pctx);
      walked = new HashSet<ReduceSinkOperator>();
      correlations = new ArrayList<IntraQueryCorrelation>();
      abort = false;
      abortReasons = new ArrayList<String>();
      fileSinkOperatorCount = 0;
    }

    public void setAbort(boolean abort) {
      this.abort = abort;
    }

    public boolean isAbort() {
      return abort;
    }

    public List<String> getAbortReasons() {
      return abortReasons;
    }

    public void addCorrelation(IntraQueryCorrelation correlation) {
      correlations.add(correlation);
    }

    public List<IntraQueryCorrelation> getCorrelations() {
      return correlations;
    }

    public boolean isWalked(ReduceSinkOperator op) {
      return walked.contains(op);
    }

    public void addWalked(ReduceSinkOperator op) {
      walked.add(op);
    }

    public void addWalkedAll(Collection<ReduceSinkOperator> c) {
      walked.addAll(c);
    }

    public void removeWalked(ReduceSinkOperator op) {
      walked.remove(op);
    }

    public void removeWalkedAll(Collection<ReduceSinkOperator> c) {
      walked.removeAll(c);
    }

    public void incrementFileSinkOperatorCount() {
      fileSinkOperatorCount++;
      if (fileSinkOperatorCount == 2) {
        abort = true;
        abortReasons.add(
            "-- Currently, a query with multiple FileSinkOperators are not supported.");
      }
    }
  }

}
