/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hive.ql.parse;

import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Stack;

import org.antlr.runtime.CommonToken;
import org.antlr.runtime.tree.TreeWizard;
import org.antlr.runtime.tree.TreeWizard.ContextVisitor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.PTFPartition;
import org.apache.hadoop.hive.ql.exec.WindowFunctionInfo;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderSpec;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFInputSpec;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputSpec;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputType;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionExpression;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionSpec;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionedTableFunctionSpec;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.BoundarySpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowExpressionSpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFrameSpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowSpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowType;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.PTFDesc;
import org.apache.hadoop.hive.ql.plan.PTFDeserializer;
import org.apache.hadoop.hive.ql.plan.ptf.BoundaryDef;
import org.apache.hadoop.hive.ql.plan.ptf.OrderDef;
import org.apache.hadoop.hive.ql.plan.ptf.OrderExpressionDef;
import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef;
import org.apache.hadoop.hive.ql.plan.ptf.PTFInputDef;
import org.apache.hadoop.hive.ql.plan.ptf.PTFQueryInputDef;
import org.apache.hadoop.hive.ql.plan.ptf.PartitionDef;
import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef;
import org.apache.hadoop.hive.ql.plan.ptf.ShapeDetails;
import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef;
import org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef;
import org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLeadLag;
import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator;
import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver;
import org.apache.hadoop.hive.ql.udf.ptf.WindowingTableFunction.WindowingTableFunctionResolver;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class PTFTranslator {

  private static final Logger LOG = LoggerFactory.getLogger("org.apache.hadoop.hive.ql.parse");

  HiveConf hCfg;
  LeadLagInfo llInfo;
  SemanticAnalyzer semAly;
  UnparseTranslator unparseT;
  RowResolver inputRR;
  PTFDesc ptfDesc;
  PTFInvocationSpec ptfInvocation;
  WindowingSpec windowingSpec;

  private void init(SemanticAnalyzer semAly,
      HiveConf hCfg,
      RowResolver inputRR,
      UnparseTranslator unparseT) {
    this.semAly = semAly;
    this.hCfg = hCfg;
    this.inputRR = inputRR;
    this.unparseT = unparseT;
    llInfo = new LeadLagInfo();

  }

  public PTFDesc translate(PTFInvocationSpec qSpec,
      SemanticAnalyzer semAly,
      HiveConf hCfg,
      RowResolver inputRR,
      UnparseTranslator unparseT)
      throws SemanticException {
    init(semAly, hCfg, inputRR, unparseT);
    ptfInvocation = qSpec;
    ptfDesc = new PTFDesc();
    ptfDesc.setCfg(hCfg);
    ptfDesc.setLlInfo(llInfo);
    translatePTFChain();
    PTFDeserializer.alterOutputOIForStreaming(ptfDesc);
    return ptfDesc;
  }

  public PTFDesc translate(WindowingSpec wdwSpec, SemanticAnalyzer semAly, HiveConf hCfg,
      RowResolver inputRR,
      UnparseTranslator unparseT)
      throws SemanticException {
    init(semAly, hCfg, inputRR, unparseT);
    windowingSpec = wdwSpec;
    ptfDesc = new PTFDesc();
    ptfDesc.setCfg(hCfg);
    ptfDesc.setLlInfo(llInfo);
    WindowTableFunctionDef wdwTFnDef = new WindowTableFunctionDef();
    ptfDesc.setFuncDef(wdwTFnDef);

    PTFQueryInputSpec inpSpec = new PTFQueryInputSpec();
    inpSpec.setType(PTFQueryInputType.WINDOWING);
    wdwTFnDef.setInput(translate(inpSpec, 0));
    ShapeDetails inpShape = wdwTFnDef.getInput().getOutputShape();

    WindowingTableFunctionResolver tFn = (WindowingTableFunctionResolver)
        FunctionRegistry.getTableFunctionResolver(FunctionRegistry.WINDOWING_TABLE_FUNCTION);
    if (tFn == null) {
      throw new SemanticException(String.format("Internal Error: Unknown Table Function %s",
          FunctionRegistry.WINDOWING_TABLE_FUNCTION));
    }
    wdwTFnDef.setName(FunctionRegistry.WINDOWING_TABLE_FUNCTION);
    wdwTFnDef.setResolverClassName(tFn.getClass().getName());
    wdwTFnDef.setAlias("ptf_" + 1);
    wdwTFnDef.setExpressionTreeString(null);
    wdwTFnDef.setTransformsRawInput(false);
    tFn.initialize(hCfg, ptfDesc, wdwTFnDef);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    wdwTFnDef.setTFunction(tEval);
    wdwTFnDef.setCarryForwardNames(tFn.carryForwardNames());
    wdwTFnDef.setRawInputShape(inpShape);

    PartitioningSpec partiSpec = wdwSpec.getQueryPartitioningSpec();
    if (partiSpec == null) {
      throw new SemanticException(
          "Invalid use of Windowing: there is no Partitioning associated with Windowing");
    }
    PartitionDef partDef = translate(inpShape, wdwSpec.getQueryPartitionSpec());
    OrderDef ordDef = translate(inpShape, wdwSpec.getQueryOrderSpec(), partDef);

    wdwTFnDef.setPartition(partDef);
    wdwTFnDef.setOrder(ordDef);

    /*
     * process Wdw functions
     */
    ArrayList<WindowFunctionDef> windowFunctions = new ArrayList<WindowFunctionDef>();
    if (wdwSpec.getWindowExpressions() != null) {
      for (WindowExpressionSpec expr : wdwSpec.getWindowExpressions()) {
        if (expr instanceof WindowFunctionSpec) {
          WindowFunctionDef wFnDef = translate(wdwTFnDef, (WindowFunctionSpec) expr);
          windowFunctions.add(wFnDef);
        }
      }
      wdwTFnDef.setWindowFunctions(windowFunctions);
    }

    /*
     * set outputFromWdwFnProcessing
     */
    ArrayList<String> aliases = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    for (WindowFunctionDef wFnDef : windowFunctions) {
      aliases.add(wFnDef.getAlias());
      if (wFnDef.isPivotResult()) {
        fieldOIs.add(((ListObjectInspector) wFnDef.getOI()).getListElementObjectInspector());
      } else {
        fieldOIs.add(wFnDef.getOI());
      }
    }
    PTFTranslator.addInputColumnsToList(inpShape, aliases, fieldOIs);
    StructObjectInspector wdwOutOI = ObjectInspectorFactory.getStandardStructObjectInspector(
        aliases, fieldOIs);
    tFn.setWdwProcessingOutputOI(wdwOutOI);
    RowResolver wdwOutRR = buildRowResolverForWindowing(wdwTFnDef);
    ShapeDetails wdwOutShape = setupShape(wdwOutOI, null, wdwOutRR);
    wdwTFnDef.setOutputShape(wdwOutShape);

    tFn.setupOutputOI();

    PTFDeserializer.alterOutputOIForStreaming(ptfDesc);

    return ptfDesc;
  }

  private void translatePTFChain() throws SemanticException {

    Deque<PTFInputSpec> ptfChain = new ArrayDeque<PTFInvocationSpec.PTFInputSpec>();
    PTFInputSpec currentSpec = ptfInvocation.getFunction();
    while (currentSpec != null) {
      ptfChain.push(currentSpec);
      currentSpec = currentSpec.getInput();
    }

    int inputNum = 0;
    PTFInputDef currentDef = null;
    while (!ptfChain.isEmpty()) {
      currentSpec = ptfChain.pop();

      if (currentSpec instanceof PTFQueryInputSpec) {
        currentDef = translate((PTFQueryInputSpec) currentSpec, inputNum);
      }
      else {
        currentDef = translate((PartitionedTableFunctionSpec) currentSpec,
            currentDef,
            inputNum);
      }
      inputNum++;
    }
    ptfDesc.setFuncDef((PartitionedTableFunctionDef) currentDef);
  }

  private PTFQueryInputDef translate(PTFQueryInputSpec spec,
      int inpNum) throws SemanticException
  {
    PTFQueryInputDef def = new PTFQueryInputDef();
    StructObjectInspector oi = PTFTranslator.getStandardStructOI(inputRR);
    ShapeDetails shp = setupShape(oi, null, inputRR);
    def.setOutputShape(shp);
    def.setType(spec.getType());
    def.setAlias(spec.getSource() == null ? "ptf_" + inpNum : spec.getSource());
    return def;
  }

  private PartitionedTableFunctionDef translate(PartitionedTableFunctionSpec spec,
      PTFInputDef inpDef,
      int inpNum)
      throws SemanticException {
    TableFunctionResolver tFn = FunctionRegistry.getTableFunctionResolver(spec.getName());
    if (tFn == null) {
      throw new SemanticException(String.format("Unknown Table Function %s",
          spec.getName()));
    }
    PartitionedTableFunctionDef def = new PartitionedTableFunctionDef();
    def.setInput(inpDef);
    def.setName(spec.getName());
    def.setResolverClassName(tFn.getClass().getName());
    def.setAlias(spec.getAlias() == null ? "ptf_" + inpNum : spec.getAlias());
    def.setExpressionTreeString(spec.getAstNode().toStringTree());
    def.setTransformsRawInput(tFn.transformsRawInput());
    /*
     * translate args
     */
    List<ASTNode> args = spec.getArgs();
    if (args != null)
    {
      for (ASTNode expr : args)
      {
        PTFExpressionDef argDef = null;
        try {
          argDef = buildExpressionDef(inpDef.getOutputShape(), expr);
        } catch (HiveException he) {
          throw new SemanticException(he);
        }
        def.addArg(argDef);
      }
    }

    tFn.initialize(hCfg, ptfDesc, def);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    def.setTFunction(tEval);
    def.setCarryForwardNames(tFn.carryForwardNames());
    tFn.setupRawInputOI();

    if (tFn.transformsRawInput()) {
      StructObjectInspector rawInOutOI = tEval.getRawInputOI();
      List<String> rawInOutColNames = tFn.getRawInputColumnNames();
      RowResolver rawInRR = buildRowResolverForPTF(def.getName(),
          spec.getAlias(),
          rawInOutOI,
          rawInOutColNames,
          inpDef.getOutputShape().getRr());
      ShapeDetails rawInpShape = setupTableFnShape(def.getName(),
          inpDef.getOutputShape(),
          rawInOutOI,
          rawInOutColNames,
          rawInRR);
      def.setRawInputShape(rawInpShape);
    }
    else {
      def.setRawInputShape(inpDef.getOutputShape());
    }

    translatePartitioning(def, spec);
    tFn.setupOutputOI();

    StructObjectInspector outputOI = tEval.getOutputOI();
    List<String> outColNames = tFn.getOutputColumnNames();
    RowResolver outRR = buildRowResolverForPTF(def.getName(),
        spec.getAlias(),
        outputOI,
        outColNames,
        def.getRawInputShape().getRr());
    ShapeDetails outputShape = setupTableFnShape(def.getName(),
        inpDef.getOutputShape(),
        outputOI,
        outColNames,
        outRR);
    def.setOutputShape(outputShape);
    def.setReferencedColumns(tFn.getReferencedColumns());

    return def;
  }

  private WindowFunctionDef translate(WindowTableFunctionDef wdwTFnDef,
      WindowFunctionSpec spec) throws SemanticException {
    WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(spec.getName());
    if (wFnInfo == null) {
      throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(spec.getName()));
    }
    WindowFunctionDef def = new WindowFunctionDef();
    def.setName(spec.getName());
    def.setAlias(spec.getAlias());
    def.setDistinct(spec.isDistinct());
    def.setExpressionTreeString(spec.getExpression().toStringTree());
    def.setStar(spec.isStar());
    def.setPivotResult(wFnInfo.isPivotResult());
    ShapeDetails inpShape = wdwTFnDef.getRawInputShape();

    /*
     * translate args
     */
    ArrayList<ASTNode> args = spec.getArgs();
    if (args != null)
    {
      for (ASTNode expr : args)
      {
        PTFExpressionDef argDef = null;
        try {
          argDef = buildExpressionDef(inpShape, expr);
        } catch (HiveException he) {
          throw new SemanticException(he);
        }
        def.addArg(argDef);
      }
    }

    if (FunctionRegistry.isRankingFunction(spec.getName())){
      setupRankingArgs(wdwTFnDef, def, spec);
    }

    WindowSpec wdwSpec = spec.getWindowSpec();
    if (wdwSpec != null) {
      String desc = spec.toString();

      WindowFrameDef wdwFrame = translate(spec.getName(), inpShape, wdwSpec);
      if (!wFnInfo.isSupportsWindow())
      {
        BoundarySpec start = wdwSpec.getWindowFrame().getStart();
        if (start.getAmt() != BoundarySpec.UNBOUNDED_AMOUNT) {
          throw new SemanticException(
              String.format("Expecting left window frame boundary for " +
                  "function %s to be unbounded. Found : %d", desc, start.getAmt()));
        }
        BoundarySpec end = wdwSpec.getWindowFrame().getEnd();
        if (end.getAmt() != BoundarySpec.UNBOUNDED_AMOUNT) {
          throw new SemanticException(
              String.format("Expecting right window frame boundary for " +
                  "function %s to be unbounded. Found : %d", desc, start.getAmt()));
        }
      }
      def.setWindowFrame(wdwFrame);
    }

    try {
      setupWdwFnEvaluator(def);
    } catch (HiveException he) {
      throw new SemanticException(he);
    }

    return def;
  }

  private void translatePartitioning(PartitionedTableFunctionDef def,
      PartitionedTableFunctionSpec spec)
      throws SemanticException {

    applyConstantPartition(spec);
    if (spec.getPartition() == null) {
      return;
    }
    PartitionDef partDef = translate(def.getRawInputShape(), spec.getPartition());
    OrderDef orderDef = translate(def.getRawInputShape(), spec.getOrder(), partDef);
    def.setPartition(partDef);
    def.setOrder(orderDef);
  }

  /*
   * If this the first PPTF in the chain and there is no partition specified
   * then assume the user wants to include the entire input in 1 partition.
   */
  private static void applyConstantPartition(PartitionedTableFunctionSpec spec) {
    if (spec.getPartition() != null) {
      return;
    }
    PTFInputSpec iSpec = spec.getInput();
    if (iSpec instanceof PTFInputSpec) {
      PartitionSpec partSpec = new PartitionSpec();
      PartitionExpression partExpr = new PartitionExpression();
      partExpr.setExpression(new ASTNode(new CommonToken(HiveParser.Number, "0")));
      partSpec.addExpression(partExpr);
      spec.setPartition(partSpec);
    }
  }

  private PartitionDef translate(ShapeDetails inpShape, PartitionSpec spec)
      throws SemanticException {
    if (spec == null || spec.getExpressions() == null || spec.getExpressions().size() == 0) {
      return null;
    }

    PartitionDef pDef = new PartitionDef();
    for (PartitionExpression pExpr : spec.getExpressions()) {
      PTFExpressionDef expDef = translate(inpShape, pExpr);
      pDef.addExpression(expDef);
    }
    return pDef;
  }

  private PTFExpressionDef translate(ShapeDetails inpShape,
      PartitionExpression pExpr) throws SemanticException {
    PTFExpressionDef expDef = null;
    try {
      expDef = buildExpressionDef(inpShape, pExpr.getExpression());
    } catch (HiveException he) {
      throw new SemanticException(he);
    }
    PTFTranslator.validateComparable(expDef.getOI(),
        String.format("Partition Expression %s is not a comparable expression", pExpr
            .getExpression().toStringTree()));
    return expDef;
  }

  private OrderDef translate(ShapeDetails inpShape,
      OrderSpec spec,
      PartitionDef partitionDef) throws SemanticException {

    OrderDef def = new OrderDef();
    if (null == spec) {
      return def;
    }

    for (OrderExpression oExpr : spec.getExpressions())
    {
      OrderExpressionDef oexpDef = translate(inpShape, oExpr);
      def.addExpression(oexpDef);
    }

    return def;
  }

  private OrderExpressionDef translate(ShapeDetails inpShape,
      OrderExpression oExpr)
      throws SemanticException {
    OrderExpressionDef oexpDef = new OrderExpressionDef();
    oexpDef.setOrder(oExpr.getOrder());
    oexpDef.setNullOrder(oExpr.getNullOrder());
    try {
      PTFExpressionDef expDef = buildExpressionDef(inpShape, oExpr.getExpression());
      oexpDef.setExpressionTreeString(expDef.getExpressionTreeString());
      oexpDef.setExprEvaluator(expDef.getExprEvaluator());
      oexpDef.setExprNode(expDef.getExprNode());
      oexpDef.setOI(expDef.getOI());
    } catch (HiveException he) {
      throw new SemanticException(he);
    }
    PTFTranslator.validateComparable(oexpDef.getOI(),
        String.format("Partition Expression %s is not a comparable expression",
            oExpr.getExpression().toStringTree()));
    return oexpDef;
  }

  private WindowFrameDef translate(String wFnName, ShapeDetails inpShape, WindowSpec spec)
      throws SemanticException {
    /*
     * Since we componentize Windowing, no need to translate
     * the Partition & Order specs of individual WFns.
     */
    return translate(inpShape, spec.getWindowFrame(), spec.getOrder().getExpressions());
  }

  private WindowFrameDef translate(ShapeDetails inpShape,
      WindowFrameSpec spec,
      List<OrderExpression> orderExpressions)
      throws SemanticException {
    if (spec == null) {
      return null;
    }

    BoundarySpec s = spec.getStart();
    BoundarySpec e = spec.getEnd();
    int cmp = s.compareTo(e);
    if (cmp > 0) {
      throw new SemanticException(String.format(
          "Window range invalid, start boundary is greater than end boundary: %s", spec));
    }

    WindowFrameDef winFrame = new WindowFrameDef(
        spec.getWindowType(),
        new BoundaryDef(s.direction, s.getAmt()),
        new BoundaryDef(e.direction, e.getAmt()));
    if (winFrame.getWindowType() == WindowType.RANGE) {
      winFrame.setOrderDef(buildOrderExpressions(inpShape, orderExpressions));
    }
    return winFrame;
  }

  /**
   * Collect order expressions for RANGE based windowing
   * @throws SemanticException
   */
  private OrderDef buildOrderExpressions(ShapeDetails inpShape, List<OrderExpression> orderExpressions)
      throws SemanticException {
    OrderDef orderDef = new OrderDef();
    for (OrderExpression oe : orderExpressions) {
      PTFTranslator.validateNoLeadLagInValueBoundarySpec(oe.getExpression());
      PTFExpressionDef exprDef = null;
      try {
        exprDef = buildExpressionDef(inpShape, oe.getExpression());
      } catch (HiveException he) {
        throw new SemanticException(he);
      }
      PTFTranslator.validateValueBoundaryExprType(exprDef.getOI());
      OrderExpressionDef orderExprDef = new OrderExpressionDef(exprDef);
      orderExprDef.setOrder(oe.getOrder());
      orderExprDef.setNullOrder(oe.getNullOrder());
      orderDef.addExpression(orderExprDef);
    }
    return orderDef;
  }

  static void setupWdwFnEvaluator(WindowFunctionDef def) throws HiveException {
    List<PTFExpressionDef> args = def.getArgs();
    List<ObjectInspector> argOIs = new ArrayList<ObjectInspector>();
    ObjectInspector[] funcArgOIs = null;

    if (args != null) {
      for (PTFExpressionDef arg : args) {
        argOIs.add(arg.getOI());
      }
      funcArgOIs = new ObjectInspector[args.size()];
      funcArgOIs = argOIs.toArray(funcArgOIs);
    }

    GenericUDAFEvaluator wFnEval = FunctionRegistry.getGenericWindowingEvaluator(def.getName(),
        argOIs,
        def.isDistinct(), def.isStar());
    ObjectInspector OI = wFnEval.init(GenericUDAFEvaluator.Mode.COMPLETE, funcArgOIs);
    def.setWFnEval(wFnEval);
    def.setOI(OI);
  }

  private static void validateValueBoundaryExprType(ObjectInspector OI)
      throws SemanticException {
    if (!OI.getCategory().equals(Category.PRIMITIVE)) {
      throw new SemanticException(
          String.format(
              "Value Boundary expression must be of primitive type. Found: %s",
              OI.getTypeName()));
    }

    PrimitiveObjectInspector pOI = (PrimitiveObjectInspector) OI;
    PrimitiveCategory pC = pOI.getPrimitiveCategory();

    switch (pC)
    {
    case BOOLEAN:
    case BYTE:
    case DOUBLE:
    case FLOAT:
    case INT:
    case LONG:
    case SHORT:
    case DECIMAL:
    case TIMESTAMP:
    case DATE:
    case STRING:
    case VARCHAR:
    case CHAR:
      break;
    default:
      throw new SemanticException(
          String.format("Primitive type %s not supported in Value Boundary expression",
              pC));
    }

  }

  private ShapeDetails setupTableFnShape(String fnName, ShapeDetails inpShape,
      StructObjectInspector OI, List<String> columnNames, RowResolver rr)
      throws SemanticException {
    if (FunctionRegistry.isNoopFunction(fnName)) {
      return setupShapeForNoop(inpShape, OI, columnNames, rr);
    }
    return setupShape(OI, columnNames, rr);
  }

  private ShapeDetails setupShape(StructObjectInspector OI,
      List<String> columnNames,
      RowResolver rr) throws SemanticException {
    Map<String, String> serdePropsMap = new LinkedHashMap<String, String>();
    AbstractSerDe serde = null;
    ShapeDetails shp = new ShapeDetails();

    try {
      serde = PTFTranslator.createLazyBinarySerDe(hCfg, OI, serdePropsMap);
      StructObjectInspector outOI = PTFPartition.setupPartitionOutputOI(serde, OI);
      shp.setOI(outOI);
    } catch (SerDeException se) {
      throw new SemanticException(se);
    }

    shp.setRr(rr);
    shp.setSerde(serde);
    shp.setSerdeClassName(serde.getClass().getName());
    shp.setSerdeProps(serdePropsMap);
    shp.setColumnNames(columnNames);

    TypeCheckCtx tCtx = new TypeCheckCtx(rr);
    tCtx.setUnparseTranslator(unparseT);
    shp.setTypeCheckCtx(tCtx);

    return shp;
  }

  private ShapeDetails copyShape(ShapeDetails src) {
    ShapeDetails dest = new ShapeDetails();
    dest.setSerdeClassName(src.getSerdeClassName());
    dest.setSerdeProps(src.getSerdeProps());
    dest.setColumnNames(src.getColumnNames());
    dest.setOI(src.getOI());
    dest.setSerde(src.getSerde());
    dest.setRr(src.getRr());
    dest.setTypeCheckCtx(src.getTypeCheckCtx());

    return dest;
  }

  private ShapeDetails setupShapeForNoop(ShapeDetails inpShape,
      StructObjectInspector OI,
      List<String> columnNames,
      RowResolver rr) throws SemanticException {
    ShapeDetails shp = new ShapeDetails();

    shp.setRr(rr);
    shp.setOI(inpShape.getOI());
    shp.setSerde(inpShape.getSerde());
    shp.setSerdeClassName(inpShape.getSerde().getClass().getName());
    shp.setSerdeProps(inpShape.getSerdeProps());
    shp.setColumnNames(columnNames);

    TypeCheckCtx tCtx = new TypeCheckCtx(rr);
    tCtx.setUnparseTranslator(unparseT);
    shp.setTypeCheckCtx(tCtx);

    return shp;
  }

  protected static ArrayList<OrderExpression> addPartitionExpressionsToOrderList(
      ArrayList<PartitionExpression> partCols,
      ArrayList<OrderExpression> orderCols) throws SemanticException {
    int numOfPartColumns = 0;
    int chkSize = partCols.size();

    chkSize = chkSize > orderCols.size() ? orderCols.size() : chkSize;
    for (int i = 0; i < chkSize; i++) {
      if (orderCols.get(i).getExpression().toStringTree()
          .equals(partCols.get(i).getExpression().toStringTree())) {
        numOfPartColumns++;
      } else {
        break;
      }
    }

    if (numOfPartColumns != 0 && numOfPartColumns != partCols.size()) {
      List<String> partitionColumnNames = new ArrayList<String>();
      for (PartitionExpression partitionExpression : partCols) {
        ASTNode column = partitionExpression.getExpression();
        if (column != null && column.getChildCount() > 0) {
          partitionColumnNames.add(column.getChild(0).getText());
        }
      }
      throw new SemanticException(
          String.format(
              "all partition columns %s must be in order clause or none should be specified",
              partitionColumnNames.toString()));
    }
    ArrayList<OrderExpression> combinedOrdExprs = new ArrayList<OrderExpression>();
    if (numOfPartColumns == 0) {
      for (PartitionExpression partCol : partCols) {
        OrderExpression orderCol = new OrderExpression(partCol);
        combinedOrdExprs.add(orderCol);
      }
    }
    combinedOrdExprs.addAll(orderCols);
    return combinedOrdExprs;
  }

  private void setupRankingArgs(WindowTableFunctionDef wdwTFnDef,
      WindowFunctionDef wFnDef,
      WindowFunctionSpec wSpec)
      throws SemanticException {
    if (wSpec.getArgs().size() > 0) {
      throw new SemanticException("Ranking Functions can take no arguments");
    }
    OrderDef oDef = wdwTFnDef.getOrder();
    List<OrderExpressionDef> oExprs = oDef.getExpressions();
    for (OrderExpressionDef oExpr : oExprs) {
      wFnDef.addArg(oExpr);
    }
  }

  /*
   * Expr translation helper methods
   */
  public PTFExpressionDef buildExpressionDef(ShapeDetails inpShape, ASTNode arg)
      throws HiveException {
    PTFExpressionDef argDef = new PTFExpressionDef();

    ExprNodeDesc exprNode = semAly.genExprNodeDesc(arg, inpShape.getRr(),
        inpShape.getTypeCheckCtx());
    ExprNodeEvaluator exprEval = WindowingExprNodeEvaluatorFactory.get(llInfo, exprNode);
    ObjectInspector oi = initExprNodeEvaluator(exprEval, exprNode, inpShape);

    argDef.setExpressionTreeString(arg.toStringTree());
    argDef.setExprNode(exprNode);
    argDef.setExprEvaluator(exprEval);
    argDef.setOI(oi);
    return argDef;
  }

  private ObjectInspector initExprNodeEvaluator(ExprNodeEvaluator exprEval,
      ExprNodeDesc exprNode,
      ShapeDetails inpShape)
      throws HiveException {
    ObjectInspector outOI;
    outOI = exprEval.initialize(inpShape.getOI());

    /*
     * if there are any LeadLag functions in this Expression Tree: - setup a
     * duplicate Evaluator for the 1st arg of the LLFuncDesc - initialize it
     * using the InputInfo provided for this Expr tree - set the duplicate
     * evaluator on the LLUDF instance.
     */
    List<ExprNodeGenericFuncDesc> llFuncExprs = llInfo.getLLFuncExprsInTopExpr(exprNode);
    if (llFuncExprs != null)
    {
      for (ExprNodeGenericFuncDesc llFuncExpr : llFuncExprs)
      {
        ExprNodeDesc firstArg = llFuncExpr.getChildren().get(0);
        ExprNodeEvaluator dupExprEval = WindowingExprNodeEvaluatorFactory.get(llInfo, firstArg);
        dupExprEval.initialize(inpShape.getOI());
        GenericUDFLeadLag llFn = (GenericUDFLeadLag) llFuncExpr.getGenericUDF();
        llFn.setExprEvaluator(dupExprEval);
      }
    }

    return outOI;
  }

  /*
   * OI & Serde helper methods
   */

  protected static AbstractSerDe createLazyBinarySerDe(Configuration cfg,
      StructObjectInspector oi, Map<String, String> serdePropsMap) throws SerDeException {
    serdePropsMap = serdePropsMap == null ? new LinkedHashMap<String, String>() : serdePropsMap;

    PTFDeserializer.addOIPropertiestoSerDePropsMap(oi, serdePropsMap);

    AbstractSerDe serDe = new LazyBinarySerDe();
    Properties p = new Properties();
    p.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS,
        serdePropsMap.get(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS));
    p.setProperty(
        org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES,
        serdePropsMap.get(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES));
    SerDeUtils.initializeSerDe(serDe, cfg, p, null);
    return serDe;
  }

  @SuppressWarnings({"unchecked"})

  private static ArrayList<? extends Object>[] getTypeMap(
      StructObjectInspector oi) {
    StructTypeInfo t = (StructTypeInfo) TypeInfoUtils
        .getTypeInfoFromObjectInspector(oi);
    ArrayList<String> fnames = t.getAllStructFieldNames();
    ArrayList<TypeInfo> fields = t.getAllStructFieldTypeInfos();
    return new ArrayList<?>[] {fnames, fields};
  }

  /**
   * For each column on the input RR, construct a StructField for it
   * OI is constructed using the list of input column names and
   * their corresponding OIs.
   *
   * @param rr
   * @return
   */
  public static StructObjectInspector getStandardStructOI(RowResolver rr) {
    StructObjectInspector oi;
    ArrayList<ColumnInfo> colLists = rr.getColumnInfos();
    ArrayList<String> structFieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>();
    for (ColumnInfo columnInfo : colLists) {
      String colName = columnInfo.getInternalName();
      ObjectInspector colOI = columnInfo.getObjectInspector();
      structFieldNames.add(colName);
      structFieldObjectInspectors.add(colOI);
    }
    oi = ObjectInspectorFactory.getStandardStructObjectInspector(structFieldNames,
        structFieldObjectInspectors);
    return oi;

  }

  protected static void validateComparable(ObjectInspector OI, String errMsg)
      throws SemanticException {
    if (!ObjectInspectorUtils.compareSupported(OI)) {
      throw new SemanticException(errMsg);
    }
  }

  private static void addInputColumnsToList(ShapeDetails shape,
      ArrayList<String> fieldNames, ArrayList<ObjectInspector> fieldOIs) {
    StructObjectInspector OI = shape.getOI();
    for (StructField f : OI.getAllStructFieldRefs()) {
      fieldNames.add(f.getFieldName());
      fieldOIs.add(f.getFieldObjectInspector());
    }
  }

  /*
   * RowResolver helper methods
   */

  protected static RowResolver buildRowResolverForPTF(String tbFnName, String tabAlias,
      StructObjectInspector rowObjectInspector,
      List<String> outputColNames, RowResolver inputRR) throws SemanticException {

    if (FunctionRegistry.isNoopFunction(tbFnName)) {
      return buildRowResolverForNoop(tabAlias, rowObjectInspector, inputRR);
    }

    RowResolver rwsch = new RowResolver();
    List<? extends StructField> fields = rowObjectInspector.getAllStructFieldRefs();
    for (int i = 0; i < fields.size(); i++) {
      ColumnInfo colInfo = new ColumnInfo(fields.get(i).getFieldName(),
          TypeInfoUtils.getTypeInfoFromObjectInspector(fields.get(i)
              .getFieldObjectInspector()),
          tabAlias,
          false);
      rwsch.put(tabAlias, outputColNames.get(i), colInfo);
    }
    return rwsch;
  }

  protected RowResolver buildRowResolverForWindowing(WindowTableFunctionDef def)
      throws SemanticException {
    RowResolver rr = new RowResolver();
    HashMap<String, WindowExpressionSpec> aliasToExprMap = windowingSpec.getAliasToWdwExpr();

    /*
     * add Window Functions
     */
    for (WindowFunctionDef wFnDef : def.getWindowFunctions()) {
      ASTNode ast = aliasToExprMap.get(wFnDef.getAlias()).getExpression();
      ObjectInspector wFnOI = null;
      if (wFnDef.isPivotResult()) {
        wFnOI = ((ListObjectInspector) wFnDef.getOI()).getListElementObjectInspector();
      }
      else {
        wFnOI = wFnDef.getOI();
      }
      ColumnInfo cInfo = new ColumnInfo(wFnDef.getAlias(),
          TypeInfoUtils.getTypeInfoFromObjectInspector(wFnOI),
          null, true, true);
      rr.putExpression(ast, cInfo);
    }

    RowResolver inpRR = def.getRawInputShape().getRr();
    /*
     * add columns from inpRR
     */
    for (ColumnInfo inpCInfo : inputRR.getColumnInfos()) {
      ColumnInfo cInfo = new ColumnInfo(inpCInfo);

      ASTNode inExpr = PTFTranslator.getASTNode(inpCInfo, inpRR);
      if (inExpr != null) {
        rr.putExpression(inExpr, cInfo);
      } else {
        String[] tabColAlias = inputRR.reverseLookup(inpCInfo.getInternalName());
        if (tabColAlias != null) {
          rr.put(tabColAlias[0], tabColAlias[1], cInfo);
        } else {
          rr.put(inpCInfo.getTabAlias(), inpCInfo.getAlias(), cInfo);
        }
      }
      
      String[] altMapping = inputRR.getAlternateMappings(inpCInfo.getInternalName());
      if ( altMapping != null ) {
        rr.put(altMapping[0], altMapping[1], cInfo);
      }
    }

    return rr;
  }

  protected static RowResolver buildRowResolverForNoop(String tabAlias,
      StructObjectInspector rowObjectInspector,
      RowResolver inputRowResolver) throws SemanticException {
    LOG.info("QueryTranslationInfo::getRowResolver invoked on ObjectInspector");
    RowResolver rwsch = new RowResolver();
    List<? extends StructField> fields = rowObjectInspector.getAllStructFieldRefs();
    for (int i = 0; i < fields.size(); i++) {
      StructField field = fields.get(i);
      String internalName = field.getFieldName();
      String[] tabColAlias = inputRowResolver == null ? null : inputRowResolver
          .reverseLookup(internalName);
      String colTabAlias = tabColAlias == null ? tabAlias : tabColAlias[0];
      String colAlias = tabColAlias == null ? null : tabColAlias[1];
      ColumnInfo inpColInfo;
      ColumnInfo colInfo;

      if (tabColAlias != null) {
        inpColInfo = inputRowResolver.get(colTabAlias, colAlias);
      } else {
        /*
         * for the Virtual columns:
         * - the internalName is UPPER Case and the alias is lower case
         * - since we put them in an OI, the fieldnames became lower cased.
         * - so we look in the inputRR for the fieldName as an alias.
         */
        inpColInfo = inputRowResolver == null ? null : inputRowResolver
            .get(tabAlias, internalName);
        colAlias = inpColInfo != null ? inpColInfo.getInternalName() : colAlias;
      }

      if (inpColInfo != null) {
        colInfo = new ColumnInfo(inpColInfo);
        colInfo.setTabAlias(tabAlias);
      } else {
        colInfo = new ColumnInfo(fields.get(i).getFieldName(),
            TypeInfoUtils.getTypeInfoFromObjectInspector(fields.get(i)
                .getFieldObjectInspector()),
            tabAlias,
            false);
        colAlias = colInfo.getInternalName();
      }

      ASTNode expr = inputRowResolver == null ? null : PTFTranslator.getASTNode(inpColInfo,
          inputRowResolver);

      if (expr != null) {
        rwsch.putExpression(expr, colInfo);
      } else {
        rwsch.put(tabAlias, colAlias, colInfo);
      }
    }
    return rwsch;
  }

  /*
   * If the cInfo is for an ASTNode, this function returns the ASTNode that it is for.
   */
  public static ASTNode getASTNode(ColumnInfo cInfo, RowResolver rr) throws SemanticException {
    for (Map.Entry<String, ASTNode> entry : rr.getExpressionMap().entrySet()) {
      ASTNode expr = entry.getValue();
      if (rr.getExpression(expr).equals(cInfo)) {
        return expr;
      }
    }
    return null;
  }

  /*
   * Utility to visit all nodes in an AST tree.
   */
  public static void visit(Object t, ContextVisitor visitor) {
    _visit(t, null, 0, visitor);
  }

  /** Do the recursive work for visit */
  private static void _visit(Object t, Object parent, int childIndex, ContextVisitor visitor) {
    if (t == null) {
      return;
    }
    visitor.visit(t, parent, childIndex, null);
    int n = ParseDriver.adaptor.getChildCount(t);
    for (int i = 0; i < n; i++) {
      Object child = ParseDriver.adaptor.getChild(t, i);
      _visit(child, t, i, visitor);
    }
  }

  public static ArrayList<PTFInvocationSpec> componentize(PTFInvocationSpec ptfInvocation)
      throws SemanticException {

    ArrayList<PTFInvocationSpec> componentInvocations = new ArrayList<PTFInvocationSpec>();

    Stack<PTFInputSpec> ptfChain = new Stack<PTFInvocationSpec.PTFInputSpec>();
    PTFInputSpec spec = ptfInvocation.getFunction();
    while (spec instanceof PartitionedTableFunctionSpec) {
      ptfChain.push(spec);
      spec = spec.getInput();
    }

    PartitionedTableFunctionSpec prevFn = (PartitionedTableFunctionSpec) ptfChain.pop();
    applyConstantPartition(prevFn);
    PartitionSpec partSpec = prevFn.getPartition();
    OrderSpec orderSpec = prevFn.getOrder();

    if (partSpec == null) {
      // oops this should have been caught before trying to componentize
      throw new SemanticException(
          "No Partitioning specification specified at start of a PTFChain");
    }
    if (orderSpec == null) {
      orderSpec = new OrderSpec(partSpec);
      prevFn.setOrder(orderSpec);
    }

    while (!ptfChain.isEmpty()) {
      PartitionedTableFunctionSpec currentFn = (PartitionedTableFunctionSpec) ptfChain.pop();
      String fnName = currentFn.getName();
      if (!FunctionRegistry.isTableFunction(fnName)) {
        throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(fnName));
      }
      boolean transformsRawInput = FunctionRegistry.getTableFunctionResolver(fnName)
          .transformsRawInput();

      /*
       * if the current table function has no partition info specified: inherit it from the PTF up
       * the chain.
       */
      if (currentFn.getPartition() == null) {
        currentFn.setPartition(prevFn.getPartition());
        if (currentFn.getOrder() == null) {
          currentFn.setOrder(prevFn.getOrder());
        }
      }
      /*
       * If the current table function has no order info specified;
       */
      if (currentFn.getOrder() == null) {
        currentFn.setOrder(new OrderSpec(currentFn.getPartition()));
      }

      if (!currentFn.getPartition().equals(partSpec) ||
          !currentFn.getOrder().equals(orderSpec) ||
          transformsRawInput) {
        PTFInvocationSpec component = new PTFInvocationSpec();
        component.setFunction(prevFn);
        componentInvocations.add(component);
        PTFQueryInputSpec cQInSpec = new PTFQueryInputSpec();
        cQInSpec.setType(PTFQueryInputType.PTFCOMPONENT);
        currentFn.setInput(cQInSpec);
      }

      prevFn = currentFn;
      partSpec = prevFn.getPartition();
      orderSpec = prevFn.getOrder();
    }
    componentInvocations.add(ptfInvocation);
    return componentInvocations;
  }

  public static void validateNoLeadLagInValueBoundarySpec(ASTNode node)
      throws SemanticException {
    String errMsg = "Lead/Lag not allowed in ValueBoundary Spec";
    TreeWizard tw = new TreeWizard(ParseDriver.adaptor, HiveParser.tokenNames);
    ValidateNoLeadLag visitor = new ValidateNoLeadLag(errMsg);
    tw.visit(node, HiveParser.TOK_FUNCTION, visitor);
    visitor.checkValid();
  }

  public static class ValidateNoLeadLag implements ContextVisitor {
    String errMsg;
    boolean throwError = false;
    ASTNode errorNode;

    public ValidateNoLeadLag(String errMsg) {
      this.errMsg = errMsg;
    }

    @SuppressWarnings("rawtypes")
    @Override
    public void visit(Object t, Object parent, int childIndex, Map labels) {
      ASTNode expr = (ASTNode) t;
      ASTNode nameNode = (ASTNode) expr.getChild(0);
      if (nameNode.getText().equals(FunctionRegistry.LEAD_FUNC_NAME)
          || nameNode.getText()
              .equals(FunctionRegistry.LAG_FUNC_NAME)) {
        throwError = true;
        errorNode = expr;
      }
    }

    void checkValid() throws SemanticException {
      if (throwError) {
        throw new SemanticException(errMsg + errorNode.toStringTree());
      }
    }
  }
}
