/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hive.beeline;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.PrintStream;
import java.io.StringBufferInputStream;
import java.io.UnsupportedEncodingException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.google.common.base.Function;
import com.google.common.collect.Lists;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hive.jdbc.Utils;
import org.apache.hive.jdbc.miniHS2.MiniHS2;
import org.apache.hive.jdbc.miniHS2.MiniHS2.MiniClusterType;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;

/**
 * TestBeeLineWithArgs - executes tests of the command-line arguments to BeeLine
 *
 */
public class TestBeeLineWithArgs {
  private enum OutStream {
    ERR, OUT
  }

  // Default location of HiveServer2
  private static final String tableName = "TestBeelineTable1";
  private static final String tableComment = "Test table comment";
  private static MiniHS2 miniHS2;
  private static final String userName = System.getProperty("user.name");

  private List<String> getBaseArgs(String jdbcUrl) {
    List<String> argList = new ArrayList<>(8);
    argList.add("-d");
    argList.add(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER);
    argList.add("-u");
    argList.add(jdbcUrl);
    argList.add("-n");
    argList.add(userName);
    return argList;
  }

  /**
   * Start up a local Hive Server 2 for these tests
   */
  @BeforeClass
  public static void preTests() throws Exception {
    HiveConf hiveConf = new HiveConf();
    hiveConf.setVar(HiveConf.ConfVars.HIVE_LOCK_MANAGER,
        "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
    hiveConf.setBoolVar(HiveConf.ConfVars.HIVEOPTIMIZEMETADATAQUERIES, false);
    hiveConf.set(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "verbose");
    miniHS2 = new MiniHS2(hiveConf, MiniClusterType.TEZ);

    Map<String, String> confOverlay = new HashMap<String, String>();
    miniHS2.start(confOverlay);

    createTable();
  }

  /**
   * Create table for use by tests
   * @throws ClassNotFoundException
   * @throws SQLException
   */
  private static void createTable() throws ClassNotFoundException, SQLException {
    Class.forName(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER);
    Connection con = DriverManager.getConnection(miniHS2.getBaseJdbcURL(),
        userName , "");

    assertNotNull("Connection is null", con);
    assertFalse("Connection should not be closed", con.isClosed());
    Statement stmt = con.createStatement();
    assertNotNull("Statement is null", stmt);

    stmt.execute("set hive.support.concurrency = false");

    HiveConf conf = new HiveConf();
    String dataFileDir = conf.get("test.data.files").replace('\\', '/')
        .replace("c:", "");
    Path dataFilePath = new Path(dataFileDir, "kv1.txt");
    // drop table. ignore error.
    try {
      stmt.execute("drop table " + tableName);
    } catch (Exception ex) {
      fail(ex.toString() + " " + ExceptionUtils.getStackTrace(ex));
    }

    // create table
    stmt.execute("create table " + tableName
        + " (under_col int comment 'the under column', value string) comment '"
        + tableComment + "'");

    // load data
    stmt.execute("load data local inpath '"
        + dataFilePath.toString() + "' into table " + tableName);
  }

  /**
   * Shut down a local Hive Server 2 for these tests
   */
  @AfterClass
  public static void postTests() {
    if (miniHS2.isStarted()) {
      miniHS2.stop();
    }
  }

  /**
   * Execute a script with "beeline -f or -i"
   * @param argList List of arguments for beeline
   * @param inputStream input stream if any
   * @param streamType if output from STDERR or STDOUT needs to be returned
   * @return The stderr and stdout from running the script
   * @throws Throwable
   */
  private static String testCommandLineScript(List<String> argList, InputStream inputStream,
      OutStream streamType)
      throws Throwable {
    BeeLine beeLine = new BeeLine();
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    PrintStream beelineOutputStream = new PrintStream(os);
    switch (streamType) {
    case OUT:
      beeLine.setOutputStream(beelineOutputStream);
      break;
    case ERR:
      beeLine.setErrorStream(beelineOutputStream);
      break;
    default:
      throw new RuntimeException("Unexpected outstream type " + streamType);
    }
    String[] args = argList.toArray(new String[argList.size()]);
    beeLine.begin(args, inputStream);
    String output = os.toString("UTF8");

    beeLine.close();
    return output;
  }

  /**
   * Attempt to execute a simple script file with the -f and -i option to
   * BeeLine to test for presence of an expected pattern in the output (stdout
   * or stderr), fail if not found. Print PASSED or FAILED
   *
   * @param expectedRegex
   *          Text to look for in command output (stdout)
   * @param shouldMatch
   *          true if the pattern should be found, false if it should not
   * @throws Exception
   *           on command execution error
   */
  private void testScriptFile(String scriptText, List<String> argList, String expectedRegex,
      boolean shouldMatch) throws Throwable {
    testScriptFile(scriptText, argList, OutStream.OUT,
        Collections.singletonList(new Tuple<>(expectedRegex, shouldMatch))
    );
  }

  /**
   * Attempt to execute a simple script file with the -f and -i option
   * to BeeLine to test for presence of an expected pattern
   * in the output (stdout or stderr), fail if not found.
   * Print PASSED or FAILED
   * @param argList arguments
   * @param outType output stream type
   * @param expectedRegex Text to look for in command output (stdout)
   * @param shouldMatch true if the pattern should be found, false if it should not
   * @throws Throwable
   */
  private void testScriptFile(String scriptText, List<String> argList, OutStream outType,
      String expectedRegex, boolean shouldMatch) throws Throwable {
    testScriptFile(scriptText, argList, outType,
        Collections.singletonList(new Tuple<>(expectedRegex, shouldMatch))
    );
  }

  private void testScriptFile(String scriptText, List<String> argList, OutStream streamType,
      List<Tuple<String>> expectedMatches) throws Throwable {
    testScriptFile(scriptText, argList, streamType, expectedMatches,
        Arrays.asList(Modes.values()));
  }

  /**
   * Attempt to execute a simple script file with the -f or -i option
   * to BeeLine (or both) to  test for presence of an expected pattern
   * in the output (stdout or stderr), fail if not found.
   * Print PASSED or FAILED
   * @param scriptText script to test the output for
   * @param argList arguments to be passed to the script file to execute and produce output
   * @param streamType Whether match should be done against STDERR or STDOUT
   * @param expectedMatches List of Tuple's defining the pattern to match and result of matching
   * @param modes testing modes we have to run the script as
   * @throws Exception on command execution error
   */
  private void testScriptFile(String scriptText, List<String> argList,
      OutStream streamType, List<Tuple<String>> expectedMatches, List<Modes> modes)
      throws Throwable {
    // Put the script content in a temp file
    File scriptFile = File.createTempFile(this.getClass().getSimpleName(), "temp");
    System.out.println("script file is " + scriptFile.getAbsolutePath());
    scriptFile.deleteOnExit();
    PrintStream os = new PrintStream(new FileOutputStream(scriptFile));
    os.print(scriptText);
    os.close();

    List<Tuple<Pattern>> patternsToBeMatched = Lists.transform(expectedMatches,
        new Function<Tuple<String>, Tuple<Pattern>>() {
          @Override
          public Tuple<Pattern> apply(Tuple<String> tuple) {
            return new Tuple<>(
                Pattern.compile(".*" + tuple.pattern + ".*", Pattern.DOTALL),
                tuple.shouldMatch
            );
          }
        });

    for (Modes mode : modes) {
      String output = mode.output(scriptFile, argList, streamType);
      for (Tuple<Pattern> patternToMatch : patternsToBeMatched) {
        Matcher m = patternToMatch.pattern.matcher(output);
        boolean matches = m.matches();
        if (patternToMatch.shouldMatch != matches) {
          //failed
          fail("Output" + output + " should" + (patternToMatch.shouldMatch ? "" : " not") +
              " contain " + patternToMatch.pattern.pattern());
        }
      }
    }
    scriptFile.delete();
  }

  /*
    We are testing for both type of modes always so not passing that as a parameter for now
  */
  enum Modes {
    INIT {
      @Override
      String output(File scriptFile, List<String> argList, OutStream streamType) throws Throwable {
        List<String> copy = new ArrayList<>(argList);
        copy.add("-i");
        copy.add(scriptFile.getAbsolutePath());
        return testCommandLineScript(copy, new StringBufferInputStream("!quit\n"), streamType);
      }
    }, SCRIPT {
      @Override
      String output(File scriptFile, List<String> argList, OutStream streamType) throws Throwable {
        List<String> copy = new ArrayList<>(argList);
        copy.add("-f");
        copy.add(scriptFile.getAbsolutePath());
        return testCommandLineScript(copy, null, streamType);
      }
    };

    abstract String output(File scriptFile, List<String> argList, OutStream streamType)
        throws Throwable;
  }


  /**
   * Attempt to execute the enclosed query with the -e option to BeeLine
   * Test for presence of an expected pattern
   * in the output (stdout or stderr), fail if not found
   * Print PASSED or FAILED
   * @param expectedPattern Text to look for in command output/error
   * @param shouldMatch true if the pattern should be found, false if it should not
   * @throws Exception on command execution error
   */

  private void testCommandEnclosedQuery(String enclosedQuery, String expectedPattern,
      boolean shouldMatch, List<String> argList, OutStream out) throws Throwable {

    List<String> copy = new ArrayList<String>(argList);
    copy.add("-e");
    copy.add(enclosedQuery);

    String output = testCommandLineScript(copy, null, out);
    boolean matches = output.contains(expectedPattern);
    if (shouldMatch != matches) {
      //failed
      fail("Output" + output + " should" +  (shouldMatch ? "" : " not") +
          " contain " + expectedPattern);
    }
  }

  /**
   * Test that BeeLine will read comment lines that start with whitespace
   * @throws Throwable
   */
  @Test
  public void testWhitespaceBeforeCommentScriptFile() throws Throwable {
    final String SCRIPT_TEXT = " 	 	-- comment has spaces and tabs before it\n 	 	# comment has spaces and tabs before it\n";
    final String EXPECTED_PATTERN = "cannot recognize input near '<EOF>'";
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, false);
  }

  /**
   * Attempt to execute a simple script file with the -f option to BeeLine
   * Test for presence of an expected pattern
   * in the output (stdout or stderr), fail if not found
   * Print PASSED or FAILED
   */
  @Test
  public void testPositiveScriptFile() throws Throwable {
    final String SCRIPT_TEXT = "show databases;\n";
    final String EXPECTED_PATTERN = " default ";
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  /**
   * Fix to HIVE-10541: Beeline requires a newline at the end of each query in a file.
   * Otherwise, the last line of cmd in the script will be ignored.
   */
  @Test
  public void testLastLineCmdInScriptFile() throws Throwable {
    final String SCRIPT_TEXT = "show databases;\nshow tables;";
    final String EXPECTED_PATTERN = " testbeelinetable1 ";
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  /**
   * Test Beeline -hivevar option. User can specify --hivevar name=value on Beeline command line.
   * In the script, user should be able to use it in the form of ${name}, which will be substituted with
   * the value.
   * @throws Throwable
   */
  @Test
  public void testBeelineHiveVariable() throws Throwable {
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--hivevar");
    argList.add("DUMMY_TBL=dummy");
    final String SCRIPT_TEXT = "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table  ${DUMMY_TBL};";
    final String EXPECTED_PATTERN = "dummy";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  @Test
  public void testBeelineHiveConfVariable() throws Throwable {
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--hiveconf");
    argList.add("test.hive.table.name=dummy");
    final String SCRIPT_TEXT = "create table ${hiveconf:test.hive.table.name} (d int);\nshow tables;\n"
        + " drop table ${hiveconf:test.hive.table.name};\n";
    final String EXPECTED_PATTERN = "dummy";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  /**
   * Test Beeline -hivevar option. User can specify --hivevar name=value on Beeline command line.
   * This test defines multiple variables using repeated --hivevar or --hiveconf flags.
   * @throws Throwable
   */
  @Test
  public void testBeelineMultiHiveVariable() throws Throwable {
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--hivevar");
    argList.add("TABLE_NAME=dummy2");

    argList.add("--hiveconf");
    argList.add("COLUMN_NAME=d");

    argList.add("--hivevar");
    argList.add("COMMAND=create");
    argList.add("--hivevar");
    argList.add("OBJECT=table");

    argList.add("--hiveconf");
    argList.add("COLUMN_TYPE=int");

    final String SCRIPT_TEXT = "${COMMAND} ${OBJECT} ${TABLE_NAME} "
        + "(${hiveconf:COLUMN_NAME} ${hiveconf:COLUMN_TYPE});"
        + "\nshow tables;\n drop ${OBJECT} ${TABLE_NAME};\n";
    final String EXPECTED_PATTERN = "dummy2";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  /**
   * Attempt to execute a simple script file with the -f option to BeeLine
   * The first command should fail and the second command should not execute
   * Print PASSED or FAILED
   */
  @Test
  public void testBreakOnErrorScriptFile() throws Throwable {
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    final String SCRIPT_TEXT = "select * from abcdefg01;\nshow databases;\n";
    final String EXPECTED_PATTERN = " default ";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, false);
  }

  @Test
  public void testTabInScriptFile() throws Throwable {
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    final String SCRIPT_TEXT = "CREATE\tTABLE IF NOT EXISTS testTabInScriptFile\n(id\tint);\nSHOW TABLES;"
        + "\ndrop table testTabInScriptFile";
    final String EXPECTED_PATTERN = "testTabInScriptFile";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  @Test
  public void testBeelineShellCommand() throws Throwable {
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    final String SCRIPT_TEXT = "!sh echo \"hello world.\" > hw.txt\n!sh cat hw.txt\n!rm hw.txt";
    final String EXPECTED_PATTERN = "hello world";

    testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT,
        Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)),
        Collections.singletonList(Modes.SCRIPT)
    );
  }

  /**
   * Select null from table , check how null is printed
   * Print PASSED or FAILED
   */
  @Test
  public void testNullDefault() throws Throwable {
    final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
        "select null from " + tableName + " limit 1 ;\n";
    final String EXPECTED_PATTERN = "NULL";
    testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true);
  }

  /**
   * Select null from table , check if default null is printed differently
   * Print PASSED or FAILED
   */
  @Test
  public void testNullNonEmpty() throws Throwable {
    final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
        "!set nullemptystring false\n select null from " + tableName + " limit 1 ;\n";
    final String EXPECTED_PATTERN = "NULL";
    testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true);
  }

  @Test
  public void testGetVariableValue() throws Throwable {
    final String SCRIPT_TEXT = "set env:TERM;";
    final String EXPECTED_PATTERN = "env:TERM";
    testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true);
  }

  /**
   * Select null from table , check if setting null to empty string works.
   * Original beeline/sqlline used to print nulls as empty strings.
   * Also test csv2 output format
   * Print PASSED or FAILED
   */
  @Test
  public void testNullEmpty() throws Throwable {
    final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
        "!set nullemptystring true\n select 'abc',null,'def' from " + tableName + " limit 1 ;\n";
    final String EXPECTED_PATTERN = "abc,,def";

    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--outputformat=csv2");

    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  /**
   * Test writing output using DSV format, with custom delimiter ";"
   */
  @Test
  public void testDSVOutput() throws Throwable {
    String SCRIPT_TEXT = getFormatTestQuery();
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--outputformat=dsv");
    argList.add("--delimiterForDSV=;");

    final String EXPECTED_PATTERN = "1;NULL;defg;ab\"c;1.0";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  /**
   * Test writing output using TSV (new) format
   */
  @Test
  public void testTSV2Output() throws Throwable {
    String SCRIPT_TEXT = getFormatTestQuery();
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--outputformat=tsv2");

    final String EXPECTED_PATTERN = "1\tNULL\tdefg\tab\"c\t1.0";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  /**
   * Test writing output using TSV deprecated format
   */
  @Test
  public void testTSVOutput() throws Throwable {
    String SCRIPT_TEXT = getFormatTestQuery();
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--outputformat=tsv");

    final String EXPECTED_PATTERN = "'1'\t'NULL'\t'defg'\t'ab\"c\'\t'1.0'";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  /**
   * Test writing output using new TSV format
   */
  @Test
  public void testTSV2OutputWithDoubleQuotes() throws Throwable {
    String SCRIPT_TEXT = getFormatTestQueryForEableQuotes();
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--outputformat=tsv2");
    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV,"false");

    final String EXPECTED_PATTERN = "1\tNULL\tdefg\t\"ab\"\"c\"\t\"\"\"aa\"\"\"\t1.0";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true");
  }

  /**
   * Test writing output using TSV deprecated format
   */
  @Test
  public void testTSVOutputWithDoubleQuotes() throws Throwable {
    String SCRIPT_TEXT = getFormatTestQueryForEableQuotes();
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--outputformat=tsv");
    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false");

    final String EXPECTED_PATTERN = "'1'\t'NULL'\t'defg'\t'ab\"c'\t'\"aa\"'\t'1.0'";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true");
  }

  /**
   * Test writing output using new CSV format
   */
  @Test
  public void testCSV2OutputWithDoubleQuotes() throws Throwable {
    String SCRIPT_TEXT = getFormatTestQueryForEableQuotes();
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--outputformat=csv2");
    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false");

    final String EXPECTED_PATTERN = "1,NULL,defg,\"ab\"\"c\",\"\"\"aa\"\"\",1.0";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true");
  }

  /**
   * Test writing output using CSV deprecated format
   */
  @Test
  public void testCSVOutputWithDoubleQuotes() throws Throwable {
    String SCRIPT_TEXT = getFormatTestQueryForEableQuotes();
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--outputformat=csv");
    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false");

    final String EXPECTED_PATTERN = "'1','NULL','defg','ab\"c','\"aa\"','1.0'";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true");
  }

  /**
   * Test writing output using DSV format, with custom delimiter ";"
   */
  @Test
  public void testDSVOutputWithDoubleQuotes() throws Throwable {
    String SCRIPT_TEXT = getFormatTestQueryForEableQuotes();
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--outputformat=dsv");
    argList.add("--delimiterForDSV=;");
    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false");

    final String EXPECTED_PATTERN = "1;NULL;defg;\"ab\"\"c\";\"\"\"aa\"\"\";1.0";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true");
  }

  /**
   * Test writing output using TSV deprecated format
   * Check for deprecation message
   */
  @Test
  public void testTSVOutputDeprecation() throws Throwable {
    String SCRIPT_TEXT = getFormatTestQuery();
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--outputformat=tsv");

    final String EXPECTED_PATTERN = "Format tsv is deprecated, please use tsv2";
    testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true);
  }

  /**
   * Test writing output using CSV deprecated format
   * Check for deprecation message
   */
  @Test
  public void testCSVOutputDeprecation() throws Throwable {
    String SCRIPT_TEXT = getFormatTestQuery();
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--outputformat=csv");

    final String EXPECTED_PATTERN = "Format csv is deprecated, please use csv2";
    testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR,
        Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)));
  }

  /**
   * Test writing output using CSV deprecated format
   */
  @Test
  public void testCSVOutput() throws Throwable {
    String SCRIPT_TEXT = getFormatTestQuery();
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--outputformat=csv");
    final String EXPECTED_PATTERN = "'1','NULL','defg','ab\"c\','1.0'";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  private String getFormatTestQuery() {
    return "set hive.support.concurrency = false;\n" +
        "select 1, null, 'defg', 'ab\"c', 1.0D from " + tableName + " limit 1 ;\n";
  }

  private String getFormatTestQueryForEableQuotes() {
    return "set hive.support.concurrency = false;\n" +
        "select 1, null, 'defg', 'ab\"c', '\"aa\"', 1.0D from " + tableName + " limit 1 ;\n";
  }

  /**
   * Select null from table , check if setting null to empty string works - Using beeling cmd line
   *  argument.
   * Original beeline/sqlline used to print nulls as empty strings
   * Print PASSED or FAILED
   */
  @Test
  public void testNullEmptyCmdArg() throws Throwable {
    final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
        "select 'abc',null,'def' from " + tableName + " limit 1 ;\n";
    final String EXPECTED_PATTERN = "'abc','','def'";

    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--nullemptystring=true");
    argList.add("--outputformat=csv");

    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  /**
   * Attempt to execute a missing script file with the -f option to BeeLine
   */
  @Test
  public void testNegativeScriptFile() throws Throwable {
    final String EXPECTED_PATTERN = " default ";

    // Create and delete a temp file
    File scriptFile = File.createTempFile("beelinenegative", "temp");
    scriptFile.delete();

    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("-f");
    argList.add(scriptFile.getAbsolutePath());

    try {
      String output = testCommandLineScript(argList, null, OutStream.OUT);
      if (output.contains(EXPECTED_PATTERN)) {
        fail("Output: " + output +  " Negative pattern: " + EXPECTED_PATTERN);
      }
    } catch (Throwable e) {
      e.printStackTrace();
      throw e;
    }
  }

  /**
   * HIVE-4566
   * @throws UnsupportedEncodingException
   */
  @Test
  public void testNPE() throws UnsupportedEncodingException {
    BeeLine beeLine = new BeeLine();

    ByteArrayOutputStream os = new ByteArrayOutputStream();
    PrintStream beelineOutputStream = new PrintStream(os);
    beeLine.setOutputStream(beelineOutputStream);
    beeLine.setErrorStream(beelineOutputStream);

    beeLine.runCommands( new String[] {"!typeinfo"} );
    String output = os.toString("UTF8");
    Assert.assertFalse( output.contains("java.lang.NullPointerException") );
    assertTrue(output.contains("No current connection"));

    beeLine.runCommands( new String[] {"!nativesql"} );
    output = os.toString("UTF8");
    Assert.assertFalse( output.contains("java.lang.NullPointerException") );
    assertTrue(output.contains("No current connection"));

    System.out.println(">>> PASSED " + "testNPE" );
  }

  @Test
  public void testHiveVarSubstitution() throws Throwable {
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL() + "#D_TBL=dummy_t");
    final String SCRIPT_TEXT = "create table ${D_TBL} (d int);\nshow tables;\ndrop  table ${D_TBL};\n";
    final String EXPECTED_PATTERN = "dummy_t";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  @Test
  public void testEmbeddedBeelineConnection() throws Throwable{
    String embeddedJdbcURL = Utils.URL_PREFIX+"/Default";
    List<String> argList = getBaseArgs(embeddedJdbcURL);
    argList.add("--hivevar");
    argList.add("DUMMY_TBL=embedded_table");
    // Set to non-zk lock manager to avoid trying to connect to zookeeper
    final String SCRIPT_TEXT =
        "set hive.lock.manager=org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager;\n" +
            "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};\n";
    final String EXPECTED_PATTERN = "embedded_table";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  /**
   * Test Beeline could show the query progress for time-consuming query.
   * @throws Throwable
   */
  @Test
  public void testQueryProgress() throws Throwable {
    final String SCRIPT_TEXT =
        "set hive.support.concurrency = false;\n"
            + "set hive.server2.logging.operation.level=execution;\n"
            + "select count(*) from " + tableName + ";\n";
    // Check for part of log message as well as part of progress information
    final String EXPECTED_PATTERN = "ELAPSED TIME";
    final String UNEXPECTED_PATTERN = "(?=Reducer 2\\:).*(?=Map 1\\:)";
    testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR,
        Arrays.asList(
            new Tuple<>(EXPECTED_PATTERN, true),
            new Tuple<>(UNEXPECTED_PATTERN, false)
        )
    );
  }

  /**
   * Test Beeline could show the query progress for time-consuming query when hive.exec.parallel
   * is true
   *
   * We have changed the pattern to not look of the progress bar as the test runs fine individually
   * and also as part of the whole class, on CI however they are batched and that might have caused
   * some issue, it needs more investigation for the same
   *
   * @throws Throwable
   */
  @Test
  public void testQueryProgressParallel() throws Throwable {
    final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
        "set hive.exec.parallel = true;\n" +
        "select count(*) from " + tableName + ";\n";
    // Check for part of log message as well as part of progress information
    final String EXPECTED_PATTERN = "Number of reducers determined to be.";
    testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR,
        EXPECTED_PATTERN, true
    );
  }

  /**
   * Test Beeline will hide the query progress when silent option is set.
   * @throws Throwable
   */
  @Test
  public void testQueryProgressHidden() throws Throwable {
    final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
        "!set silent true\n" +
        "select count(*) from " + tableName + ";\n";
    final String EXPECTED_PATTERN = "Executing command";
    testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR,
        EXPECTED_PATTERN, false);
  }

  @Test
  public void testQueryProgressWithHiveServer2ProgressBarDisabled()
      throws Throwable {
    final String SCRIPT_TEXT =
        "set hive.support.concurrency = false;\nset hive.server2.in.place.progress=false;\n" +
            "select count(*) from " + tableName + ";\n";
    // Check for part of log message as well as part of progress information
    final String EXPECTED_PATTERN = "(?=Reducer 2\\:).*(?=Map 1\\:)";
    testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR,
        Arrays.asList(
            new Tuple<>(EXPECTED_PATTERN, true),
            new Tuple<>("ELAPSED TIME", false))
    );
  }

  @Test
  public void testMultiCommandsInOneline() throws Throwable {
    final String SCRIPT_TEXT = "drop table if exists multiCmdTbl;create table multiCmdTbl "
        +"(key int);show tables; --multicommands in one line";
    final String EXPECTED_PATTERN = " multicmdtbl ";
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);

    final String SCRIPT_TEXT_DROP = "drop table multiCmdTbl;show tables;";
    testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false);
  }

  @Test
  public void testMultiCommandsInOneEnclosedQuery() throws Throwable {
    final String QUERY_TEXT = "drop table if exists multiCmdTbl;create table multiCmdTbl "
        +"(key int);show tables; --multicommands in one line";
    final String EXPECTED_PATTERN = " multicmdtbl ";
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    testCommandEnclosedQuery(QUERY_TEXT, EXPECTED_PATTERN, true, argList, OutStream.OUT);

    final String QUERY_TEXT_DROP = "drop table multiCmdTbl;show tables;";
    testCommandEnclosedQuery(QUERY_TEXT_DROP, EXPECTED_PATTERN, false, argList, OutStream.OUT);
  }

  @Test
  public void testOneCommandInMultiLines() throws Throwable {
    final String SCRIPT_TEXT = "drop table if exists multiCmdTbl;create table \nmultiCmdTbl "
        + "(key int);show tables; --one command in multiple lines";
    final String EXPECTED_PATTERN = " multicmdtbl ";
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);

    final String SCRIPT_TEXT_DROP = "drop table\nmultiCmdTbl;show tables;";
    testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false);
  }

  @Test
  public void testEscapeSemiColonInQueries() throws Throwable {
    final String SCRIPT_TEXT = "drop table if exists multiCmdTbl;create table multiCmdTbl "
        + "(key int, value string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\;' LINES "
        + " TERMINATED BY '\\n';show tables; --one command in multiple lines";
    final String EXPECTED_PATTERN = " multicmdtbl ";
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);

    final String SCRIPT_TEXT_DROP = "drop table\nmultiCmdTbl;show tables;";
    testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false);
  }

  @Test
  public void testEscapeSemiColonInEnclosedQuery() throws Throwable {
    final String QUERY_TEXT = "drop table if exists multiCmdTbl;create table multiCmdTbl "
        + "(key int, value string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\;' LINES "
        + " TERMINATED BY '\\n';show tables;";
    final String EXPECTED_PATTERN = " multicmdtbl ";
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    testCommandEnclosedQuery(QUERY_TEXT, EXPECTED_PATTERN, true, argList, OutStream.OUT);

    final String QUERY_TEXT_DROP = "drop table multiCmdTbl;show tables;";
    testCommandEnclosedQuery(QUERY_TEXT_DROP, EXPECTED_PATTERN, false, argList, OutStream.OUT);
  }

  @Test
  public void testEmbeddedBeelineOutputs() throws Throwable{
    String embeddedJdbcURL = Utils.URL_PREFIX+"/Default";
    List<String> argList = getBaseArgs(embeddedJdbcURL);
    // Set to non-zk lock manager to avoid trying to connect to zookeeper
    final String SCRIPT_TEXT = "set hive.lock.manager=org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager;\n"
        + "set hive.compute.query.using.stats=false;\n"
        + "create table if not exists embeddedBeelineOutputs(d int);\n"
        + "set a=1;\nselect count(*) from embeddedBeelineOutputs;\n"
        + "drop table embeddedBeelineOutputs;\n";
    final String EXPECTED_PATTERN = "Stage-1 map =";
    testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true);
  }

  @Test
  public void testConnectionUrlWithSemiColon() throws Throwable{
    List<String> argList = getBaseArgs(miniHS2.getJdbcURL("default", "sess_var_list?var1=value1"));
    final String SCRIPT_TEXT = "set var1";
    final String EXPECTED_PATTERN = "var1=value1";
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  /**
   * Test Beeline !connect with beeline saved vars
   * @throws Throwable
   */
  @Test
  public void testBeelineConnectEnvVar() throws Throwable {
    final String jdbcUrl = miniHS2.getBaseJdbcURL();
    List<String> argList = new ArrayList<>();
    argList.add("-u");
    argList.add("blue");
    argList.add("-d");
    argList.add(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER);

    final String SCRIPT_TEXT =
        "create table blueconnecttest (d int);\nshow tables;\ndrop table blueconnecttest;\n";
    final String EXPECTED_PATTERN = "blueconnecttest";

    // We go through these hijinxes because java considers System.getEnv
    // to be read-only, and offers no way to set an env var from within
    // a process, only for processes that we sub-spawn.

    final BeeLineOpts.Env baseEnv = BeeLineOpts.getEnv();
    BeeLineOpts.Env newEnv = new BeeLineOpts.Env() {
      @Override
      public String get(String envVar) {
        if (envVar.equalsIgnoreCase("BEELINE_URL_BLUE")){
          return jdbcUrl;
        } else {
          return baseEnv.get(envVar);
        }
      }
    };
    BeeLineOpts.setEnv(newEnv);

    testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT,
        Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)),
        Collections.singletonList(Modes.SCRIPT));
  }

  /**
   * Test that if we !close, we can still !reconnect
   * @throws Throwable
   */
  @Test
  public void testBeelineReconnect() throws  Throwable {
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    final String SCRIPT_TEXT =
        "!close\n" +
            "!reconnect\n\n\n" +
            "create table reconnecttest (d int);\nshow tables;\ndrop table reconnecttest;\n";
    final String EXPECTED_PATTERN = "reconnecttest";

    testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT,
        Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)),
        Collections.singletonList(Modes.SCRIPT));

  }

  /**
   * Attempt to execute a simple script file with the usage of user & password variables in URL.
   * Test for presence of an expected pattern
   * in the output (stdout or stderr), fail if not found
   * Print PASSED or FAILED
   */
  @Test
  public void testConnectionWithURLParams() throws Throwable {
    final String EXPECTED_PATTERN = " hivetest ";
    List<String> argList = new ArrayList<>();
    argList.add("-d");
    argList.add(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER);
    argList.add("-u");
    argList.add(miniHS2.getBaseJdbcURL() + ";user=hivetest;password=hive");
    String SCRIPT_TEXT = "select current_user();";

    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  /**
   * Test that Beeline queries don't treat semicolons inside quotations as query-ending characters.
   */
  @Test
  public void testQueryNonEscapedSemiColon() throws Throwable {
    String SCRIPT_TEXT = "drop table if exists nonEscapedSemiColon;create table nonEscapedSemiColon "
        + "(key int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';';show tables;";
    final String EXPECTED_PATTERN = " nonEscapedSemiColon ";
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  @Test
  public void testSelectQueryWithNonEscapedSemiColon() throws Throwable {
    String SCRIPT_TEXT = "select ';', \"';'\", '\";\"', '\\';', ';\\'', '\\\";', ';\\\"' from " + tableName + ";";
    final String EXPECTED_PATTERN = ";\t';'\t\";\"\t';\t;'\t\";\t;\"";
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--outputformat=tsv2");
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  /**
   * Attempt to execute a simple script file with the usage of user & password variables in URL.
   * Test for presence of an expected pattern
   * in the output (stdout or stderr), fail if not found
   * Print PASSED or FAILED
   */
  @Test
  public void testShowDbInPrompt() throws Throwable {
    final String EXPECTED_PATTERN = " \\(default\\)>";
    List<String> argList = new ArrayList<>();
    argList.add("--showDbInPrompt");
    argList.add("-u");
    argList.add(miniHS2.getBaseJdbcURL() + ";user=hivetest;password=hive");
    String SCRIPT_TEXT = "select current_user();";

    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }

  @Test
  public void testBeelineShellCommandWithoutConn() throws Throwable {
    List<String> argList = new ArrayList<>();
    final String SCRIPT_TEXT = "!sh echo hello world";
    final String EXPECTED_PATTERN = "hello world";
    testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT,
        Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)),
        Collections.singletonList(Modes.SCRIPT));
  }

  /**
   * Attempt to execute Beeline with force option to continue running script even after errors.
   * Test for presence of an expected pattern to match the output of a valid command at the end.
   */
  @Test
  public void testBeelineWithForce() throws Throwable {
    final String SCRIPT_TEXT = "drop table does_not_exist;\ncreate table incomplete_syntax(a, string, );\n "
        + "drop table if exists new_table;\n create table new_table(foo int, bar string);\n "
        + "desc new_table;\n";
    final String EXPECTED_PATTERN = "2 rows selected";
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--force");
    testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true);
  }

  private static class Tuple<K> {
    final K pattern;
    final boolean shouldMatch;

    Tuple(K pattern, boolean shouldMatch) {
      this.pattern = pattern;
      this.shouldMatch = shouldMatch;
    }
  }

  /**
   * Test that Beeline can handle \\ characters within a string literal. Either at the beginning, middle, or end of the
   * literal.
   */
  @Test
  public void testBackslashInLiteral() throws Throwable {
    String SCRIPT_TEXT = "select 'hello\\\\', '\\\\hello', 'hel\\\\lo', '\\\\' as literal;";
    final String EXPECTED_PATTERN = "hello\\\\\t\\\\hello\thel\\\\lo\t\\\\";
    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
    argList.add("--outputformat=tsv2");
    testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
  }
}
