/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hive.metastore.txn;

import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
import org.apache.hadoop.hive.metastore.api.AddDynamicPartitions;
import org.apache.hadoop.hive.metastore.api.CheckLockRequest;
import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
import org.apache.hadoop.hive.metastore.api.CompactionRequest;
import org.apache.hadoop.hive.metastore.api.CompactionResponse;
import org.apache.hadoop.hive.metastore.api.CompactionType;
import org.apache.hadoop.hive.metastore.api.DataOperationType;
import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
import org.apache.hadoop.hive.metastore.api.HeartbeatRequest;
import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeRequest;
import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse;
import org.apache.hadoop.hive.metastore.api.LockComponent;
import org.apache.hadoop.hive.metastore.api.LockLevel;
import org.apache.hadoop.hive.metastore.api.LockRequest;
import org.apache.hadoop.hive.metastore.api.LockResponse;
import org.apache.hadoop.hive.metastore.api.LockState;
import org.apache.hadoop.hive.metastore.api.LockType;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchLockException;
import org.apache.hadoop.hive.metastore.api.NoSuchTxnException;
import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
import org.apache.hadoop.hive.metastore.api.ShowLocksRequest;
import org.apache.hadoop.hive.metastore.api.ShowLocksResponse;
import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement;
import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
import org.apache.hadoop.hive.metastore.api.TxnInfo;
import org.apache.hadoop.hive.metastore.api.TxnOpenException;
import org.apache.hadoop.hive.metastore.api.TxnState;
import org.apache.hadoop.hive.metastore.api.UnlockRequest;
import org.apache.hadoop.util.StringUtils;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.core.config.Configuration;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;

import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertNull;
import static junit.framework.Assert.assertTrue;
import static junit.framework.Assert.fail;

/**
 * Tests for TxnHandler.
 */
public class TestTxnHandler {
  static final private String CLASS_NAME = TxnHandler.class.getName();
  private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME);

  private HiveConf conf = new HiveConf();
  private TxnStore txnHandler;

  public TestTxnHandler() throws Exception {
    TxnDbUtil.setConfValues(conf);
    LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
    Configuration conf = ctx.getConfiguration();
    conf.getLoggerConfig(CLASS_NAME).setLevel(Level.DEBUG);
    ctx.updateLoggers(conf);
    tearDown();
  }

  @Test
  public void testValidTxnsEmpty() throws Exception {
    GetOpenTxnsInfoResponse txnsInfo = txnHandler.getOpenTxnsInfo();
    assertEquals(0L, txnsInfo.getTxn_high_water_mark());
    assertTrue(txnsInfo.getOpen_txns().isEmpty());
    GetOpenTxnsResponse txns = txnHandler.getOpenTxns();
    assertEquals(0L, txns.getTxn_high_water_mark());
    assertTrue(txns.getOpen_txns().isEmpty());
  }

  @Test
  public void testOpenTxn() throws Exception {
    long first = openTxn();
    assertEquals(1L, first);
    long second = openTxn();
    assertEquals(2L, second);
    GetOpenTxnsInfoResponse txnsInfo = txnHandler.getOpenTxnsInfo();
    assertEquals(2L, txnsInfo.getTxn_high_water_mark());
    assertEquals(2, txnsInfo.getOpen_txns().size());
    assertEquals(1L, txnsInfo.getOpen_txns().get(0).getId());
    assertEquals(TxnState.OPEN, txnsInfo.getOpen_txns().get(0).getState());
    assertEquals(2L, txnsInfo.getOpen_txns().get(1).getId());
    assertEquals(TxnState.OPEN, txnsInfo.getOpen_txns().get(1).getState());
    assertEquals("me", txnsInfo.getOpen_txns().get(1).getUser());
    assertEquals("localhost", txnsInfo.getOpen_txns().get(1).getHostname());

    GetOpenTxnsResponse txns = txnHandler.getOpenTxns();
    assertEquals(2L, txns.getTxn_high_water_mark());
    assertEquals(2, txns.getOpen_txns().size());
    boolean[] saw = new boolean[3];
    for (int i = 0; i < saw.length; i++) saw[i] = false;
    for (Long tid : txns.getOpen_txns()) {
      saw[tid.intValue()] = true;
    }
    for (int i = 1; i < saw.length; i++) assertTrue(saw[i]);
  }

  @Test
  public void testAbortTxn() throws Exception {
    OpenTxnsResponse openedTxns = txnHandler.openTxns(new OpenTxnRequest(3, "me", "localhost"));
    List<Long> txnList = openedTxns.getTxn_ids();
    long first = txnList.get(0);
    assertEquals(1L, first);
    long second = txnList.get(1);
    assertEquals(2L, second);
    txnHandler.abortTxn(new AbortTxnRequest(1));
    List<String> parts = new ArrayList<String>();
    parts.add("p=1");
    AddDynamicPartitions adp = new AddDynamicPartitions(3, "default", "T", parts);
    adp.setOperationType(DataOperationType.INSERT);
    txnHandler.addDynamicPartitions(adp);
    GetOpenTxnsInfoResponse txnsInfo = txnHandler.getOpenTxnsInfo();
    assertEquals(3, txnsInfo.getTxn_high_water_mark());
    assertEquals(3, txnsInfo.getOpen_txns().size());
    assertEquals(1L, txnsInfo.getOpen_txns().get(0).getId());
    assertEquals(TxnState.ABORTED, txnsInfo.getOpen_txns().get(0).getState());
    assertEquals(2L, txnsInfo.getOpen_txns().get(1).getId());
    assertEquals(TxnState.OPEN, txnsInfo.getOpen_txns().get(1).getState());
    assertEquals(3, txnsInfo.getOpen_txns().get(2).getId());
    assertEquals(TxnState.OPEN, txnsInfo.getOpen_txns().get(2).getState());

    GetOpenTxnsResponse txns = txnHandler.getOpenTxns();
    assertEquals(3, txns.getTxn_high_water_mark());
    assertEquals(3, txns.getOpen_txns().size());
    boolean[] saw = new boolean[4];
    for (int i = 0; i < saw.length; i++) saw[i] = false;
    for (Long tid : txns.getOpen_txns()) {
      saw[tid.intValue()] = true;
    }
    for (int i = 1; i < saw.length; i++) assertTrue(saw[i]);
    txnHandler.commitTxn(new CommitTxnRequest(2));
    //this succeeds as abortTxn is idempotent
    txnHandler.abortTxn(new AbortTxnRequest(1));
    boolean gotException = false;
    try {
      txnHandler.abortTxn(new AbortTxnRequest(2));
    }
    catch(NoSuchTxnException ex) {
      gotException = true;
      //if this wasn't an empty txn, we'd get a better msg
      Assert.assertEquals("No such transaction " + JavaUtils.txnIdToString(2), ex.getMessage());
    }
    Assert.assertTrue(gotException);
    gotException = false;
    txnHandler.commitTxn(new CommitTxnRequest(3));
    try {
      txnHandler.abortTxn(new AbortTxnRequest(3));
    }
    catch(NoSuchTxnException ex) {
      gotException = true;
      //txn 3 is not empty txn, so we get a better msg
      Assert.assertEquals("Transaction " + JavaUtils.txnIdToString(3) + " is already committed.", ex.getMessage());
    }
    Assert.assertTrue(gotException);
    
    gotException = false;
    try {
      txnHandler.abortTxn(new AbortTxnRequest(4));
    }
    catch(NoSuchTxnException ex) {
      gotException = true;
      Assert.assertEquals("No such transaction " + JavaUtils.txnIdToString(4), ex.getMessage());
    }
    Assert.assertTrue(gotException);
  }

  @Test
  public void testAbortInvalidTxn() throws Exception {
    boolean caught = false;
    try {
      txnHandler.abortTxn(new AbortTxnRequest(195L));
    } catch (NoSuchTxnException e) {
      caught = true;
    }
    assertTrue(caught);
  }

  @Test
  public void testValidTxnsNoneOpen() throws Exception {
    txnHandler.openTxns(new OpenTxnRequest(2, "me", "localhost"));
    txnHandler.commitTxn(new CommitTxnRequest(1));
    txnHandler.commitTxn(new CommitTxnRequest(2));
    GetOpenTxnsInfoResponse txnsInfo = txnHandler.getOpenTxnsInfo();
    assertEquals(2L, txnsInfo.getTxn_high_water_mark());
    assertEquals(0, txnsInfo.getOpen_txns().size());
    GetOpenTxnsResponse txns = txnHandler.getOpenTxns();
    assertEquals(2L, txns.getTxn_high_water_mark());
    assertEquals(0, txns.getOpen_txns().size());
  }

  @Test
  public void testValidTxnsSomeOpen() throws Exception {
    txnHandler.openTxns(new OpenTxnRequest(3, "me", "localhost"));
    txnHandler.abortTxn(new AbortTxnRequest(1));
    txnHandler.commitTxn(new CommitTxnRequest(2));
    GetOpenTxnsInfoResponse txnsInfo = txnHandler.getOpenTxnsInfo();
    assertEquals(3L, txnsInfo.getTxn_high_water_mark());
    assertEquals(2, txnsInfo.getOpen_txns().size());
    assertEquals(1L, txnsInfo.getOpen_txns().get(0).getId());
    assertEquals(TxnState.ABORTED, txnsInfo.getOpen_txns().get(0).getState());
    assertEquals(3L, txnsInfo.getOpen_txns().get(1).getId());
    assertEquals(TxnState.OPEN, txnsInfo.getOpen_txns().get(1).getState());

    GetOpenTxnsResponse txns = txnHandler.getOpenTxns();
    assertEquals(3L, txns.getTxn_high_water_mark());
    assertEquals(2, txns.getOpen_txns().size());
    boolean[] saw = new boolean[4];
    for (int i = 0; i < saw.length; i++) saw[i] = false;
    for (Long tid : txns.getOpen_txns()) {
      saw[tid.intValue()] = true;
    }
    assertTrue(saw[1]);
    assertFalse(saw[2]);
    assertTrue(saw[3]);
  }

  @Test
  public void testLockDifferentDBs() throws Exception {
    // Test that two different databases don't collide on their locks
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "yourdb");
    comp.setOperationType(DataOperationType.NO_TXN);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
  }

  @Test
  public void testLockSameDB() throws Exception {
    // Test that two different databases don't collide on their locks
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setOperationType(DataOperationType.NO_TXN);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);
  }

  @Test
  public void testLockDbLocksTable() throws Exception {
    // Test that locking a database prevents locking of tables in the database
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setOperationType(DataOperationType.NO_TXN);
    comp.setTablename("mytable");
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);
  }

  @Test
  public void testLockDbDoesNotLockTableInDifferentDB() throws Exception {
    // Test that locking a database prevents locking of tables in the database
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "yourdb");
    comp.setOperationType(DataOperationType.NO_TXN);
    comp.setTablename("mytable");
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
  }

  @Test
  public void testLockDifferentTables() throws Exception {
    // Test that two different tables don't collide on their locks
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setOperationType(DataOperationType.NO_TXN);
    comp.setTablename("mytable");
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setOperationType(DataOperationType.NO_TXN);
    comp.setTablename("yourtable");
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
  }

  @Test
  public void testLockSameTable() throws Exception {
    // Test that two different tables don't collide on their locks
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setOperationType(DataOperationType.NO_TXN);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);
  }

  @Test
  public void testLockTableLocksPartition() throws Exception {
    // Test that locking a table prevents locking of partitions of the table
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);
  }

  @Test
  public void testLockDifferentTableDoesntLockPartition() throws Exception {
    // Test that locking a table prevents locking of partitions of the table
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("yourtable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
  }

  @Test
  public void testLockDifferentPartitions() throws Exception {
    // Test that two different partitions don't collide on their locks
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("yourpartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
  }

  @Test
  public void testLockSamePartition() throws Exception {
    // Test that two different partitions don't collide on their locks
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);
  }

  @Test
  public void testLockSRSR() throws Exception {
    // Test that two shared read locks can share a partition
    LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.INSERT);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.SELECT);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
  }

  @Test
  public void testLockESRSR() throws Exception {
    // Test that exclusive lock blocks shared reads
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.INSERT);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);

    comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.SELECT);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);
  }

  @Test
  public void testLockSRSW() throws Exception {
    // Test that write can acquire after read
    LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.INSERT);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.DELETE);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    req.setTxnid(openTxn());
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
  }

  @Test
  public void testLockESRSW() throws Exception {
    // Test that exclusive lock blocks read and write
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.SELECT);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);

    comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.UPDATE);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    req.setTxnid(openTxn());
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);
  }

  @Test
  public void testLockSRE() throws Exception {
    // Test that read blocks exclusive
    LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.SELECT);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);
  }

  @Test
  public void testLockESRE() throws Exception {
    // Test that exclusive blocks read and exclusive
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.SELECT);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);

    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);
  }

  @Test
  public void testLockSWSR() throws Exception {
    // Test that read can acquire after write
    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.UPDATE);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    req.setTxnid(openTxn());
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.SELECT);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
  }

  @Test
  public void testLockSWSWSR() throws Exception {
    // Test that write blocks write but read can still acquire
    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.UPDATE);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    req.setTxnid(openTxn());
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.DELETE);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    req.setTxnid(openTxn());
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);

    comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.INSERT);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
  }

  @Test
  public void testWrongLockForOperation() throws Exception {
    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    req.setTxnid(openTxn());
    Exception expectedError = null;
    try {
      LockResponse res = txnHandler.lock(req);
    }
    catch(Exception e) {
      expectedError = e;
    }
    Assert.assertTrue(expectedError != null && expectedError.getMessage().contains("Unexpected DataOperationType"));
  }
  @Test
  public void testLockSWSWSW() throws Exception {
    // Test that write blocks two writes
    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.DELETE);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    req.setTxnid(openTxn());
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.DELETE);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    req.setTxnid(openTxn());
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);

    comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.DELETE);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    req.setTxnid(openTxn());
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);
  }

  @Test
  public void testLockEESW() throws Exception {
    // Test that exclusive blocks exclusive and write
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);

    comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.DELETE);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    req.setTxnid(openTxn());
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);
  }

  @Test
  public void testLockEESR() throws Exception {
    // Test that exclusive blocks exclusive and read
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);

    comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.SELECT);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);
  }

  @Test
  public void testCheckLockAcquireAfterWaiting() throws Exception {
    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.DELETE);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    long txnId = openTxn();
    req.setTxnid(txnId);
    LockResponse res = txnHandler.lock(req);
    long lockid1 = res.getLockid();
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.UPDATE);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    req.setTxnid(openTxn());
    res = txnHandler.lock(req);
    long lockid2 = res.getLockid();
    assertTrue(res.getState() == LockState.WAITING);

    txnHandler.abortTxn(new AbortTxnRequest(txnId));
    res = txnHandler.checkLock(new CheckLockRequest(lockid2));
    assertTrue(res.getState() == LockState.ACQUIRED);
  }

  @Test
  public void testCheckLockNoSuchLock() throws Exception {
    try {
      txnHandler.checkLock(new CheckLockRequest(23L));
      fail("Allowed to check lock on non-existent lock");
    } catch (NoSuchLockException e) {
    }
  }

  @Test
  public void testCheckLockTxnAborted() throws Exception {
    // Test that when a transaction is aborted, the heartbeat fails
    long txnid = openTxn();
    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.DELETE);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    LockResponse res = txnHandler.lock(req);
    long lockid = res.getLockid();
    txnHandler.abortTxn(new AbortTxnRequest(txnid));
    try {
      // This will throw NoSuchLockException (even though it's the
      // transaction we've closed) because that will have deleted the lock.
      txnHandler.checkLock(new CheckLockRequest(lockid));
      fail("Allowed to check lock on aborted transaction.");
    } catch (NoSuchLockException e) {
    }
  }

  @Test
  public void testMultipleLock() throws Exception {
    // Test more than one lock can be handled in a lock request
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(2);
    components.add(comp);

    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("anotherpartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    long lockid = res.getLockid();
    assertTrue(res.getState() == LockState.ACQUIRED);
    res = txnHandler.checkLock(new CheckLockRequest(lockid));
    assertTrue(res.getState() == LockState.ACQUIRED);
    txnHandler.unlock(new UnlockRequest(lockid));
    assertEquals(0, txnHandler.numLocksInLockTable());
  }

  @Test
  public void testMultipleLockWait() throws Exception {
    // Test that two shared read locks can share a partition
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(2);
    components.add(comp);

    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("anotherpartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    long lockid1 = res.getLockid();
    assertTrue(res.getState() == LockState.ACQUIRED);


    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    components = new ArrayList<LockComponent>(1);
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    long lockid2 = res.getLockid();
    assertTrue(res.getState() == LockState.WAITING);

    txnHandler.unlock(new UnlockRequest(lockid1));

    res = txnHandler.checkLock(new CheckLockRequest(lockid2));
    assertTrue(res.getState() == LockState.ACQUIRED);
  }

  @Test
  public void testUnlockOnCommit() throws Exception {
    // Test that committing unlocks
    long txnid = openTxn();
    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB,  "mydb");
    comp.setTablename("mytable");
    comp.setOperationType(DataOperationType.DELETE);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
    txnHandler.commitTxn(new CommitTxnRequest(txnid));
    assertEquals(0, txnHandler.numLocksInLockTable());
  }

  @Test
  public void testUnlockOnAbort() throws Exception {
    // Test that committing unlocks
    long txnid = openTxn();
    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setOperationType(DataOperationType.UPDATE);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
    txnHandler.abortTxn(new AbortTxnRequest(txnid));
    assertEquals(0, txnHandler.numLocksInLockTable());
  }

  @Test
  public void testUnlockWithTxn() throws Exception {
    LOG.debug("Starting testUnlockWithTxn");
    // Test that attempting to unlock locks associated with a transaction
    // generates an error
    long txnid = openTxn();
    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.DELETE);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    LockResponse res = txnHandler.lock(req);
    long lockid = res.getLockid();
    try {
      txnHandler.unlock(new UnlockRequest(lockid));
      fail("Allowed to unlock lock associated with transaction.");
    } catch (TxnOpenException e) {
    }
  }

  @Test
  public void testHeartbeatTxnAborted() throws Exception {
    // Test that when a transaction is aborted, the heartbeat fails
    openTxn();
    txnHandler.abortTxn(new AbortTxnRequest(1));
    HeartbeatRequest h = new HeartbeatRequest();
    h.setTxnid(1);
    try {
      txnHandler.heartbeat(h);
      fail("Told there was a txn, when it should have been aborted.");
    } catch (TxnAbortedException e) {
    }
 }

  @Test
  public void testHeartbeatNoTxn() throws Exception {
    // Test that when a transaction is aborted, the heartbeat fails
    HeartbeatRequest h = new HeartbeatRequest();
    h.setTxnid(939393L);
    try {
      txnHandler.heartbeat(h);
      fail("Told there was a txn, when there wasn't.");
    } catch (NoSuchTxnException e) {
    }
  }

  @Test
  public void testHeartbeatLock() throws Exception {
    conf.setTimeVar(HiveConf.ConfVars.HIVE_TXN_TIMEOUT, 1, TimeUnit.SECONDS);
    HeartbeatRequest h = new HeartbeatRequest();
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
    h.setLockid(res.getLockid());
    for (int i = 0; i < 30; i++) {
      try {
        txnHandler.heartbeat(h);
      } catch (NoSuchLockException e) {
        fail("Told there was no lock, when the heartbeat should have kept it.");
      }
    }
  }

  @Test
  public void heartbeatTxnRange() throws Exception {
    long txnid = openTxn();
    assertEquals(1, txnid);
    txnid = openTxn();
    txnid = openTxn();
    HeartbeatTxnRangeResponse rsp =
        txnHandler.heartbeatTxnRange(new HeartbeatTxnRangeRequest(1, 3));
    assertEquals(0, rsp.getAborted().size());
    assertEquals(0, rsp.getNosuch().size());
  }

  @Test
  public void heartbeatTxnRangeOneCommitted() throws Exception {
    long txnid = openTxn();
    assertEquals(1, txnid);
    txnHandler.commitTxn(new CommitTxnRequest(1));
    txnid = openTxn();
    txnid = openTxn();
    HeartbeatTxnRangeResponse rsp =
      txnHandler.heartbeatTxnRange(new HeartbeatTxnRangeRequest(1, 3));
    assertEquals(1, rsp.getNosuchSize());
    Long txn = rsp.getNosuch().iterator().next();
    assertEquals(1L, (long)txn);
    assertEquals(0, rsp.getAborted().size());
  }

  @Test
  public void heartbeatTxnRangeOneAborted() throws Exception {
    long txnid = openTxn();
    assertEquals(1, txnid);
    txnid = openTxn();
    txnid = openTxn();
    txnHandler.abortTxn(new AbortTxnRequest(3));
    HeartbeatTxnRangeResponse rsp =
      txnHandler.heartbeatTxnRange(new HeartbeatTxnRangeRequest(1, 3));
    assertEquals(1, rsp.getAbortedSize());
    Long txn = rsp.getAborted().iterator().next();
    assertEquals(3L, (long)txn);
    assertEquals(0, rsp.getNosuch().size());
  }

  @Test
  public void testLockTimeout() throws Exception {
    long timeout = txnHandler.setTimeout(1);
    try {
      LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
      comp.setTablename("mytable");
      comp.setPartitionname("mypartition");
      comp.setOperationType(DataOperationType.NO_TXN);
      List<LockComponent> components = new ArrayList<LockComponent>(1);
      components.add(comp);
      LockRequest req = new LockRequest(components, "me", "localhost");
      LockResponse res = txnHandler.lock(req);
      assertTrue(res.getState() == LockState.ACQUIRED);
      Thread.sleep(10);
      txnHandler.performTimeOuts();
      txnHandler.checkLock(new CheckLockRequest(res.getLockid()));
      fail("Told there was a lock, when it should have timed out.");
    } catch (NoSuchLockException e) {
    } finally {
      txnHandler.setTimeout(timeout);
    }
  }

  @Test
  public void testRecoverManyTimeouts() throws Exception {
    long timeout = txnHandler.setTimeout(1);
    try {
      txnHandler.openTxns(new OpenTxnRequest(503, "me", "localhost"));
      Thread.sleep(10);
      txnHandler.performTimeOuts();
      GetOpenTxnsInfoResponse rsp = txnHandler.getOpenTxnsInfo();
      int numAborted = 0;
      for (TxnInfo txnInfo : rsp.getOpen_txns()) {
        assertEquals(TxnState.ABORTED, txnInfo.getState());
        numAborted++;
      }
      assertEquals(503, numAborted);
    } finally {
      txnHandler.setTimeout(timeout);
    }


  }

  @Test
  public void testHeartbeatNoLock() throws Exception {
    HeartbeatRequest h = new HeartbeatRequest();
    h.setLockid(29389839L);
    try {
      txnHandler.heartbeat(h);
      fail("Told there was a lock, when there wasn't.");
    } catch (NoSuchLockException e) {
    }
  }

  @Test
  public void testCompactMajorWithPartition() throws Exception {
    CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MAJOR);
    rqst.setPartitionname("ds=today");
    txnHandler.compact(rqst);
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    assertEquals(1, compacts.size());
    ShowCompactResponseElement c = compacts.get(0);
    assertEquals("foo", c.getDbname());
    assertEquals("bar", c.getTablename());
    assertEquals("ds=today", c.getPartitionname());
    assertEquals(CompactionType.MAJOR, c.getType());
    assertEquals("initiated", c.getState());
    assertEquals(0L, c.getStart());
  }

  /**
   * Once a Compaction for a given resource is scheduled/working, we should not
   * schedule another one to prevent concurrent compactions for the same resource.
   * @throws Exception
   */
  @Test
  public void testCompactWhenAlreadyCompacting() throws Exception {
    CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MAJOR);
    rqst.setPartitionname("ds=today");
    CompactionResponse resp = txnHandler.compact(rqst);
    Assert.assertEquals(resp, new CompactionResponse(1, TxnStore.INITIATED_RESPONSE, true));

    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    assertEquals(1, compacts.size());

    rqst.setType(CompactionType.MINOR);
    resp = txnHandler.compact(rqst);
    Assert.assertEquals(resp, new CompactionResponse(1, TxnStore.INITIATED_RESPONSE, false));

    rsp = txnHandler.showCompact(new ShowCompactRequest());
    compacts = rsp.getCompacts();
    assertEquals(1, compacts.size());
    ShowCompactResponseElement c = compacts.get(0);
    assertEquals("foo", c.getDbname());
    assertEquals("bar", c.getTablename());
    assertEquals("ds=today", c.getPartitionname());
    assertEquals(CompactionType.MAJOR, c.getType());
    assertEquals("initiated", c.getState());
    assertEquals(0L, c.getStart());
  }

  @Test
  public void testCompactMinorNoPartition() throws Exception {
    CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
    rqst.setRunas("fred");
    txnHandler.compact(rqst);

    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    assertEquals(1, compacts.size());
    ShowCompactResponseElement c = compacts.get(0);
    assertEquals("foo", c.getDbname());
    assertEquals("bar", c.getTablename());
    assertNull(c.getPartitionname());
    assertEquals(CompactionType.MINOR, c.getType());
    assertEquals("initiated", c.getState());
    assertEquals(0L, c.getStart());
    assertEquals("fred", c.getRunAs());
  }

  @Test
  public void showLocks() throws Exception {
    long begining = System.currentTimeMillis();
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);

    // Open txn
    long txnid = openTxn();
    comp = new LockComponent(LockType.SHARED_READ, LockLevel.TABLE, "mydb");
    comp.setTablename("mytable");
    comp.setOperationType(DataOperationType.SELECT);
    components = new ArrayList<LockComponent>(1);
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    res = txnHandler.lock(req);

    // Locks not associated with a txn
    components = new ArrayList<LockComponent>(1);
    comp = new LockComponent(LockType.SHARED_READ, LockLevel.PARTITION, "yourdb");
    comp.setTablename("yourtable");
    comp.setPartitionname("yourpartition");
    comp.setOperationType(DataOperationType.INSERT);
    components.add(comp);
    req = new LockRequest(components, "you", "remotehost");
    res = txnHandler.lock(req);

    ShowLocksResponse rsp = txnHandler.showLocks(new ShowLocksRequest());
    List<ShowLocksResponseElement> locks = rsp.getLocks();
    assertEquals(3, locks.size());
    boolean[] saw = new boolean[locks.size()];
    for (int i = 0; i < saw.length; i++) saw[i] = false;
    for (ShowLocksResponseElement lock : locks) {
      if (lock.getLockid() == 1) {
        assertEquals(0, lock.getTxnid());
        assertEquals("mydb", lock.getDbname());
        assertNull(lock.getTablename());
        assertNull(lock.getPartname());
        assertEquals(LockState.ACQUIRED, lock.getState());
        assertEquals(LockType.EXCLUSIVE, lock.getType());
        assertTrue(lock.toString(), 0 != lock.getLastheartbeat());
        assertTrue("Expected acquired at " + lock.getAcquiredat() + " to be between " + begining
            + " and " + System.currentTimeMillis(),
            begining <= lock.getAcquiredat() && System.currentTimeMillis() >= lock.getAcquiredat());
        assertEquals("me", lock.getUser());
        assertEquals("localhost", lock.getHostname());
        saw[0] = true;
      } else if (lock.getLockid() == 2) {
        assertEquals(1, lock.getTxnid());
        assertEquals("mydb", lock.getDbname());
        assertEquals("mytable", lock.getTablename());
        assertNull(lock.getPartname());
        assertEquals(LockState.WAITING, lock.getState());
        assertEquals(LockType.SHARED_READ, lock.getType());
        assertTrue(lock.toString(), 0 == lock.getLastheartbeat() &&
          lock.getTxnid() != 0);
        assertEquals(0, lock.getAcquiredat());
        assertEquals("me", lock.getUser());
        assertEquals("localhost", lock.getHostname());
        saw[1] = true;
      } else if (lock.getLockid() == 3) {
        assertEquals(0, lock.getTxnid());
        assertEquals("yourdb", lock.getDbname());
        assertEquals("yourtable", lock.getTablename());
        assertEquals("yourpartition", lock.getPartname());
        assertEquals(LockState.ACQUIRED, lock.getState());
        assertEquals(LockType.SHARED_READ, lock.getType());
        assertTrue(lock.toString(), begining <= lock.getLastheartbeat() &&
            System.currentTimeMillis() >= lock.getLastheartbeat());
        assertTrue(begining <= lock.getAcquiredat() &&
            System.currentTimeMillis() >= lock.getAcquiredat());
        assertEquals("you", lock.getUser());
        assertEquals("remotehost", lock.getHostname());
        saw[2] = true;
      } else {
        fail("Unknown lock id");
      }
    }
    for (int i = 0; i < saw.length; i++) assertTrue("Didn't see lock id " + i, saw[i]);
  }

  @Test
  @Ignore("Wedges Derby")
  public void deadlockDetected() throws Exception {
    LOG.debug("Starting deadlock test");
    if (txnHandler instanceof TxnHandler) {
      final TxnHandler tHndlr = (TxnHandler)txnHandler;
      Connection conn = tHndlr.getDbConn(Connection.TRANSACTION_SERIALIZABLE);
      Statement stmt = conn.createStatement();
      long now = tHndlr.getDbTime(conn);
      stmt.executeUpdate("insert into TXNS (txn_id, txn_state, txn_started, txn_last_heartbeat, " +
          "txn_user, txn_host) values (1, 'o', " + now + ", " + now + ", 'shagy', " +
          "'scooby.com')");
      stmt.executeUpdate("insert into HIVE_LOCKS (hl_lock_ext_id, hl_lock_int_id, hl_txnid, " +
          "hl_db, hl_table, hl_partition, hl_lock_state, hl_lock_type, hl_last_heartbeat, " +
          "hl_user, hl_host) values (1, 1, 1, 'mydb', 'mytable', 'mypartition', '" +
          tHndlr.LOCK_WAITING + "', '" + tHndlr.LOCK_EXCLUSIVE + "', " + now + ", 'fred', " +
          "'scooby.com')");
      conn.commit();
      tHndlr.closeDbConn(conn);

      final AtomicBoolean sawDeadlock = new AtomicBoolean();

      final Connection conn1 = tHndlr.getDbConn(Connection.TRANSACTION_SERIALIZABLE);
      final Connection conn2 = tHndlr.getDbConn(Connection.TRANSACTION_SERIALIZABLE);
      try {

        for (int i = 0; i < 5; i++) {
          Thread t1 = new Thread() {
            @Override
            public void run() {
              try {
                try {
                  updateTxns(conn1);
                  updateLocks(conn1);
                  Thread.sleep(1000);
                  conn1.commit();
                  LOG.debug("no exception, no deadlock");
                } catch (SQLException e) {
                  try {
                    tHndlr.checkRetryable(conn1, e, "thread t1");
                    LOG.debug("Got an exception, but not a deadlock, SQLState is " +
                        e.getSQLState() + " class of exception is " + e.getClass().getName() +
                        " msg is <" + e.getMessage() + ">");
                  } catch (TxnHandler.RetryException de) {
                    LOG.debug("Forced a deadlock, SQLState is " + e.getSQLState() + " class of " +
                        "exception is " + e.getClass().getName() + " msg is <" + e
                        .getMessage() + ">");
                    sawDeadlock.set(true);
                  }
                }
                conn1.rollback();
              } catch (Exception e) {
                throw new RuntimeException(e);
              }
            }
          };

          Thread t2 = new Thread() {
            @Override
            public void run() {
              try {
                try {
                  updateLocks(conn2);
                  updateTxns(conn2);
                  Thread.sleep(1000);
                  conn2.commit();
                  LOG.debug("no exception, no deadlock");
                } catch (SQLException e) {
                  try {
                    tHndlr.checkRetryable(conn2, e, "thread t2");
                    LOG.debug("Got an exception, but not a deadlock, SQLState is " +
                        e.getSQLState() + " class of exception is " + e.getClass().getName() +
                        " msg is <" + e.getMessage() + ">");
                  } catch (TxnHandler.RetryException de) {
                    LOG.debug("Forced a deadlock, SQLState is " + e.getSQLState() + " class of " +
                        "exception is " + e.getClass().getName() + " msg is <" + e
                        .getMessage() + ">");
                    sawDeadlock.set(true);
                  }
                }
                conn2.rollback();
              } catch (Exception e) {
                throw new RuntimeException(e);
              }
            }
          };

          t1.start();
          t2.start();
          t1.join();
          t2.join();
          if (sawDeadlock.get()) break;
        }
        assertTrue(sawDeadlock.get());
      } finally {
        conn1.rollback();
        tHndlr.closeDbConn(conn1);
        conn2.rollback();
        tHndlr.closeDbConn(conn2);
      }
    }
  }

  /**
   * This cannnot be run against Derby (thus in UT) but it can run againt MySQL.
   * 1. add to metastore/pom.xml
   *     <dependency>
   *      <groupId>mysql</groupId>
   *      <artifactId>mysql-connector-java</artifactId>
   *      <version>5.1.30</version>
   *     </dependency>
   * 2. Hack in the c'tor of this class
   *     conf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, "jdbc:mysql://localhost/metastore");
   *      conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME, "hive");
   *      conf.setVar(HiveConf.ConfVars.METASTOREPWD, "hive");
   *      conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, "com.mysql.jdbc.Driver");
   * 3. Remove TxnDbUtil.prepDb(); in TxnHandler.checkQFileTestHack()
   *      
   */
  @Ignore("multiple threads wedge Derby")
  @Test
  public void testMutexAPI() throws Exception {
    final TxnStore.MutexAPI api =  txnHandler.getMutexAPI();
    final AtomicInteger stepTracker = new AtomicInteger(0);
    /**
     * counter = 0;
     * Thread1 counter=1, lock, wait 3s, check counter(should be 2), counter=3, unlock 
     * Thread2 counter=2, lock (and block), inc counter, should be 4
     */
    Thread t1 = new Thread("MutexTest1") {
      public void run() {
        try {
          stepTracker.incrementAndGet();//now 1
          TxnStore.MutexAPI.LockHandle handle = api.acquireLock(TxnHandler.MUTEX_KEY.HouseKeeper.name());
          Thread.sleep(4000);
          //stepTracker should now be 2 which indicates t2 has started
          Assert.assertEquals("Thread2 should have started by now but not done work", 2, stepTracker.get());
          stepTracker.incrementAndGet();//now 3
          handle.releaseLocks();
        }
        catch(Exception ex) {
          throw new RuntimeException(ex.getMessage(), ex);
        }
      }
    };
    t1.setDaemon(true);
    ErrorHandle ueh1 = new ErrorHandle();
    t1.setUncaughtExceptionHandler(ueh1);
    Thread t2 = new Thread("MutexTest2") {
      public void run() {
        try {
          stepTracker.incrementAndGet();//now 2
          //this should block until t1 unlocks
          TxnStore.MutexAPI.LockHandle handle = api.acquireLock(TxnHandler.MUTEX_KEY.HouseKeeper.name());
          stepTracker.incrementAndGet();//now 4
          Assert.assertEquals(4, stepTracker.get());
          handle.releaseLocks();
          stepTracker.incrementAndGet();//now 5
        }
        catch(Exception ex) {
          throw new RuntimeException(ex.getMessage(), ex);
        }
      }
    };
    t2.setDaemon(true);
    ErrorHandle ueh2 = new ErrorHandle();
    t2.setUncaughtExceptionHandler(ueh2);
    t1.start();
    try {
      Thread.sleep(1000);
    }
    catch(InterruptedException ex) {
      LOG.info("Sleep was interrupted");
    }
    t2.start();
    t1.join(6000);//so that test doesn't block
    t2.join(6000);

    if(ueh1.error != null) {
      Assert.assertTrue("Unexpected error from t1: " + StringUtils.stringifyException(ueh1.error), false);
    }
    if (ueh2.error != null) {
      Assert.assertTrue("Unexpected error from t2: " + StringUtils.stringifyException(ueh2.error), false);
    }
    Assert.assertEquals("5 means both threads have completed", 5, stepTracker.get());
  }
  private final static class ErrorHandle implements Thread.UncaughtExceptionHandler {
    Throwable error = null;
    @Override
    public void uncaughtException(Thread t, Throwable e) {
      LOG.error("Uncaught exception from " + t.getName() + ": " + e.getMessage());
      error = e;
    }
  }

  @Test
  public void testRetryableRegex() throws Exception {
    SQLException sqlException = new SQLException("ORA-08177: can't serialize access for this transaction", "72000");
    // Note that we have 3 regex'es below
    conf.setVar(HiveConf.ConfVars.HIVE_TXN_RETRYABLE_SQLEX_REGEX, "^Deadlock detected, roll back,.*08177.*,.*08178.*");
    boolean result = TxnHandler.isRetryable(conf, sqlException);
    Assert.assertTrue("regex should be retryable", result);

    sqlException = new SQLException("This error message, has comma in it");
    conf.setVar(HiveConf.ConfVars.HIVE_TXN_RETRYABLE_SQLEX_REGEX, ".*comma.*");
    result = TxnHandler.isRetryable(conf, sqlException);
    Assert.assertTrue("regex should be retryable", result);
  }

  private void updateTxns(Connection conn) throws SQLException {
    Statement stmt = conn.createStatement();
    stmt.executeUpdate("update TXNS set txn_last_heartbeat = txn_last_heartbeat + 1");
  }

  private void updateLocks(Connection conn) throws SQLException {
    Statement stmt = conn.createStatement();
    stmt.executeUpdate("update HIVE_LOCKS set hl_last_heartbeat = hl_last_heartbeat + 1");
  }

  @Before
  public void setUp() throws Exception {
    TxnDbUtil.prepDb();
    txnHandler = TxnUtils.getTxnStore(conf);
  }

  @After
  public void tearDown() throws Exception {
    TxnDbUtil.cleanDb();
  }

  private long openTxn() throws MetaException {
    List<Long> txns = txnHandler.openTxns(new OpenTxnRequest(1, "me", "localhost")).getTxn_ids();
    return txns.get(0);
  }

}
