/*
 * Copyright (C) 2005, 2006, 2007, 2008, 2011, 2012 Apple Inc. All rights
 * reserved.
 * Copyright (C) 2008 David Levin <levin@chromium.org>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library
 * General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public License
 * along with this library; see the file COPYING.LIB.  If not, write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 *
 */

#ifndef WTF_HashTable_h
#define WTF_HashTable_h

#include "wtf/Alignment.h"
#include "wtf/Allocator.h"
#include "wtf/Assertions.h"
#include "wtf/ConditionalDestructor.h"
#include "wtf/HashTraits.h"
#include "wtf/PtrUtil.h"
#include "wtf/allocator/PartitionAllocator.h"
#include <memory>

#define DUMP_HASHTABLE_STATS 0
#define DUMP_HASHTABLE_STATS_PER_TABLE 0

#if DUMP_HASHTABLE_STATS
#include "wtf/Atomics.h"
#include "wtf/Threading.h"
#endif

#if DUMP_HASHTABLE_STATS_PER_TABLE
#include "wtf/DataLog.h"
#endif

#if DUMP_HASHTABLE_STATS
#if DUMP_HASHTABLE_STATS_PER_TABLE
#define UPDATE_PROBE_COUNTS()                         \
  ++probeCount;                                       \
  HashTableStats::recordCollisionAtCount(probeCount); \
  ++perTableProbeCount;                               \
  m_stats->recordCollisionAtCount(perTableProbeCount)
#define UPDATE_ACCESS_COUNTS()                   \
  atomicIncrement(&HashTableStats::numAccesses); \
  int probeCount = 0;                            \
  ++m_stats->numAccesses;                        \
  int perTableProbeCount = 0
#else
#define UPDATE_PROBE_COUNTS() \
  ++probeCount;               \
  HashTableStats::recordCollisionAtCount(probeCount)
#define UPDATE_ACCESS_COUNTS()                   \
  atomicIncrement(&HashTableStats::numAccesses); \
  int probeCount = 0
#endif
#else
#if DUMP_HASHTABLE_STATS_PER_TABLE
#define UPDATE_PROBE_COUNTS() \
  ++perTableProbeCount;       \
  m_stats->recordCollisionAtCount(perTableProbeCount)
#define UPDATE_ACCESS_COUNTS() \
  ++m_stats->numAccesses;      \
  int perTableProbeCount = 0
#else
#define UPDATE_PROBE_COUNTS() \
  do {                        \
  } while (0)
#define UPDATE_ACCESS_COUNTS() \
  do {                         \
  } while (0)
#endif
#endif

namespace WTF {

#if DUMP_HASHTABLE_STATS

struct WTF_EXPORT HashTableStats {
  STATIC_ONLY(HashTableStats);
  // The following variables are all atomically incremented when modified.
  static int numAccesses;
  static int numRehashes;
  static int numRemoves;
  static int numReinserts;

  // The following variables are only modified in the recordCollisionAtCount
  // method within a mutex.
  static int maxCollisions;
  static int numCollisions;
  static int collisionGraph[4096];

  static void recordCollisionAtCount(int count);
  static void dumpStats();
};

#endif

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
class HashTable;
template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
class HashTableIterator;
template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
class HashTableConstIterator;
template <typename Value,
          typename HashFunctions,
          typename HashTraits,
          typename Allocator>
class LinkedHashSet;
template <WeakHandlingFlag x,
          typename T,
          typename U,
          typename V,
          typename W,
          typename X,
          typename Y,
          typename Z>
struct WeakProcessingHashTableHelper;

typedef enum { HashItemKnownGood } HashItemKnownGoodTag;

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
class HashTableConstIterator final {
  DISALLOW_NEW();

 private:
  typedef HashTable<Key,
                    Value,
                    Extractor,
                    HashFunctions,
                    Traits,
                    KeyTraits,
                    Allocator>
      HashTableType;
  typedef HashTableIterator<Key,
                            Value,
                            Extractor,
                            HashFunctions,
                            Traits,
                            KeyTraits,
                            Allocator>
      iterator;
  typedef HashTableConstIterator<Key,
                                 Value,
                                 Extractor,
                                 HashFunctions,
                                 Traits,
                                 KeyTraits,
                                 Allocator>
      const_iterator;
  typedef Value ValueType;
  typedef typename Traits::IteratorConstGetType GetType;
  typedef const ValueType* PointerType;

  friend class HashTable<Key,
                         Value,
                         Extractor,
                         HashFunctions,
                         Traits,
                         KeyTraits,
                         Allocator>;
  friend class HashTableIterator<Key,
                                 Value,
                                 Extractor,
                                 HashFunctions,
                                 Traits,
                                 KeyTraits,
                                 Allocator>;

  void skipEmptyBuckets() {
    while (m_position != m_endPosition &&
           HashTableType::isEmptyOrDeletedBucket(*m_position))
      ++m_position;
  }

  HashTableConstIterator(PointerType position,
                         PointerType endPosition,
                         const HashTableType* container)
      : m_position(position),
        m_endPosition(endPosition)
#if ENABLE(ASSERT)
        ,
        m_container(container),
        m_containerModifications(container->modifications())
#endif
  {
    skipEmptyBuckets();
  }

  HashTableConstIterator(PointerType position,
                         PointerType endPosition,
                         const HashTableType* container,
                         HashItemKnownGoodTag)
      : m_position(position),
        m_endPosition(endPosition)
#if ENABLE(ASSERT)
        ,
        m_container(container),
        m_containerModifications(container->modifications())
#endif
  {
    ASSERT(m_containerModifications == m_container->modifications());
  }

  void checkModifications() const {
    // HashTable and collections that build on it do not support
    // modifications while there is an iterator in use. The exception is
    // ListHashSet, which has its own iterators that tolerate modification
    // of the underlying set.
    ASSERT(m_containerModifications == m_container->modifications());
    ASSERT(!m_container->accessForbidden());
  }

 public:
  HashTableConstIterator() {}

  GetType get() const {
    checkModifications();
    return m_position;
  }
  typename Traits::IteratorConstReferenceType operator*() const {
    return Traits::getToReferenceConstConversion(get());
  }
  GetType operator->() const { return get(); }

  const_iterator& operator++() {
    ASSERT(m_position != m_endPosition);
    checkModifications();
    ++m_position;
    skipEmptyBuckets();
    return *this;
  }

  // postfix ++ intentionally omitted

  // Comparison.
  bool operator==(const const_iterator& other) const {
    return m_position == other.m_position;
  }
  bool operator!=(const const_iterator& other) const {
    return m_position != other.m_position;
  }
  bool operator==(const iterator& other) const {
    return *this == static_cast<const_iterator>(other);
  }
  bool operator!=(const iterator& other) const {
    return *this != static_cast<const_iterator>(other);
  }

  std::ostream& printTo(std::ostream& stream) const {
    if (m_position == m_endPosition)
      return stream << "iterator representing <end>";
    // TODO(tkent): Change |m_position| to |*m_position| to show the
    // pointed object. It requires a lot of new stream printer functions.
    return stream << "iterator pointing to " << m_position;
  }

 private:
  PointerType m_position;
  PointerType m_endPosition;
#if ENABLE(ASSERT)
  const HashTableType* m_container;
  int64_t m_containerModifications;
#endif
};

template <typename Key,
          typename Value,
          typename Extractor,
          typename Hash,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
std::ostream& operator<<(std::ostream& stream,
                         const HashTableConstIterator<Key,
                                                      Value,
                                                      Extractor,
                                                      Hash,
                                                      Traits,
                                                      KeyTraits,
                                                      Allocator>& iterator) {
  return iterator.printTo(stream);
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
class HashTableIterator final {
  DISALLOW_NEW();

 private:
  typedef HashTable<Key,
                    Value,
                    Extractor,
                    HashFunctions,
                    Traits,
                    KeyTraits,
                    Allocator>
      HashTableType;
  typedef HashTableIterator<Key,
                            Value,
                            Extractor,
                            HashFunctions,
                            Traits,
                            KeyTraits,
                            Allocator>
      iterator;
  typedef HashTableConstIterator<Key,
                                 Value,
                                 Extractor,
                                 HashFunctions,
                                 Traits,
                                 KeyTraits,
                                 Allocator>
      const_iterator;
  typedef Value ValueType;
  typedef typename Traits::IteratorGetType GetType;
  typedef ValueType* PointerType;

  friend class HashTable<Key,
                         Value,
                         Extractor,
                         HashFunctions,
                         Traits,
                         KeyTraits,
                         Allocator>;

  HashTableIterator(PointerType pos,
                    PointerType end,
                    const HashTableType* container)
      : m_iterator(pos, end, container) {}
  HashTableIterator(PointerType pos,
                    PointerType end,
                    const HashTableType* container,
                    HashItemKnownGoodTag tag)
      : m_iterator(pos, end, container, tag) {}

 public:
  HashTableIterator() {}

  // default copy, assignment and destructor are OK

  GetType get() const { return const_cast<GetType>(m_iterator.get()); }
  typename Traits::IteratorReferenceType operator*() const {
    return Traits::getToReferenceConversion(get());
  }
  GetType operator->() const { return get(); }

  iterator& operator++() {
    ++m_iterator;
    return *this;
  }

  // postfix ++ intentionally omitted

  // Comparison.
  bool operator==(const iterator& other) const {
    return m_iterator == other.m_iterator;
  }
  bool operator!=(const iterator& other) const {
    return m_iterator != other.m_iterator;
  }
  bool operator==(const const_iterator& other) const {
    return m_iterator == other;
  }
  bool operator!=(const const_iterator& other) const {
    return m_iterator != other;
  }

  operator const_iterator() const { return m_iterator; }
  std::ostream& printTo(std::ostream& stream) const {
    return m_iterator.printTo(stream);
  }

 private:
  const_iterator m_iterator;
};

template <typename Key,
          typename Value,
          typename Extractor,
          typename Hash,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
std::ostream& operator<<(std::ostream& stream,
                         const HashTableIterator<Key,
                                                 Value,
                                                 Extractor,
                                                 Hash,
                                                 Traits,
                                                 KeyTraits,
                                                 Allocator>& iterator) {
  return iterator.printTo(stream);
}

using std::swap;

template <typename T, typename Allocator, bool enterGCForbiddenScope>
struct Mover {
  STATIC_ONLY(Mover);
  static void move(T&& from, T& to) {
    to.~T();
    new (NotNull, &to) T(std::move(from));
  }
};

template <typename T, typename Allocator>
struct Mover<T, Allocator, true> {
  STATIC_ONLY(Mover);
  static void move(T&& from, T& to) {
    to.~T();
    Allocator::enterGCForbiddenScope();
    new (NotNull, &to) T(std::move(from));
    Allocator::leaveGCForbiddenScope();
  }
};

template <typename HashFunctions>
class IdentityHashTranslator {
  STATIC_ONLY(IdentityHashTranslator);

 public:
  template <typename T>
  static unsigned hash(const T& key) {
    return HashFunctions::hash(key);
  }
  template <typename T, typename U>
  static bool equal(const T& a, const U& b) {
    return HashFunctions::equal(a, b);
  }
  template <typename T, typename U, typename V>
  static void translate(T& location, U&&, V&& value) {
    location = std::forward<V>(value);
  }
};

template <typename HashTableType, typename ValueType>
struct HashTableAddResult final {
  STACK_ALLOCATED();
  HashTableAddResult(const HashTableType* container,
                     ValueType* storedValue,
                     bool isNewEntry)
      : storedValue(storedValue),
        isNewEntry(isNewEntry)
#if ENABLE(SECURITY_ASSERT)
        ,
        m_container(container),
        m_containerModifications(container->modifications())
#endif
  {
    ALLOW_UNUSED_LOCAL(container);
    DCHECK(container);
  }

  ValueType* storedValue;
  bool isNewEntry;

#if ENABLE(SECURITY_ASSERT)
  ~HashTableAddResult() {
    // If rehash happened before accessing storedValue, it's
    // use-after-free. Any modification may cause a rehash, so we check for
    // modifications here.

    // Rehash after accessing storedValue is harmless but will assert if the
    // AddResult destructor takes place after a modification. You may need
    // to limit the scope of the AddResult.
    SECURITY_DCHECK(m_containerModifications == m_container->modifications());
  }

 private:
  const HashTableType* m_container;
  const int64_t m_containerModifications;
#endif
};

template <typename Value, typename Extractor, typename KeyTraits>
struct HashTableHelper {
  STATIC_ONLY(HashTableHelper);
  static bool isEmptyBucket(const Value& value) {
    return isHashTraitsEmptyValue<KeyTraits>(Extractor::extract(value));
  }
  static bool isDeletedBucket(const Value& value) {
    return KeyTraits::isDeletedValue(Extractor::extract(value));
  }
  static bool isEmptyOrDeletedBucket(const Value& value) {
    return isEmptyBucket(value) || isDeletedBucket(value);
  }
};

template <typename HashTranslator,
          typename KeyTraits,
          bool safeToCompareToEmptyOrDeleted>
struct HashTableKeyChecker {
  STATIC_ONLY(HashTableKeyChecker);
  // There's no simple generic way to make this check if
  // safeToCompareToEmptyOrDeleted is false, so the check always passes.
  template <typename T>
  static bool checkKey(const T&) {
    return true;
  }
};

template <typename HashTranslator, typename KeyTraits>
struct HashTableKeyChecker<HashTranslator, KeyTraits, true> {
  STATIC_ONLY(HashTableKeyChecker);
  template <typename T>
  static bool checkKey(const T& key) {
    // FIXME : Check also equality to the deleted value.
    return !HashTranslator::equal(KeyTraits::emptyValue(), key);
  }
};

// Note: empty or deleted key values are not allowed, using them may lead to
// undefined behavior.  For pointer keys this means that null pointers are not
// allowed unless you supply custom key traits.
template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
class HashTable final
    : public ConditionalDestructor<HashTable<Key,
                                             Value,
                                             Extractor,
                                             HashFunctions,
                                             Traits,
                                             KeyTraits,
                                             Allocator>,
                                   Allocator::isGarbageCollected> {
  DISALLOW_NEW();

 public:
  typedef HashTableIterator<Key,
                            Value,
                            Extractor,
                            HashFunctions,
                            Traits,
                            KeyTraits,
                            Allocator>
      iterator;
  typedef HashTableConstIterator<Key,
                                 Value,
                                 Extractor,
                                 HashFunctions,
                                 Traits,
                                 KeyTraits,
                                 Allocator>
      const_iterator;
  typedef Traits ValueTraits;
  typedef Key KeyType;
  typedef typename KeyTraits::PeekInType KeyPeekInType;
  typedef Value ValueType;
  typedef Extractor ExtractorType;
  typedef KeyTraits KeyTraitsType;
  typedef IdentityHashTranslator<HashFunctions> IdentityTranslatorType;
  typedef HashTableAddResult<HashTable, ValueType> AddResult;

#if DUMP_HASHTABLE_STATS_PER_TABLE
  struct Stats {
    DISALLOW_NEW(Stats);
    Stats()
        : numAccesses(0),
          numRehashes(0),
          numRemoves(0),
          numReinserts(0),
          maxCollisions(0),
          numCollisions(0),
          collisionGraph() {}

    int numAccesses;
    int numRehashes;
    int numRemoves;
    int numReinserts;

    int maxCollisions;
    int numCollisions;
    int collisionGraph[4096];

    void recordCollisionAtCount(int count) {
      if (count > maxCollisions)
        maxCollisions = count;
      numCollisions++;
      collisionGraph[count]++;
    }

    void dumpStats() {
      dataLogF("\nWTF::HashTable::Stats dump\n\n");
      dataLogF("%d accesses\n", numAccesses);
      dataLogF("%d total collisions, average %.2f probes per access\n",
               numCollisions,
               1.0 * (numAccesses + numCollisions) / numAccesses);
      dataLogF("longest collision chain: %d\n", maxCollisions);
      for (int i = 1; i <= maxCollisions; i++) {
        dataLogF(
            "  %d lookups with exactly %d collisions (%.2f%% , %.2f%% with "
            "this many or more)\n",
            collisionGraph[i], i,
            100.0 * (collisionGraph[i] - collisionGraph[i + 1]) / numAccesses,
            100.0 * collisionGraph[i] / numAccesses);
      }
      dataLogF("%d rehashes\n", numRehashes);
      dataLogF("%d reinserts\n", numReinserts);
    }
  };
#endif

  HashTable();
  void finalize() {
    ASSERT(!Allocator::isGarbageCollected);
    if (LIKELY(!m_table))
      return;
    ASSERT(!m_accessForbidden);
#if ENABLE(ASSERT)
    m_accessForbidden = true;
#endif
    deleteAllBucketsAndDeallocate(m_table, m_tableSize);
#if ENABLE(ASSERT)
    m_accessForbidden = false;
#endif
    m_table = nullptr;
  }

  HashTable(const HashTable&);
  HashTable(HashTable&&);
  void swap(HashTable&);
  HashTable& operator=(const HashTable&);
  HashTable& operator=(HashTable&&);

  // When the hash table is empty, just return the same iterator for end as
  // for begin.  This is more efficient because we don't have to skip all the
  // empty and deleted buckets, and iterating an empty table is a common case
  // that's worth optimizing.
  iterator begin() { return isEmpty() ? end() : makeIterator(m_table); }
  iterator end() { return makeKnownGoodIterator(m_table + m_tableSize); }
  const_iterator begin() const {
    return isEmpty() ? end() : makeConstIterator(m_table);
  }
  const_iterator end() const {
    return makeKnownGoodConstIterator(m_table + m_tableSize);
  }

  unsigned size() const {
    ASSERT(!m_accessForbidden);
    return m_keyCount;
  }
  unsigned capacity() const {
    ASSERT(!m_accessForbidden);
    return m_tableSize;
  }
  bool isEmpty() const {
    ASSERT(!m_accessForbidden);
    return !m_keyCount;
  }

  void reserveCapacityForSize(unsigned size);

  template <typename IncomingValueType>
  AddResult add(IncomingValueType&& value) {
    return add<IdentityTranslatorType>(Extractor::extract(value),
                                       std::forward<IncomingValueType>(value));
  }

  // A special version of add() that finds the object by hashing and comparing
  // with some other type, to avoid the cost of type conversion if the object
  // is already in the table.
  template <typename HashTranslator, typename T, typename Extra>
  AddResult add(T&& key, Extra&&);
  template <typename HashTranslator, typename T, typename Extra>
  AddResult addPassingHashCode(T&& key, Extra&&);

  iterator find(KeyPeekInType key) { return find<IdentityTranslatorType>(key); }
  const_iterator find(KeyPeekInType key) const {
    return find<IdentityTranslatorType>(key);
  }
  bool contains(KeyPeekInType key) const {
    return contains<IdentityTranslatorType>(key);
  }

  template <typename HashTranslator, typename T>
  iterator find(const T&);
  template <typename HashTranslator, typename T>
  const_iterator find(const T&) const;
  template <typename HashTranslator, typename T>
  bool contains(const T&) const;

  void remove(KeyPeekInType);
  void remove(iterator);
  void remove(const_iterator);
  void clear();

  static bool isEmptyBucket(const ValueType& value) {
    return isHashTraitsEmptyValue<KeyTraits>(Extractor::extract(value));
  }
  static bool isDeletedBucket(const ValueType& value) {
    return KeyTraits::isDeletedValue(Extractor::extract(value));
  }
  static bool isEmptyOrDeletedBucket(const ValueType& value) {
    return HashTableHelper<ValueType, Extractor,
                           KeyTraits>::isEmptyOrDeletedBucket(value);
  }

  ValueType* lookup(KeyPeekInType key) {
    return lookup<IdentityTranslatorType, KeyPeekInType>(key);
  }
  template <typename HashTranslator, typename T>
  ValueType* lookup(const T&);
  template <typename HashTranslator, typename T>
  const ValueType* lookup(const T&) const;

  template <typename VisitorDispatcher>
  void trace(VisitorDispatcher);

#if ENABLE(ASSERT)
  bool accessForbidden() const { return m_accessForbidden; }
  int64_t modifications() const { return m_modifications; }
  void registerModification() { m_modifications++; }
  // HashTable and collections that build on it do not support modifications
  // while there is an iterator in use. The exception is ListHashSet, which
  // has its own iterators that tolerate modification of the underlying set.
  void checkModifications(int64_t mods) const {
    ASSERT(mods == m_modifications);
  }
#else
  int64_t modifications() const { return 0; }
  void registerModification() {}
  void checkModifications(int64_t mods) const {}
#endif

 private:
  static ValueType* allocateTable(unsigned size);
  static void deleteAllBucketsAndDeallocate(ValueType* table, unsigned size);

  typedef std::pair<ValueType*, bool> LookupType;
  typedef std::pair<LookupType, unsigned> FullLookupType;

  LookupType lookupForWriting(const Key& key) {
    return lookupForWriting<IdentityTranslatorType>(key);
  }
  template <typename HashTranslator, typename T>
  FullLookupType fullLookupForWriting(const T&);
  template <typename HashTranslator, typename T>
  LookupType lookupForWriting(const T&);

  void remove(ValueType*);

  bool shouldExpand() const {
    return (m_keyCount + m_deletedCount) * m_maxLoad >= m_tableSize;
  }
  bool mustRehashInPlace() const {
    return m_keyCount * m_minLoad < m_tableSize * 2;
  }
  bool shouldShrink() const {
    // isAllocationAllowed check should be at the last because it's
    // expensive.
    return m_keyCount * m_minLoad < m_tableSize &&
           m_tableSize > KeyTraits::minimumTableSize &&
           Allocator::isAllocationAllowed();
  }
  ValueType* expand(ValueType* entry = 0);
  void shrink() { rehash(m_tableSize / 2, 0); }

  ValueType* expandBuffer(unsigned newTableSize, ValueType* entry, bool&);
  ValueType* rehashTo(ValueType* newTable,
                      unsigned newTableSize,
                      ValueType* entry);
  ValueType* rehash(unsigned newTableSize, ValueType* entry);
  ValueType* reinsert(ValueType&&);

  static void initializeBucket(ValueType& bucket);
  static void deleteBucket(ValueType& bucket) {
    bucket.~ValueType();
    Traits::constructDeletedValue(bucket, Allocator::isGarbageCollected);
  }

  FullLookupType makeLookupResult(ValueType* position,
                                  bool found,
                                  unsigned hash) {
    return FullLookupType(LookupType(position, found), hash);
  }

  iterator makeIterator(ValueType* pos) {
    return iterator(pos, m_table + m_tableSize, this);
  }
  const_iterator makeConstIterator(ValueType* pos) const {
    return const_iterator(pos, m_table + m_tableSize, this);
  }
  iterator makeKnownGoodIterator(ValueType* pos) {
    return iterator(pos, m_table + m_tableSize, this, HashItemKnownGood);
  }
  const_iterator makeKnownGoodConstIterator(ValueType* pos) const {
    return const_iterator(pos, m_table + m_tableSize, this, HashItemKnownGood);
  }

  static const unsigned m_maxLoad = 2;
  static const unsigned m_minLoad = 6;

  unsigned tableSizeMask() const {
    size_t mask = m_tableSize - 1;
    ASSERT((mask & m_tableSize) == 0);
    return mask;
  }

  void setEnqueued() { m_queueFlag = true; }
  void clearEnqueued() { m_queueFlag = false; }
  bool enqueued() { return m_queueFlag; }

  ValueType* m_table;
  unsigned m_tableSize;
  unsigned m_keyCount;
#if ENABLE(ASSERT)
  unsigned m_deletedCount : 30;
  unsigned m_queueFlag : 1;
  unsigned m_accessForbidden : 1;
  unsigned m_modifications;
#else
  unsigned m_deletedCount : 31;
  unsigned m_queueFlag : 1;
#endif

#if DUMP_HASHTABLE_STATS_PER_TABLE
 public:
  mutable std::unique_ptr<Stats> m_stats;
#endif

  template <WeakHandlingFlag x,
            typename T,
            typename U,
            typename V,
            typename W,
            typename X,
            typename Y,
            typename Z>
  friend struct WeakProcessingHashTableHelper;
  template <typename T, typename U, typename V, typename W>
  friend class LinkedHashSet;
};

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
inline HashTable<Key,
                 Value,
                 Extractor,
                 HashFunctions,
                 Traits,
                 KeyTraits,
                 Allocator>::HashTable()
    : m_table(nullptr),
      m_tableSize(0),
      m_keyCount(0),
      m_deletedCount(0),
      m_queueFlag(false)
#if ENABLE(ASSERT)
      ,
      m_accessForbidden(false),
      m_modifications(0)
#endif
#if DUMP_HASHTABLE_STATS_PER_TABLE
      ,
      m_stats(wrapUnique(new Stats))
#endif
{
  static_assert(Allocator::isGarbageCollected ||
                    (!IsPointerToGarbageCollectedType<Key>::value &&
                     !IsPointerToGarbageCollectedType<Value>::value),
                "Cannot put raw pointers to garbage-collected classes into an "
                "off-heap collection.");
}

inline unsigned doubleHash(unsigned key) {
  key = ~key + (key >> 23);
  key ^= (key << 12);
  key ^= (key >> 7);
  key ^= (key << 2);
  key ^= (key >> 20);
  return key;
}

inline unsigned calculateCapacity(unsigned size) {
  for (unsigned mask = size; mask; mask >>= 1)
    size |= mask;         // 00110101010 -> 00111111111
  return (size + 1) * 2;  // 00111111111 -> 10000000000
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
void HashTable<Key,
               Value,
               Extractor,
               HashFunctions,
               Traits,
               KeyTraits,
               Allocator>::reserveCapacityForSize(unsigned newSize) {
  unsigned newCapacity = calculateCapacity(newSize);
  if (newCapacity < KeyTraits::minimumTableSize)
    newCapacity = KeyTraits::minimumTableSize;

  if (newCapacity > capacity()) {
    RELEASE_ASSERT(!static_cast<int>(
        newCapacity >>
        31));  // HashTable capacity should not overflow 32bit int.
    rehash(newCapacity, 0);
  }
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
template <typename HashTranslator, typename T>
inline Value*
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    lookup(const T& key) {
  return const_cast<Value*>(
      const_cast<const HashTable*>(this)->lookup<HashTranslator>(key));
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
template <typename HashTranslator, typename T>
inline const Value*
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    lookup(const T& key) const {
  ASSERT(!m_accessForbidden);
  ASSERT((HashTableKeyChecker<
          HashTranslator, KeyTraits,
          HashFunctions::safeToCompareToEmptyOrDeleted>::checkKey(key)));
  const ValueType* table = m_table;
  if (!table)
    return nullptr;

  size_t k = 0;
  size_t sizeMask = tableSizeMask();
  unsigned h = HashTranslator::hash(key);
  size_t i = h & sizeMask;

  UPDATE_ACCESS_COUNTS();

  while (1) {
    const ValueType* entry = table + i;

    if (HashFunctions::safeToCompareToEmptyOrDeleted) {
      if (HashTranslator::equal(Extractor::extract(*entry), key))
        return entry;

      if (isEmptyBucket(*entry))
        return nullptr;
    } else {
      if (isEmptyBucket(*entry))
        return nullptr;

      if (!isDeletedBucket(*entry) &&
          HashTranslator::equal(Extractor::extract(*entry), key))
        return entry;
    }
    UPDATE_PROBE_COUNTS();
    if (!k)
      k = 1 | doubleHash(h);
    i = (i + k) & sizeMask;
  }
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
template <typename HashTranslator, typename T>
inline typename HashTable<Key,
                          Value,
                          Extractor,
                          HashFunctions,
                          Traits,
                          KeyTraits,
                          Allocator>::LookupType
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    lookupForWriting(const T& key) {
  ASSERT(!m_accessForbidden);
  ASSERT(m_table);
  registerModification();

  ValueType* table = m_table;
  size_t k = 0;
  size_t sizeMask = tableSizeMask();
  unsigned h = HashTranslator::hash(key);
  size_t i = h & sizeMask;

  UPDATE_ACCESS_COUNTS();

  ValueType* deletedEntry = nullptr;

  while (1) {
    ValueType* entry = table + i;

    if (isEmptyBucket(*entry))
      return LookupType(deletedEntry ? deletedEntry : entry, false);

    if (HashFunctions::safeToCompareToEmptyOrDeleted) {
      if (HashTranslator::equal(Extractor::extract(*entry), key))
        return LookupType(entry, true);

      if (isDeletedBucket(*entry))
        deletedEntry = entry;
    } else {
      if (isDeletedBucket(*entry))
        deletedEntry = entry;
      else if (HashTranslator::equal(Extractor::extract(*entry), key))
        return LookupType(entry, true);
    }
    UPDATE_PROBE_COUNTS();
    if (!k)
      k = 1 | doubleHash(h);
    i = (i + k) & sizeMask;
  }
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
template <typename HashTranslator, typename T>
inline typename HashTable<Key,
                          Value,
                          Extractor,
                          HashFunctions,
                          Traits,
                          KeyTraits,
                          Allocator>::FullLookupType
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    fullLookupForWriting(const T& key) {
  ASSERT(!m_accessForbidden);
  ASSERT(m_table);
  registerModification();

  ValueType* table = m_table;
  size_t k = 0;
  size_t sizeMask = tableSizeMask();
  unsigned h = HashTranslator::hash(key);
  size_t i = h & sizeMask;

  UPDATE_ACCESS_COUNTS();

  ValueType* deletedEntry = nullptr;

  while (1) {
    ValueType* entry = table + i;

    if (isEmptyBucket(*entry))
      return makeLookupResult(deletedEntry ? deletedEntry : entry, false, h);

    if (HashFunctions::safeToCompareToEmptyOrDeleted) {
      if (HashTranslator::equal(Extractor::extract(*entry), key))
        return makeLookupResult(entry, true, h);

      if (isDeletedBucket(*entry))
        deletedEntry = entry;
    } else {
      if (isDeletedBucket(*entry))
        deletedEntry = entry;
      else if (HashTranslator::equal(Extractor::extract(*entry), key))
        return makeLookupResult(entry, true, h);
    }
    UPDATE_PROBE_COUNTS();
    if (!k)
      k = 1 | doubleHash(h);
    i = (i + k) & sizeMask;
  }
}

template <bool emptyValueIsZero>
struct HashTableBucketInitializer;

template <>
struct HashTableBucketInitializer<false> {
  STATIC_ONLY(HashTableBucketInitializer);
  template <typename Traits, typename Value>
  static void initialize(Value& bucket) {
    new (NotNull, &bucket) Value(Traits::emptyValue());
  }
};

template <>
struct HashTableBucketInitializer<true> {
  STATIC_ONLY(HashTableBucketInitializer);
  template <typename Traits, typename Value>
  static void initialize(Value& bucket) {
    // This initializes the bucket without copying the empty value.  That
    // makes it possible to use this with types that don't support copying.
    // The memset to 0 looks like a slow operation but is optimized by the
    // compilers.
    memset(&bucket, 0, sizeof(bucket));
  }
};

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
inline void
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    initializeBucket(ValueType& bucket) {
  HashTableBucketInitializer<Traits::emptyValueIsZero>::template initialize<
      Traits>(bucket);
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
template <typename HashTranslator, typename T, typename Extra>
typename HashTable<Key,
                   Value,
                   Extractor,
                   HashFunctions,
                   Traits,
                   KeyTraits,
                   Allocator>::AddResult
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    add(T&& key, Extra&& extra) {
  ASSERT(!m_accessForbidden);
  ASSERT(Allocator::isAllocationAllowed());
  if (!m_table)
    expand();

  ASSERT(m_table);

  ValueType* table = m_table;
  size_t k = 0;
  size_t sizeMask = tableSizeMask();
  unsigned h = HashTranslator::hash(key);
  size_t i = h & sizeMask;

  UPDATE_ACCESS_COUNTS();

  ValueType* deletedEntry = nullptr;
  ValueType* entry;
  while (1) {
    entry = table + i;

    if (isEmptyBucket(*entry))
      break;

    if (HashFunctions::safeToCompareToEmptyOrDeleted) {
      if (HashTranslator::equal(Extractor::extract(*entry), key))
        return AddResult(this, entry, false);

      if (isDeletedBucket(*entry))
        deletedEntry = entry;
    } else {
      if (isDeletedBucket(*entry))
        deletedEntry = entry;
      else if (HashTranslator::equal(Extractor::extract(*entry), key))
        return AddResult(this, entry, false);
    }
    UPDATE_PROBE_COUNTS();
    if (!k)
      k = 1 | doubleHash(h);
    i = (i + k) & sizeMask;
  }

  registerModification();

  if (deletedEntry) {
    // Overwrite any data left over from last use, using placement new or
    // memset.
    initializeBucket(*deletedEntry);
    entry = deletedEntry;
    --m_deletedCount;
  }

  HashTranslator::translate(*entry, std::forward<T>(key),
                            std::forward<Extra>(extra));
  ASSERT(!isEmptyOrDeletedBucket(*entry));

  ++m_keyCount;

  if (shouldExpand()) {
    entry = expand(entry);
  } else if (Traits::weakHandlingFlag == WeakHandlingInCollections &&
             shouldShrink()) {
    // When weak hash tables are processed by the garbage collector,
    // elements with no other strong references to them will have their
    // table entries cleared. But no shrinking of the backing store is
    // allowed at that time, as allocations are prohibited during that
    // GC phase.
    //
    // With that weak processing taking care of removals, explicit
    // remove()s of elements is rarely done. Which implies that the
    // weak hash table will never be checked if it can be shrunk.
    //
    // To prevent weak hash tables with very low load factors from
    // developing, we perform it when adding elements instead.
    entry = rehash(m_tableSize / 2, entry);
  }

  return AddResult(this, entry, true);
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
template <typename HashTranslator, typename T, typename Extra>
typename HashTable<Key,
                   Value,
                   Extractor,
                   HashFunctions,
                   Traits,
                   KeyTraits,
                   Allocator>::AddResult
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    addPassingHashCode(T&& key, Extra&& extra) {
  ASSERT(!m_accessForbidden);
  ASSERT(Allocator::isAllocationAllowed());
  if (!m_table)
    expand();

  FullLookupType lookupResult = fullLookupForWriting<HashTranslator>(key);

  ValueType* entry = lookupResult.first.first;
  bool found = lookupResult.first.second;
  unsigned h = lookupResult.second;

  if (found)
    return AddResult(this, entry, false);

  registerModification();

  if (isDeletedBucket(*entry)) {
    initializeBucket(*entry);
    --m_deletedCount;
  }

  HashTranslator::translate(*entry, std::forward<T>(key),
                            std::forward<Extra>(extra), h);
  ASSERT(!isEmptyOrDeletedBucket(*entry));

  ++m_keyCount;
  if (shouldExpand())
    entry = expand(entry);

  return AddResult(this, entry, true);
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
Value*
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    reinsert(ValueType&& entry) {
  ASSERT(m_table);
  registerModification();
  ASSERT(!lookupForWriting(Extractor::extract(entry)).second);
  ASSERT(
      !isDeletedBucket(*(lookupForWriting(Extractor::extract(entry)).first)));
#if DUMP_HASHTABLE_STATS
  atomicIncrement(&HashTableStats::numReinserts);
#endif
#if DUMP_HASHTABLE_STATS_PER_TABLE
  ++m_stats->numReinserts;
#endif
  Value* newEntry = lookupForWriting(Extractor::extract(entry)).first;
  Mover<ValueType, Allocator,
        Traits::template NeedsToForbidGCOnMove<>::value>::move(std::move(entry),
                                                               *newEntry);

  return newEntry;
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
template <typename HashTranslator, typename T>
inline typename HashTable<Key,
                          Value,
                          Extractor,
                          HashFunctions,
                          Traits,
                          KeyTraits,
                          Allocator>::iterator
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    find(const T& key) {
  ValueType* entry = lookup<HashTranslator>(key);
  if (!entry)
    return end();

  return makeKnownGoodIterator(entry);
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
template <typename HashTranslator, typename T>
inline typename HashTable<Key,
                          Value,
                          Extractor,
                          HashFunctions,
                          Traits,
                          KeyTraits,
                          Allocator>::const_iterator
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    find(const T& key) const {
  ValueType* entry = const_cast<HashTable*>(this)->lookup<HashTranslator>(key);
  if (!entry)
    return end();

  return makeKnownGoodConstIterator(entry);
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
template <typename HashTranslator, typename T>
bool HashTable<Key,
               Value,
               Extractor,
               HashFunctions,
               Traits,
               KeyTraits,
               Allocator>::contains(const T& key) const {
  return const_cast<HashTable*>(this)->lookup<HashTranslator>(key);
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
void HashTable<Key,
               Value,
               Extractor,
               HashFunctions,
               Traits,
               KeyTraits,
               Allocator>::remove(ValueType* pos) {
  registerModification();
#if DUMP_HASHTABLE_STATS
  atomicIncrement(&HashTableStats::numRemoves);
#endif
#if DUMP_HASHTABLE_STATS_PER_TABLE
  ++m_stats->numRemoves;
#endif

  ASSERT(!m_accessForbidden);
#if ENABLE(ASSERT)
  m_accessForbidden = true;
#endif
  deleteBucket(*pos);
#if ENABLE(ASSERT)
  m_accessForbidden = false;
#endif
  ++m_deletedCount;
  --m_keyCount;

  if (shouldShrink())
    shrink();
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
inline void
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    remove(iterator it) {
  if (it == end())
    return;
  remove(const_cast<ValueType*>(it.m_iterator.m_position));
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
inline void
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    remove(const_iterator it) {
  if (it == end())
    return;
  remove(const_cast<ValueType*>(it.m_position));
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
inline void
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    remove(KeyPeekInType key) {
  remove(find(key));
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
Value*
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    allocateTable(unsigned size) {
  size_t allocSize = size * sizeof(ValueType);
  ValueType* result;
  // Assert that we will not use memset on things with a vtable entry.  The
  // compiler will also check this on some platforms. We would like to check
  // this on the whole value (key-value pair), but std::is_polymorphic will
  // return false for a pair of two types, even if one of the components is
  // polymorphic.
  static_assert(
      !Traits::emptyValueIsZero || !std::is_polymorphic<KeyType>::value,
      "empty value cannot be zero for things with a vtable");
  static_assert(Allocator::isGarbageCollected ||
                    ((!AllowsOnlyPlacementNew<KeyType>::value ||
                      !IsTraceable<KeyType>::value) &&
                     (!AllowsOnlyPlacementNew<ValueType>::value ||
                      !IsTraceable<ValueType>::value)),
                "Cannot put DISALLOW_NEW_EXCEPT_PLACEMENT_NEW objects that "
                "have trace methods into an off-heap HashTable");

  if (Traits::emptyValueIsZero) {
    result = Allocator::template allocateZeroedHashTableBacking<ValueType,
                                                                HashTable>(
        allocSize);
  } else {
    result = Allocator::template allocateHashTableBacking<ValueType, HashTable>(
        allocSize);
    for (unsigned i = 0; i < size; i++)
      initializeBucket(result[i]);
  }
  return result;
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
void HashTable<Key,
               Value,
               Extractor,
               HashFunctions,
               Traits,
               KeyTraits,
               Allocator>::deleteAllBucketsAndDeallocate(ValueType* table,
                                                         unsigned size) {
  if (!IsTriviallyDestructible<ValueType>::value) {
    for (unsigned i = 0; i < size; ++i) {
      // This code is called when the hash table is cleared or resized. We
      // have allocated a new backing store and we need to run the
      // destructors on the old backing store, as it is being freed. If we
      // are GCing we need to both call the destructor and mark the bucket
      // as deleted, otherwise the destructor gets called again when the
      // GC finds the backing store. With the default allocator it's
      // enough to call the destructor, since we will free the memory
      // explicitly and we won't see the memory with the bucket again.
      if (Allocator::isGarbageCollected) {
        if (!isEmptyOrDeletedBucket(table[i]))
          deleteBucket(table[i]);
      } else {
        if (!isDeletedBucket(table[i]))
          table[i].~ValueType();
      }
    }
  }
  Allocator::freeHashTableBacking(table);
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
Value*
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    expand(Value* entry) {
  unsigned newSize;
  if (!m_tableSize) {
    newSize = KeyTraits::minimumTableSize;
  } else if (mustRehashInPlace()) {
    newSize = m_tableSize;
  } else {
    newSize = m_tableSize * 2;
    RELEASE_ASSERT(newSize > m_tableSize);
  }

  return rehash(newSize, entry);
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
Value*
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    expandBuffer(unsigned newTableSize, Value* entry, bool& success) {
  success = false;
  ASSERT(m_tableSize < newTableSize);
  if (!Allocator::expandHashTableBacking(m_table,
                                         newTableSize * sizeof(ValueType)))
    return nullptr;

  success = true;

  Value* newEntry = nullptr;
  unsigned oldTableSize = m_tableSize;
  ValueType* originalTable = m_table;

  ValueType* temporaryTable = allocateTable(oldTableSize);
  for (unsigned i = 0; i < oldTableSize; i++) {
    if (&m_table[i] == entry)
      newEntry = &temporaryTable[i];
    if (isEmptyOrDeletedBucket(m_table[i])) {
      ASSERT(&m_table[i] != entry);
      if (Traits::emptyValueIsZero) {
        memset(&temporaryTable[i], 0, sizeof(ValueType));
      } else {
        initializeBucket(temporaryTable[i]);
      }
    } else {
      Mover<ValueType, Allocator,
            Traits::template NeedsToForbidGCOnMove<>::value>::
          move(std::move(m_table[i]), temporaryTable[i]);
    }
  }
  m_table = temporaryTable;

  if (Traits::emptyValueIsZero) {
    memset(originalTable, 0, newTableSize * sizeof(ValueType));
  } else {
    for (unsigned i = 0; i < newTableSize; i++)
      initializeBucket(originalTable[i]);
  }
  newEntry = rehashTo(originalTable, newTableSize, newEntry);

  ASSERT(!m_accessForbidden);
#if ENABLE(ASSERT)
  m_accessForbidden = true;
#endif
  deleteAllBucketsAndDeallocate(temporaryTable, oldTableSize);
#if ENABLE(ASSERT)
  m_accessForbidden = false;
#endif

  return newEntry;
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
Value*
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    rehashTo(ValueType* newTable, unsigned newTableSize, Value* entry) {
  unsigned oldTableSize = m_tableSize;
  ValueType* oldTable = m_table;

#if DUMP_HASHTABLE_STATS
  if (oldTableSize != 0)
    atomicIncrement(&HashTableStats::numRehashes);
#endif

#if DUMP_HASHTABLE_STATS_PER_TABLE
  if (oldTableSize != 0)
    ++m_stats->numRehashes;
#endif

  m_table = newTable;
  m_tableSize = newTableSize;

  Value* newEntry = nullptr;
  for (unsigned i = 0; i != oldTableSize; ++i) {
    if (isEmptyOrDeletedBucket(oldTable[i])) {
      ASSERT(&oldTable[i] != entry);
      continue;
    }
    Value* reinsertedEntry = reinsert(std::move(oldTable[i]));
    if (&oldTable[i] == entry) {
      ASSERT(!newEntry);
      newEntry = reinsertedEntry;
    }
  }

  m_deletedCount = 0;

  return newEntry;
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
Value*
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    rehash(unsigned newTableSize, Value* entry) {
  unsigned oldTableSize = m_tableSize;
  ValueType* oldTable = m_table;

#if DUMP_HASHTABLE_STATS
  if (oldTableSize != 0)
    atomicIncrement(&HashTableStats::numRehashes);
#endif

#if DUMP_HASHTABLE_STATS_PER_TABLE
  if (oldTableSize != 0)
    ++m_stats->numRehashes;
#endif

  // The Allocator::isGarbageCollected check is not needed.  The check is just
  // a static hint for a compiler to indicate that Base::expandBuffer returns
  // false if Allocator is a PartitionAllocator.
  if (Allocator::isGarbageCollected && newTableSize > oldTableSize) {
    bool success;
    Value* newEntry = expandBuffer(newTableSize, entry, success);
    if (success)
      return newEntry;
  }

  ValueType* newTable = allocateTable(newTableSize);
  Value* newEntry = rehashTo(newTable, newTableSize, entry);

  ASSERT(!m_accessForbidden);
#if ENABLE(ASSERT)
  m_accessForbidden = true;
#endif
  deleteAllBucketsAndDeallocate(oldTable, oldTableSize);
#if ENABLE(ASSERT)
  m_accessForbidden = false;
#endif

  return newEntry;
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
void HashTable<Key,
               Value,
               Extractor,
               HashFunctions,
               Traits,
               KeyTraits,
               Allocator>::clear() {
  registerModification();
  if (!m_table)
    return;

  ASSERT(!m_accessForbidden);
#if ENABLE(ASSERT)
  m_accessForbidden = true;
#endif
  deleteAllBucketsAndDeallocate(m_table, m_tableSize);
#if ENABLE(ASSERT)
  m_accessForbidden = false;
#endif
  m_table = nullptr;
  m_tableSize = 0;
  m_keyCount = 0;
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    HashTable(const HashTable& other)
    : m_table(nullptr),
      m_tableSize(0),
      m_keyCount(0),
      m_deletedCount(0),
      m_queueFlag(false)
#if ENABLE(ASSERT)
      ,
      m_accessForbidden(false),
      m_modifications(0)
#endif
#if DUMP_HASHTABLE_STATS_PER_TABLE
      ,
      m_stats(wrapUnique(new Stats(*other.m_stats)))
#endif
{
  // Copy the hash table the dumb way, by adding each element to the new
  // table.  It might be more efficient to copy the table slots, but it's not
  // clear that efficiency is needed.
  const_iterator end = other.end();
  for (const_iterator it = other.begin(); it != end; ++it)
    add(*it);
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
    HashTable(HashTable&& other)
    : m_table(nullptr),
      m_tableSize(0),
      m_keyCount(0),
      m_deletedCount(0),
      m_queueFlag(false)
#if ENABLE(ASSERT)
      ,
      m_accessForbidden(false),
      m_modifications(0)
#endif
#if DUMP_HASHTABLE_STATS_PER_TABLE
      ,
      m_stats(wrapUnique(new Stats(*other.m_stats)))
#endif
{
  swap(other);
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
void HashTable<Key,
               Value,
               Extractor,
               HashFunctions,
               Traits,
               KeyTraits,
               Allocator>::swap(HashTable& other) {
  ASSERT(!m_accessForbidden);
  std::swap(m_table, other.m_table);
  std::swap(m_tableSize, other.m_tableSize);
  std::swap(m_keyCount, other.m_keyCount);
  // std::swap does not work for bit fields.
  unsigned deleted = m_deletedCount;
  m_deletedCount = other.m_deletedCount;
  other.m_deletedCount = deleted;
  ASSERT(!m_queueFlag);
  ASSERT(!other.m_queueFlag);

#if ENABLE(ASSERT)
  std::swap(m_modifications, other.m_modifications);
#endif

#if DUMP_HASHTABLE_STATS_PER_TABLE
  m_stats.swap(other.m_stats);
#endif
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>&
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
operator=(const HashTable& other) {
  HashTable tmp(other);
  swap(tmp);
  return *this;
}

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>&
HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits, Allocator>::
operator=(HashTable&& other) {
  swap(other);
  return *this;
}

template <WeakHandlingFlag weakHandlingFlag,
          typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
struct WeakProcessingHashTableHelper;

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
struct WeakProcessingHashTableHelper<NoWeakHandlingInCollections,
                                     Key,
                                     Value,
                                     Extractor,
                                     HashFunctions,
                                     Traits,
                                     KeyTraits,
                                     Allocator> {
  STATIC_ONLY(WeakProcessingHashTableHelper);
  static void process(typename Allocator::Visitor* visitor, void* closure) {}
  static void ephemeronIteration(typename Allocator::Visitor* visitor,
                                 void* closure) {}
  static void ephemeronIterationDone(typename Allocator::Visitor* visitor,
                                     void* closure) {}
};

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
struct WeakProcessingHashTableHelper<WeakHandlingInCollections,
                                     Key,
                                     Value,
                                     Extractor,
                                     HashFunctions,
                                     Traits,
                                     KeyTraits,
                                     Allocator> {
  STATIC_ONLY(WeakProcessingHashTableHelper);

  using HashTableType = HashTable<Key,
                                  Value,
                                  Extractor,
                                  HashFunctions,
                                  Traits,
                                  KeyTraits,
                                  Allocator>;
  using ValueType = typename HashTableType::ValueType;

  // Used for purely weak and for weak-and-strong tables (ephemerons).
  static void process(typename Allocator::Visitor* visitor, void* closure) {
    HashTableType* table = reinterpret_cast<HashTableType*>(closure);
    if (!table->m_table)
      return;
    // Now perform weak processing (this is a no-op if the backing was
    // accessible through an iterator and was already marked strongly).
    for (ValueType* element = table->m_table + table->m_tableSize - 1;
         element >= table->m_table; element--) {
      if (!HashTableType::isEmptyOrDeletedBucket(*element)) {
        // At this stage calling trace can make no difference
        // (everything is already traced), but we use the return value
        // to remove things from the collection.

        // FIXME: This should be rewritten so that this can check if the
        // element is dead without calling trace, which is semantically
        // not correct to be called in weak processing stage.
        if (TraceInCollectionTrait<WeakHandlingInCollections,
                                   WeakPointersActWeak, ValueType,
                                   Traits>::trace(visitor, *element)) {
          table->registerModification();
          HashTableType::deleteBucket(*element);  // Also calls the destructor.
          table->m_deletedCount++;
          table->m_keyCount--;
          // We don't rehash the backing until the next add or delete,
          // because that would cause allocation during GC.
        }
      }
    }
  }

  // Called repeatedly for tables that have both weak and strong pointers.
  static void ephemeronIteration(typename Allocator::Visitor* visitor,
                                 void* closure) {
    HashTableType* table = reinterpret_cast<HashTableType*>(closure);
    ASSERT(table->m_table);
    // Check the hash table for elements that we now know will not be
    // removed by weak processing. Those elements need to have their strong
    // pointers traced.
    for (ValueType* element = table->m_table + table->m_tableSize - 1;
         element >= table->m_table; element--) {
      if (!HashTableType::isEmptyOrDeletedBucket(*element))
        TraceInCollectionTrait<WeakHandlingInCollections, WeakPointersActWeak,
                               ValueType, Traits>::trace(visitor, *element);
    }
  }

  // Called when the ephemeron iteration is done and before running the per
  // thread weak processing. It is guaranteed to be called before any thread
  // is resumed.
  static void ephemeronIterationDone(typename Allocator::Visitor* visitor,
                                     void* closure) {
    HashTableType* table = reinterpret_cast<HashTableType*>(closure);
    ASSERT(Allocator::weakTableRegistered(visitor, table));
    table->clearEnqueued();
  }
};

template <typename Key,
          typename Value,
          typename Extractor,
          typename HashFunctions,
          typename Traits,
          typename KeyTraits,
          typename Allocator>
template <typename VisitorDispatcher>
void HashTable<Key,
               Value,
               Extractor,
               HashFunctions,
               Traits,
               KeyTraits,
               Allocator>::trace(VisitorDispatcher visitor) {
  // If someone else already marked the backing and queued up the trace and/or
  // weak callback then we are done. This optimization does not happen for
  // ListHashSet since its iterator does not point at the backing.
  if (!m_table || Allocator::isHeapObjectAlive(m_table))
    return;
  // Normally, we mark the backing store without performing trace. This means
  // it is marked live, but the pointers inside it are not marked.  Instead we
  // will mark the pointers below. However, for backing stores that contain
  // weak pointers the handling is rather different.  We don't mark the
  // backing store here, so the marking GC will leave the backing unmarked. If
  // the backing is found in any other way than through its HashTable (ie from
  // an iterator) then the mark bit will be set and the pointers will be
  // marked strongly, avoiding problems with iterating over things that
  // disappear due to weak processing while we are iterating over them. We
  // register the backing store pointer for delayed marking which will take
  // place after we know if the backing is reachable from elsewhere. We also
  // register a weakProcessing callback which will perform weak processing if
  // needed.
  if (Traits::weakHandlingFlag == NoWeakHandlingInCollections) {
    Allocator::markNoTracing(visitor, m_table);
  } else {
    Allocator::registerDelayedMarkNoTracing(visitor, m_table);
    // Since we're delaying marking this HashTable, it is possible that the
    // registerWeakMembers is called multiple times (in rare
    // cases). However, it shouldn't cause any issue.
    Allocator::registerWeakMembers(
        visitor, this, m_table,
        WeakProcessingHashTableHelper<Traits::weakHandlingFlag, Key, Value,
                                      Extractor, HashFunctions, Traits,
                                      KeyTraits, Allocator>::process);
  }
  if (!IsTraceableInCollectionTrait<Traits>::value)
    return;
  if (Traits::weakHandlingFlag == WeakHandlingInCollections) {
    // If we have both strong and weak pointers in the collection then
    // we queue up the collection for fixed point iteration a la
    // Ephemerons:
    // http://dl.acm.org/citation.cfm?doid=263698.263733 - see also
    // http://www.jucs.org/jucs_14_21/eliminating_cycles_in_weak
    ASSERT(!enqueued() || Allocator::weakTableRegistered(visitor, this));
    if (!enqueued()) {
      Allocator::registerWeakTable(
          visitor, this,
          WeakProcessingHashTableHelper<
              Traits::weakHandlingFlag, Key, Value, Extractor, HashFunctions,
              Traits, KeyTraits, Allocator>::ephemeronIteration,
          WeakProcessingHashTableHelper<
              Traits::weakHandlingFlag, Key, Value, Extractor, HashFunctions,
              Traits, KeyTraits, Allocator>::ephemeronIterationDone);
      setEnqueued();
    }
    // We don't need to trace the elements here, since registering as a
    // weak table above will cause them to be traced (perhaps several
    // times). It's better to wait until everything else is traced
    // before tracing the elements for the first time; this may reduce
    // (by one) the number of iterations needed to get to a fixed point.
    return;
  }
  for (ValueType* element = m_table + m_tableSize - 1; element >= m_table;
       element--) {
    if (!isEmptyOrDeletedBucket(*element))
      Allocator::template trace<VisitorDispatcher, ValueType, Traits>(visitor,
                                                                      *element);
  }
}

// iterator adapters

template <typename HashTableType, typename Traits>
struct HashTableConstIteratorAdapter {
  STACK_ALLOCATED();
  HashTableConstIteratorAdapter() {}
  HashTableConstIteratorAdapter(
      const typename HashTableType::const_iterator& impl)
      : m_impl(impl) {}
  typedef typename Traits::IteratorConstGetType GetType;
  typedef
      typename HashTableType::ValueTraits::IteratorConstGetType SourceGetType;

  GetType get() const {
    return const_cast<GetType>(SourceGetType(m_impl.get()));
  }
  typename Traits::IteratorConstReferenceType operator*() const {
    return Traits::getToReferenceConstConversion(get());
  }
  GetType operator->() const { return get(); }

  HashTableConstIteratorAdapter& operator++() {
    ++m_impl;
    return *this;
  }
  // postfix ++ intentionally omitted

  typename HashTableType::const_iterator m_impl;
};

template <typename HashTable, typename Traits>
std::ostream& operator<<(
    std::ostream& stream,
    const HashTableConstIteratorAdapter<HashTable, Traits>& iterator) {
  return stream << iterator.m_impl;
}

template <typename HashTableType, typename Traits>
struct HashTableIteratorAdapter {
  STACK_ALLOCATED();
  typedef typename Traits::IteratorGetType GetType;
  typedef typename HashTableType::ValueTraits::IteratorGetType SourceGetType;

  HashTableIteratorAdapter() {}
  HashTableIteratorAdapter(const typename HashTableType::iterator& impl)
      : m_impl(impl) {}

  GetType get() const {
    return const_cast<GetType>(SourceGetType(m_impl.get()));
  }
  typename Traits::IteratorReferenceType operator*() const {
    return Traits::getToReferenceConversion(get());
  }
  GetType operator->() const { return get(); }

  HashTableIteratorAdapter& operator++() {
    ++m_impl;
    return *this;
  }
  // postfix ++ intentionally omitted

  operator HashTableConstIteratorAdapter<HashTableType, Traits>() {
    typename HashTableType::const_iterator i = m_impl;
    return i;
  }

  typename HashTableType::iterator m_impl;
};

template <typename HashTable, typename Traits>
std::ostream& operator<<(
    std::ostream& stream,
    const HashTableIteratorAdapter<HashTable, Traits>& iterator) {
  return stream << iterator.m_impl;
}

template <typename T, typename U>
inline bool operator==(const HashTableConstIteratorAdapter<T, U>& a,
                       const HashTableConstIteratorAdapter<T, U>& b) {
  return a.m_impl == b.m_impl;
}

template <typename T, typename U>
inline bool operator!=(const HashTableConstIteratorAdapter<T, U>& a,
                       const HashTableConstIteratorAdapter<T, U>& b) {
  return a.m_impl != b.m_impl;
}

template <typename T, typename U>
inline bool operator==(const HashTableIteratorAdapter<T, U>& a,
                       const HashTableIteratorAdapter<T, U>& b) {
  return a.m_impl == b.m_impl;
}

template <typename T, typename U>
inline bool operator!=(const HashTableIteratorAdapter<T, U>& a,
                       const HashTableIteratorAdapter<T, U>& b) {
  return a.m_impl != b.m_impl;
}

// All 4 combinations of ==, != and Const,non const.
template <typename T, typename U>
inline bool operator==(const HashTableConstIteratorAdapter<T, U>& a,
                       const HashTableIteratorAdapter<T, U>& b) {
  return a.m_impl == b.m_impl;
}

template <typename T, typename U>
inline bool operator!=(const HashTableConstIteratorAdapter<T, U>& a,
                       const HashTableIteratorAdapter<T, U>& b) {
  return a.m_impl != b.m_impl;
}

template <typename T, typename U>
inline bool operator==(const HashTableIteratorAdapter<T, U>& a,
                       const HashTableConstIteratorAdapter<T, U>& b) {
  return a.m_impl == b.m_impl;
}

template <typename T, typename U>
inline bool operator!=(const HashTableIteratorAdapter<T, U>& a,
                       const HashTableConstIteratorAdapter<T, U>& b) {
  return a.m_impl != b.m_impl;
}

template <typename Collection1, typename Collection2>
inline void removeAll(Collection1& collection, const Collection2& toBeRemoved) {
  if (collection.isEmpty() || toBeRemoved.isEmpty())
    return;
  typedef typename Collection2::const_iterator CollectionIterator;
  CollectionIterator end(toBeRemoved.end());
  for (CollectionIterator it(toBeRemoved.begin()); it != end; ++it)
    collection.remove(*it);
}

}  // namespace WTF

#include "wtf/HashIterators.h"

#endif  // WTF_HashTable_h