From fc8eda8fd90d15dd5f6b7bb70935b86168dc18e0 Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Tue, 20 Feb 2018 16:20:40 -0500 Subject: [PATCH 01/81] serialized and inserted log records for inserts --- ...timestamp_ordering_transaction_manager.cpp | 70 +++++++-- src/concurrency/transaction_context.cpp | 3 + src/concurrency/transaction_manager.cpp | 1 - .../timestamp_ordering_transaction_manager.h | 3 + src/include/concurrency/transaction_context.h | 8 + src/include/logging/checkpoint_manager.h | 69 -------- .../logging/checkpoint_manager_factory.h | 54 ------- src/include/logging/log_buffer.h | 70 +++------ src/include/logging/log_buffer_pool.h | 63 -------- src/include/logging/log_manager.h | 79 ---------- src/include/logging/log_manager_factory.h | 54 ------- src/include/logging/log_record.h | 52 +++--- .../logging/logical_checkpoint_manager.h | 60 ------- src/include/logging/logical_log_manager.h | 129 --------------- src/include/logging/logical_logger.h | 148 ------------------ src/include/logging/wal_logger.h | 49 ++++++ src/include/storage/data_table.h | 5 +- src/include/threadpool/logger_queue_pool.h | 82 ++++++++++ src/logging/checkpoint_manager_factory.cpp | 23 --- src/logging/log_buffer.cpp | 102 ++++++++---- src/logging/log_buffer_pool.cpp | 63 -------- src/logging/log_manager_factory.cpp | 24 --- src/logging/wal_log_manager.cpp | 11 ++ src/logging/wal_logger.cpp | 22 +++ src/storage/data_table.cpp | 1 - src/storage/tile_group_header.cpp | 1 - src/threadpool/logger_queue_pool.cpp | 34 ++++ 27 files changed, 407 insertions(+), 873 deletions(-) delete mode 100644 src/include/logging/checkpoint_manager.h delete mode 100644 src/include/logging/checkpoint_manager_factory.h delete mode 100644 src/include/logging/log_buffer_pool.h delete mode 100644 src/include/logging/log_manager.h delete mode 100644 src/include/logging/log_manager_factory.h delete mode 100644 src/include/logging/logical_checkpoint_manager.h delete mode 100644 src/include/logging/logical_log_manager.h delete mode 100644 src/include/logging/logical_logger.h create mode 100644 src/include/logging/wal_logger.h create mode 100644 src/include/threadpool/logger_queue_pool.h delete mode 100644 src/logging/checkpoint_manager_factory.cpp delete mode 100644 src/logging/log_buffer_pool.cpp delete mode 100644 src/logging/log_manager_factory.cpp create mode 100644 src/logging/wal_log_manager.cpp create mode 100644 src/logging/wal_logger.cpp create mode 100644 src/threadpool/logger_queue_pool.cpp diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index d56f5dd171c..c4e6db38b4d 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include +#include #include "concurrency/timestamp_ordering_transaction_manager.h" #include "catalog/manager.h" @@ -19,7 +20,10 @@ #include "common/platform.h" #include "concurrency/transaction_context.h" #include "gc/gc_manager_factory.h" -#include "logging/log_manager_factory.h" +#include "logging/log_record.h" +#include "logging/log_buffer.h" +#include "logging/wal_logger.h" +#include "threadpool/logger_queue_pool.h" #include "settings/settings_manager.h" namespace peloton { @@ -491,6 +495,26 @@ void TimestampOrderingTransactionManager::PerformInsert( // Write down the head pointer's address in tile group header tile_group_header->SetIndirection(tuple_id, index_entry_ptr); + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_INSERT, location, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); + + current_txn->GetLogBuffer()->WriteRecord(record); + + if(current_txn->GetLogBuffer()->GetSize() >= logging::LogBuffer::GetThreshold()) { + + LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); + + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer(current_txn->GetLogBuffer()); + + /* allocate a new buffer for the current transaction */ + current_txn->ResetLogBuffer(); + } + + + // Increment table insert op stats if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { @@ -694,6 +718,16 @@ void TimestampOrderingTransactionManager::PerformDelete( current_txn->RecordDelete(old_location); +// logging::LogRecord record = +// logging::LogRecordFactory::CreateTupleRecord( +// LogRecordType::TUPLE_DELETE, +// old_location, current_txn->GetEpochId(), +// current_txn->GetTransactionId(), +// current_txn->GetCommitId()); +// +// logging::WalLogger::WriteRecordToBuffer(record, current_txn->GetLogBuffer()); + + // Increment table delete op stats if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { @@ -702,6 +736,7 @@ void TimestampOrderingTransactionManager::PerformDelete( } } +// called when the current transaction creates a new version and then deletes that version. void TimestampOrderingTransactionManager::PerformDelete( TransactionContext *const current_txn, const ItemPointer &location) { PL_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); @@ -720,15 +755,30 @@ void TimestampOrderingTransactionManager::PerformDelete( // Add the old tuple into the delete set auto old_location = tile_group_header->GetNextItemPointer(tuple_id); - if (old_location.IsNull() == false) { + + if (old_location.IsNull() == false) { // update and delete by the current transaction // if this version is not newly inserted. current_txn->RecordDelete(old_location); - } else { + } else { // insert and delete by the current transaction // if this version is newly inserted. current_txn->RecordDelete(location); } - // Increment table delete op stats +// logging::LogRecord record = +// logging::LogRecordFactory::CreateTupleRecord( +// LogRecordType::TUPLE_DELETE, +// location, current_txn->GetEpochId(), +// current_txn->GetTransactionId(), +// current_txn->GetCommitId()); +// +// +// +// logging::WalLogger::WriteRecordToBuffer(record, current_txn->GetLogBuffer()); + + + + + // Increment table delete op stats if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { stats::BackendStatsContext::GetInstance()->IncrementTableDeletes( @@ -753,9 +803,9 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( ////////////////////////////////////////////////////////// auto &manager = catalog::Manager::GetInstance(); - auto &log_manager = logging::LogManager::GetInstance(); + //auto &log_manager = logging::LogManager::GetInstance(); - log_manager.StartLogging(); + //log_manager.StartLogging(); // generate transaction id. cid_t end_commit_id = current_txn->GetCommitId(); @@ -832,7 +882,7 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( gc_set->operator[](tile_group_id)[tuple_slot] = GCVersionType::COMMIT_UPDATE; - log_manager.LogUpdate(new_version); + //log_manager.LogUpdate(new_version); } else if (tuple_entry.second == RWType::DELETE) { ItemPointer new_version = @@ -865,7 +915,7 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( gc_set->operator[](tile_group_id)[tuple_slot] = GCVersionType::COMMIT_DELETE; - log_manager.LogDelete(ItemPointer(tile_group_id, tuple_slot)); + //log_manager.LogDelete(ItemPointer(tile_group_id, tuple_slot)); } else if (tuple_entry.second == RWType::INSERT) { PL_ASSERT(tile_group_header->GetTransactionId(tuple_slot) == @@ -881,7 +931,7 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( // nothing to be added to gc set. - log_manager.LogInsert(ItemPointer(tile_group_id, tuple_slot)); + //log_manager.LogInsert(ItemPointer(tile_group_id, tuple_slot)); } else if (tuple_entry.second == RWType::INS_DEL) { PL_ASSERT(tile_group_header->GetTransactionId(tuple_slot) == @@ -907,7 +957,7 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( ResultType result = current_txn->GetResult(); - log_manager.LogEnd(); + //log_manager.LogEnd(); EndTransaction(current_txn); diff --git a/src/concurrency/transaction_context.cpp b/src/concurrency/transaction_context.cpp index 2fca8420251..c459a7b2a38 100644 --- a/src/concurrency/transaction_context.cpp +++ b/src/concurrency/transaction_context.cpp @@ -18,6 +18,7 @@ #include "common/platform.h" #include "common/macros.h" #include "trigger/trigger.h" +#include "logging/log_buffer.h" #include #include @@ -87,6 +88,8 @@ void TransactionContext::Init(const size_t thread_id, gc_object_set_.reset(new GCObjectSet()); on_commit_triggers_.reset(); + + ResetLogBuffer(); } RWType TransactionContext::GetRWType(const ItemPointer &location) { diff --git a/src/concurrency/transaction_manager.cpp b/src/concurrency/transaction_manager.cpp index 569f394c96f..485d6e4cc23 100644 --- a/src/concurrency/transaction_manager.cpp +++ b/src/concurrency/transaction_manager.cpp @@ -16,7 +16,6 @@ #include "concurrency/transaction_context.h" #include "function/date_functions.h" #include "gc/gc_manager_factory.h" -#include "logging/log_manager.h" #include "settings/settings_manager.h" #include "statistics/stats_aggregator.h" #include "storage/tile_group.h" diff --git a/src/include/concurrency/timestamp_ordering_transaction_manager.h b/src/include/concurrency/timestamp_ordering_transaction_manager.h index ea4453376aa..0c9bd6cdd8c 100644 --- a/src/include/concurrency/timestamp_ordering_transaction_manager.h +++ b/src/include/concurrency/timestamp_ordering_transaction_manager.h @@ -19,6 +19,9 @@ #include "common/synchronization/spin_latch.h" namespace peloton { + +namespace logging {} + namespace concurrency { //===--------------------------------------------------------------------===// diff --git a/src/include/concurrency/transaction_context.h b/src/include/concurrency/transaction_context.h index 4314eed864c..1e6565afd3f 100644 --- a/src/include/concurrency/transaction_context.h +++ b/src/include/concurrency/transaction_context.h @@ -23,6 +23,8 @@ #include "common/item_pointer.h" #include "common/printable.h" #include "common/internal_types.h" +#include "logging/log_buffer.h" + namespace peloton { @@ -75,9 +77,13 @@ class TransactionContext : public Printable { inline uint64_t GetTimestamp() const { return timestamp_; } + inline logging::LogBuffer* GetLogBuffer() const { return log_buffer_; } + inline const std::vector& GetQueryStrings() const { return query_strings_; } + inline void ResetLogBuffer() { log_buffer_ = new logging::LogBuffer(); } + inline void SetCommitId(const cid_t commit_id) { commit_id_ = commit_id; } inline void SetEpochId(const eid_t epoch_id) { epoch_id_ = epoch_id; } @@ -208,6 +214,8 @@ class TransactionContext : public Printable { IsolationLevelType isolation_level_; std::unique_ptr on_commit_triggers_; + + logging::LogBuffer *log_buffer_; }; } // namespace concurrency diff --git a/src/include/logging/checkpoint_manager.h b/src/include/logging/checkpoint_manager.h deleted file mode 100644 index 78d8bc3c120..00000000000 --- a/src/include/logging/checkpoint_manager.h +++ /dev/null @@ -1,69 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// checkpoint_manager.h -// -// Identification: src/include/logging/checkpoint_manager.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include -#include - -#include "common/item_pointer.h" -#include "common/logger.h" -#include "common/macros.h" -#include "common/internal_types.h" - -namespace peloton { -namespace logging { - -//===--------------------------------------------------------------------===// -// checkpoint Manager -//===--------------------------------------------------------------------===// - -class CheckpointManager { - public: - CheckpointManager(const CheckpointManager &) = delete; - CheckpointManager &operator=(const CheckpointManager &) = delete; - CheckpointManager(CheckpointManager &&) = delete; - CheckpointManager &operator=(CheckpointManager &&) = delete; - - CheckpointManager() : is_running_(false) {} - - virtual ~CheckpointManager() {} - - static CheckpointManager &GetInstance() { - static CheckpointManager checkpoint_manager; - return checkpoint_manager; - } - - virtual void Reset() { is_running_ = false; } - - // Get status of whether logging threads are running or not - bool GetStatus() { return this->is_running_; } - - virtual void StartCheckpointing(std::vector> & UNUSED_ATTRIBUTE) {} - - virtual void StartCheckpointing() {} - - virtual void StopCheckpointing() {} - - virtual void RegisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} - - virtual void DeregisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} - - virtual size_t GetTableCount() { return 0; } - - protected: - volatile bool is_running_; -}; - -} // namespace logging -} // namespace peloton diff --git a/src/include/logging/checkpoint_manager_factory.h b/src/include/logging/checkpoint_manager_factory.h deleted file mode 100644 index 44492147ba3..00000000000 --- a/src/include/logging/checkpoint_manager_factory.h +++ /dev/null @@ -1,54 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// checkpoint_manager_factory.h -// -// Identification: src/include/logging/checkpoint_manager_factory.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "logging/checkpoint_manager.h" -#include "logging/logical_checkpoint_manager.h" - -namespace peloton { -namespace logging { - -class CheckpointManagerFactory { - public: - - static CheckpointManager& GetInstance() { - switch (checkpointing_type_) { - - case CheckpointingType::ON: - return LogicalCheckpointManager::GetInstance(checkpointing_thread_count_); - - default: - return CheckpointManager::GetInstance(); - } - } - - static void Configure(const int thread_count = 1) { - if (thread_count == 0) { - checkpointing_type_ = CheckpointingType::OFF; - } else { - checkpointing_type_ = CheckpointingType::ON; - checkpointing_thread_count_ = thread_count; - } - } - - inline static CheckpointingType GetCheckpointingType() { return checkpointing_type_; } - -private: - // checkpointing type - static CheckpointingType checkpointing_type_; - - static int checkpointing_thread_count_; -}; - -} // namespace logging -} // namespace peloton diff --git a/src/include/logging/log_buffer.h b/src/include/logging/log_buffer.h index b4bb727b6a3..99938b65a25 100644 --- a/src/include/logging/log_buffer.h +++ b/src/include/logging/log_buffer.h @@ -1,65 +1,45 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_buffer.h -// -// Identification: src/backend/logging/log_buffer.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// #pragma once -#include "common/macros.h" -#include "common/internal_types.h" +#include "type/serializeio.h" +#include "logging/log_record.h" -namespace peloton { -namespace logging { +namespace peloton{ +namespace logging{ -class LogBuffer { - LogBuffer(const LogBuffer &) = delete; - LogBuffer &operator=(const LogBuffer &) = delete; - LogBuffer(LogBuffer &&) = delete; - LogBuffer &operator=(LogBuffer &&) = delete; +class LogBuffer{ - friend class LogBufferPool; private: - // constant - const static size_t log_buffer_capacity_ = 1024 * 1024 * 32; // 32 MB + //TODO(gandeevan): Tune this size + const static size_t log_buffer_threshold_ = 100; // 4 KB public: - LogBuffer(const size_t thread_id, const size_t eid) : - thread_id_(thread_id), eid_(eid), size_(0){ - data_ = new char[log_buffer_capacity_]; - PL_MEMSET(data_, 0, log_buffer_capacity_); - } - ~LogBuffer() { - delete[] data_; - data_ = nullptr; - } - inline void Reset() { size_ = 0; eid_ = INVALID_EID; } + LogBuffer() : task_callback_(nullptr), + task_callback_arg_(nullptr) {} - inline char *GetData() { return data_; } + ~LogBuffer() {} - inline size_t GetSize() { return size_; } + void WriteRecord(LogRecord &record); - inline size_t GetEpochId() { return eid_; } + size_t GetSize() { return output_buffer.Size(); } - inline size_t GetThreadId() { return thread_id_; } + static size_t GetThreshold() { return log_buffer_threshold_; } - inline bool Empty() { return size_ == 0; } - - bool WriteData(const char *data, size_t len); + void SetTaskCallback(void (*task_callback)(void *), void *task_callback_arg) { + task_callback_ = task_callback; + task_callback_arg_ = task_callback_arg; + } private: - size_t thread_id_; - size_t eid_; - size_t size_; - char* data_; + CopySerializeOutput output_buffer; + + // The current callback to be invoked after logging completes. + void (*task_callback_)(void *); + void *task_callback_arg_; + + }; } -} +} \ No newline at end of file diff --git a/src/include/logging/log_buffer_pool.h b/src/include/logging/log_buffer_pool.h deleted file mode 100644 index e40925a29c5..00000000000 --- a/src/include/logging/log_buffer_pool.h +++ /dev/null @@ -1,63 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_buffer_pool.h -// -// Identification: src/logging/log_buffer_pool.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include - -#include "common/internal_types.h" -#include "common/macros.h" -#include "common/platform.h" - -#include "logging/log_buffer.h" - -namespace peloton { -namespace logging { - class LogBufferPool { - LogBufferPool(const LogBufferPool &) = delete; - LogBufferPool &operator=(const LogBufferPool &) = delete; - LogBufferPool(const LogBufferPool &&) = delete; - LogBufferPool &operator=(const LogBufferPool &&) = delete; - - public: - LogBufferPool(const size_t thread_id) : - head_(0), - tail_(buffer_queue_size_), - thread_id_(thread_id), - local_buffer_queue_(buffer_queue_size_) {} - - std::unique_ptr GetBuffer(const size_t current_eid); - - void PutBuffer(std::unique_ptr buf); - - inline size_t GetThreadId() const { return thread_id_; } - - inline size_t GetEmptySlotCount() const { - return tail_.load() - head_.load(); - } - - inline size_t GetMaxSlotCount() const { - return buffer_queue_size_; - } - - private: - static const size_t buffer_queue_size_ = 16; - std::atomic head_; - std::atomic tail_; - - size_t thread_id_; - - std::vector> local_buffer_queue_; -}; - -} -} diff --git a/src/include/logging/log_manager.h b/src/include/logging/log_manager.h deleted file mode 100644 index 123fce8f56a..00000000000 --- a/src/include/logging/log_manager.h +++ /dev/null @@ -1,79 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_manager.h -// -// Identification: src/include/logging/log_manager.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include -#include - -#include "common/item_pointer.h" -#include "common/logger.h" -#include "common/macros.h" -#include "common/internal_types.h" - -namespace peloton { -namespace logging { - -//===--------------------------------------------------------------------===// -// log Manager -//===--------------------------------------------------------------------===// - -class LogManager { - public: - LogManager(const LogManager &) = delete; - LogManager &operator=(const LogManager &) = delete; - LogManager(LogManager &&) = delete; - LogManager &operator=(LogManager &&) = delete; - - LogManager() : is_running_(false) {} - - virtual ~LogManager() {} - - static LogManager &GetInstance() { - static LogManager log_manager; - return log_manager; - } - - void Reset() { is_running_ = false; } - - // Get status of whether logging threads are running or not - bool GetStatus() { return this->is_running_; } - - virtual void StartLogging(std::vector> & UNUSED_ATTRIBUTE) {} - - virtual void StartLogging() {} - - virtual void StopLogging() {} - - virtual void RegisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} - - virtual void DeregisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} - - virtual size_t GetTableCount() { return 0; } - - virtual void LogBegin() {} - - virtual void LogEnd() {} - - virtual void LogInsert(const ItemPointer & UNUSED_ATTRIBUTE) {} - - virtual void LogUpdate(const ItemPointer & UNUSED_ATTRIBUTE) {} - - virtual void LogDelete(const ItemPointer & UNUSED_ATTRIBUTE) {} - - protected: - volatile bool is_running_; -}; - -} // namespace logging -} // namespace peloton diff --git a/src/include/logging/log_manager_factory.h b/src/include/logging/log_manager_factory.h deleted file mode 100644 index 01b9dc3d521..00000000000 --- a/src/include/logging/log_manager_factory.h +++ /dev/null @@ -1,54 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_manager_factory.h -// -// Identification: src/include/logging/log_manager_factory.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "logging/log_manager.h" -#include "logging/logical_log_manager.h" - -namespace peloton { -namespace logging { - -class LogManagerFactory { - public: - - static LogManager& GetInstance() { - switch (logging_type_) { - - case LoggingType::ON: - return LogicalLogManager::GetInstance(logging_thread_count_); - - default: - return LogManager::GetInstance(); - } - } - - static void Configure(const int thread_count = 1) { - if (thread_count == 0) { - logging_type_ = LoggingType::OFF; - } else { - logging_type_ = LoggingType::ON; - logging_thread_count_ = thread_count; - } - } - - inline static LoggingType GetLoggingType() { return logging_type_; } - -private: - // checkpointing type - static LoggingType logging_type_; - - static int logging_thread_count_; -}; - -} // namespace logging -} // namespace peloton diff --git a/src/include/logging/log_record.h b/src/include/logging/log_record.h index 4f83c8e7528..30af29cb84b 100644 --- a/src/include/logging/log_record.h +++ b/src/include/logging/log_record.h @@ -27,10 +27,11 @@ class LogRecord { friend class LogRecordFactory; private: LogRecord(LogRecordType log_type, const ItemPointer &pos, - const eid_t epoch_id, const cid_t commit_id) + const eid_t epoch_id, const txn_id_t txn_id, const cid_t commit_id) : log_record_type_(log_type), tuple_pos_(pos), - eid_(epoch_id), + eid_(epoch_id), + txn_id_(txn_id), cid_(commit_id) {} public: @@ -44,12 +45,16 @@ class LogRecord { inline void SetCommitId(const cid_t commit_id) { cid_ = commit_id; } + inline void SetTransactionId(const txn_id_t txn_id) { txn_id_ = txn_id; } + inline const ItemPointer &GetItemPointer() { return tuple_pos_; } inline eid_t GetEpochId() { return eid_; } inline cid_t GetCommitId() { return cid_; } + inline txn_id_t GetTransactionId() { return txn_id_; } + private: LogRecordType log_record_type_; @@ -57,30 +62,41 @@ class LogRecord { eid_t eid_; + txn_id_t txn_id_; + cid_t cid_; }; class LogRecordFactory { public: - static LogRecord CreateTupleRecord(const LogRecordType log_type, const ItemPointer &pos) { - PL_ASSERT(log_type == LogRecordType::TUPLE_INSERT || - log_type == LogRecordType::TUPLE_DELETE || - log_type == LogRecordType::TUPLE_UPDATE); - return LogRecord(log_type, pos, INVALID_EID, INVALID_CID); - } - static LogRecord CreateTxnRecord(const LogRecordType log_type, const cid_t commit_id) { - PL_ASSERT(log_type == LogRecordType::TRANSACTION_BEGIN || - log_type == LogRecordType::TRANSACTION_COMMIT); - return LogRecord(log_type, INVALID_ITEMPOINTER, INVALID_EID, commit_id); - } - - static LogRecord CreateEpochRecord(const LogRecordType log_type, const eid_t epoch_id) { - PL_ASSERT(log_type == LogRecordType::EPOCH_BEGIN || - log_type == LogRecordType::EPOCH_END); - return LogRecord(log_type, INVALID_ITEMPOINTER, epoch_id, INVALID_CID); + static LogRecord CreateTupleRecord(const LogRecordType log_type, + const ItemPointer &pos, eid_t current_eid, + txn_id_t txn_id, cid_t current_cid) { + PL_ASSERT(log_type == LogRecordType::TUPLE_INSERT || + log_type == LogRecordType::TUPLE_DELETE || + log_type == LogRecordType::TUPLE_UPDATE); + return LogRecord(log_type, pos, current_eid, txn_id, current_cid); } +// static LogRecord CreateTupleRecord(const LogRecordType log_type, const ItemPointer &pos) { +// PL_ASSERT(log_type == LogRecordType::TUPLE_INSERT || +// log_type == LogRecordType::TUPLE_DELETE || +// log_type == LogRecordType::TUPLE_UPDATE); +// return LogRecord(log_type, pos, INVALID_EID, INVALID_CID); +// } +// +// static LogRecord CreateTxnRecord(const LogRecordType log_type, const cid_t commit_id) { +// PL_ASSERT(log_type == LogRecordType::TRANSACTION_BEGIN || +// log_type == LogRecordType::TRANSACTION_COMMIT); +// return LogRecord(log_type, INVALID_ITEMPOINTER, INVALID_EID, commit_id); +// } +// +// static LogRecord CreateEpochRecord(const LogRecordType log_type, const eid_t epoch_id) { +// PL_ASSERT(log_type == LogRecordType::EPOCH_BEGIN || +// log_type == LogRecordType::EPOCH_END); +// return LogRecord(log_type, INVALID_ITEMPOINTER, epoch_id, INVALID_CID); +// } }; } // namespace logging diff --git a/src/include/logging/logical_checkpoint_manager.h b/src/include/logging/logical_checkpoint_manager.h deleted file mode 100644 index b93a89e680f..00000000000 --- a/src/include/logging/logical_checkpoint_manager.h +++ /dev/null @@ -1,60 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// logical_checkpoint_manager.h -// -// Identification: src/include/logging/logical_checkpoint_manager.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "logging/checkpoint_manager.h" - -namespace peloton { -namespace logging { - -//===--------------------------------------------------------------------===// -// logical checkpoint Manager -//===--------------------------------------------------------------------===// - -class LogicalCheckpointManager : public CheckpointManager { - public: - LogicalCheckpointManager(const LogicalCheckpointManager &) = delete; - LogicalCheckpointManager &operator=(const LogicalCheckpointManager &) = delete; - LogicalCheckpointManager(LogicalCheckpointManager &&) = delete; - LogicalCheckpointManager &operator=(LogicalCheckpointManager &&) = delete; - - LogicalCheckpointManager(const int thread_count) : checkpointer_thread_count_(thread_count) {} - - virtual ~LogicalCheckpointManager() {} - - static LogicalCheckpointManager &GetInstance(const int thread_count = 1) { - static LogicalCheckpointManager checkpoint_manager(thread_count); - return checkpoint_manager; - } - - virtual void Reset() { is_running_ = false; } - - virtual void StartLogging(std::vector> & UNUSED_ATTRIBUTE) {} - - virtual void StartLogging() {} - - virtual void StopLogging() {} - - virtual void RegisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} - - virtual void DeregisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} - - virtual size_t GetTableCount() { return 0; } - - private: - int checkpointer_thread_count_; - -}; - -} // namespace logging -} // namespace peloton diff --git a/src/include/logging/logical_log_manager.h b/src/include/logging/logical_log_manager.h deleted file mode 100644 index a8c46ed5e16..00000000000 --- a/src/include/logging/logical_log_manager.h +++ /dev/null @@ -1,129 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// logical_log_manager.h -// -// Identification: src/include/logging/logical_log_manager.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "logging/log_manager.h" - -namespace peloton { -namespace logging { - -//===--------------------------------------------------------------------===// -// logical log Manager -//===--------------------------------------------------------------------===// - - -/** - * logging file name layout : - * - * dir_name + "/" + prefix + "_" + epoch_id - * - * - * logging file layout : - * - * ----------------------------------------------------------------------------- - * | txn_id | database_id | table_id | operation_type | data | ... | txn_end_flag - * ----------------------------------------------------------------------------- - * - * NOTE: this layout is designed for logical logging. - * - * NOTE: tuple length can be obtained from the table schema. - * - */ - -class LogicalLogManager : public LogManager { - public: - LogicalLogManager(const LogicalLogManager &) = delete; - LogicalLogManager &operator=(const LogicalLogManager &) = delete; - LogicalLogManager(LogicalLogManager &&) = delete; - LogicalLogManager &operator=(LogicalLogManager &&) = delete; - - LogicalLogManager(const int thread_count) : logger_thread_count_(thread_count) {} - - virtual ~LogicalLogManager() {} - - static LogicalLogManager &GetInstance(const int thread_count = 1) { - static LogicalLogManager log_manager(thread_count); - return log_manager; - } - - // virtual void SetDirectories(const std::vector &logging_dirs) override { - // logger_dirs_ = logging_dirs; - - // if (logging_dirs.size() > 0) { - // pepoch_dir_ = logging_dirs.at(0); - // } - // // check the existence of logging directories. - // // if not exists, then create the directory. - // for (auto logging_dir : logging_dirs) { - // if (LoggingUtil::CheckDirectoryExistence(logging_dir.c_str()) == false) { - // LOG_INFO("Logging directory %s is not accessible or does not exist", logging_dir.c_str()); - // bool res = LoggingUtil::CreateDirectory(logging_dir.c_str(), 0700); - // if (res == false) { - // LOG_ERROR("Cannot create directory: %s", logging_dir.c_str()); - // } - // } - // } - - // logger_count_ = logging_dirs.size(); - // for (size_t i = 0; i < logger_count_; ++i) { - // loggers_.emplace_back(new PhyLogLogger(i, logging_dirs.at(i))); - // } - // } - - // virtual const std::vector &GetDirectories() override { - // return logger_dirs_; - // } - - - virtual void StartLogging(std::vector> & UNUSED_ATTRIBUTE) override {} - - virtual void StartLogging() override {} - - virtual void StopLogging() override {} - - virtual void RegisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) override {} - - virtual void DeregisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) override {} - - virtual size_t GetTableCount() override { return 0; } - - virtual void LogBegin() override {} - - virtual void LogEnd() override {} - - virtual void LogInsert(const ItemPointer & UNUSED_ATTRIBUTE) override {} - - virtual void LogUpdate(const ItemPointer & UNUSED_ATTRIBUTE) override {} - - virtual void LogDelete(const ItemPointer & UNUSED_ATTRIBUTE) override { } - - private: - int logger_thread_count_; - - // std::atomic worker_count_; - - // std::vector logger_dirs_; - - // std::vector> loggers_; - - // std::unique_ptr pepoch_thread_; - // volatile bool is_running_; - - // std::string pepoch_dir_; - - // const std::string pepoch_filename_ = "pepoch"; - -}; - -} // namespace logging -} // namespace peloton diff --git a/src/include/logging/logical_logger.h b/src/include/logging/logical_logger.h deleted file mode 100644 index 9bcde68db3c..00000000000 --- a/src/include/logging/logical_logger.h +++ /dev/null @@ -1,148 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// logical_logger.h -// -// Identification: src/logging/logical_logger.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include -#include -#include -#include - -#include "concurrency/transaction_context.h" -#include "concurrency/epoch_manager.h" -#include "logging/log_buffer.h" -#include "logging/log_record.h" -#include "logging/log_buffer_pool.h" -#include "logging/log_manager.h" -#include "logging/worker_context.h" -#include "type/types.h" -#include "common/serializer.h" -#include "common/lockfree_queue.h" -#include "common/logger.h" -#include "common/pool.h" -#include "common/synchronization/spin_lock.h" - -namespace peloton { - -namespace storage { - class TileGroupHeader; -} - -namespace logging { - - class PhyLogLogger { - - public: - PhyLogLogger(const size_t &logger_id, const std::string &log_dir) : - logger_id_(logger_id), - log_dir_(log_dir), - logger_thread_(nullptr), - is_running_(false), - logger_output_buffer_(), - persist_epoch_id_(INVALID_EID), - worker_map_lock_(), - worker_map_() {} - - ~PhyLogLogger() {} - - void StartRecovery(const size_t checkpoint_eid, const size_t persist_eid, const size_t recovery_thread_count); - void StartIndexRebulding(const size_t logger_count); - - void WaitForRecovery(); - void WaitForIndexRebuilding(); - - void StartLogging() { - is_running_ = true; - logger_thread_.reset(new std::thread(&PhyLogLogger::Run, this)); - } - - void StopLogging() { - is_running_ = false; - logger_thread_->join(); - } - - void RegisterWorker(WorkerContext *phylog_worker_ctx); - void DeregisterWorker(WorkerContext *phylog_worker_ctx); - - size_t GetPersistEpochId() const { - return persist_epoch_id_; - } - - -private: - void Run(); - - void PersistEpochBegin(FileHandle &file_handle, const size_t epoch_id); - void PersistEpochEnd(FileHandle &file_handle, const size_t epoch_id); - void PersistLogBuffer(FileHandle &file_handle, std::unique_ptr log_buffer); - - std::string GetLogFileFullPath(size_t epoch_id) { - return log_dir_ + "/" + logging_filename_prefix_ + "_" + std::to_string(logger_id_) + "_" + std::to_string(epoch_id); - } - - void GetSortedLogFileIdList(const size_t checkpoint_eid, const size_t persist_eid); - - void RunRecoveryThread(const size_t thread_id, const size_t checkpoint_eid, const size_t persist_eid); - - void RunSecIndexRebuildThread(const size_t logger_count); - - void RebuildSecIndexForTable(const size_t logger_count, storage::DataTable *table); - - bool ReplayLogFile(const size_t thread_id, FileHandle &file_handle, size_t checkpoint_eid, size_t pepoch_eid); - bool InstallTupleRecord(LogRecordType type, storage::Tuple *tuple, storage::DataTable *table, cid_t cur_cid); - - // Return value is the swapped txn id, either INVALID_TXNID or INITIAL_TXNID - txn_id_t LockTuple(storage::TileGroupHeader *tg_header, oid_t tuple_offset); - void UnlockTuple(storage::TileGroupHeader *tg_header, oid_t tuple_offset, txn_id_t new_txn_id); - - private: - size_t logger_id_; - std::string log_dir_; - - // recovery threads - std::vector> recovery_threads_; - std::vector file_eids_; - std::atomic max_replay_file_id_; - - /* Recovery */ - // TODO: Check if we can discard the recovery pool after the recovery is done. - // Since every thing is copied to the tile group and tile group related pool - std::vector> recovery_pools_; - - // logger thread - std::unique_ptr logger_thread_; - volatile bool is_running_; - - /* File system related */ - CopySerializeOutput logger_output_buffer_; - - /* Log buffers */ - size_t persist_epoch_id_; - - // The spin lock to protect the worker map. - // We only update this map when creating/terminating a new worker - common::synchronization::SpinLatch worker_map_lock_; - - // map from worker id to the worker's context. - std::unordered_map> worker_map_; - - const std::string logging_filename_prefix_ = "log"; - - const size_t sleep_period_us_ = 40000; - - const int new_file_interval_ = 500; // 500 milliseconds. - }; - - -} -} \ No newline at end of file diff --git a/src/include/logging/wal_logger.h b/src/include/logging/wal_logger.h new file mode 100644 index 00000000000..e2f4d6eba51 --- /dev/null +++ b/src/include/logging/wal_logger.h @@ -0,0 +1,49 @@ +// +// Created by Gandeevan R on 2/12/18. +// + +#pragma once + +#include "logging/log_record.h" +#include "type/serializeio.h" + + +namespace peloton { + +namespace storage { + class TileGroupHeader; +} + +namespace logging { + +#define TRANSACTION_BUFFER_LIMIT 2048; + +class WalLogger { + public: + +// WalLogger(const size_t &log_id, const std::string &log_dir) +// : log_id_(log_id), log_dir_(log_dir) {} + + WalLogger() {} + + ~WalLogger() {} + + + +// private: +// std::string GetLogFileFullPath() { +// return log_dir_ + "/" + logging_filename_prefix_ + "_" + +// std::to_string(log_id_); +// } + + +// private: +// size_t log_id_; +// std::string log_dir_; +// +// const std::string logging_filename_prefix_ = "logfile"; +}; +} +} + + diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index 764bd66402e..959735e5d8f 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -47,9 +47,7 @@ namespace index { class Index; } -namespace logging { -class LogManager; -} + namespace concurrency { class TransactionContext; @@ -78,7 +76,6 @@ class DataTable : public AbstractTable { friend class TileGroup; friend class TileGroupFactory; friend class TableFactory; - friend class logging::LogManager; DataTable() = delete; DataTable(DataTable const &) = delete; diff --git a/src/include/threadpool/logger_queue_pool.h b/src/include/threadpool/logger_queue_pool.h new file mode 100644 index 00000000000..cbc1141b403 --- /dev/null +++ b/src/include/threadpool/logger_queue_pool.h @@ -0,0 +1,82 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// logger_queue_pool.h +// +// Identification: test/threadpool/Logger_queue_pool.h +// +// Copyright (c) 2015-17, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once +#include "worker_pool.h" +#include "logging/log_buffer.h" +#include "logging/wal_logger.h" + +// TODO: tune these variables +constexpr static size_t kDefaultLoggerQueueSize = 32; +constexpr static size_t kDefaultLoggerPoolSize = 1; + +namespace peloton { +namespace threadpool { + +using LoggerQueue = peloton::LockFreeQueue; +void LoggerFunc(std::atomic_bool *is_running, LoggerQueue *logger_queue); + + + /** +* @class LoggerQueuePool +* @brief Wrapper class for single queue and single pool +* One should use this if possible. +*/ +class LoggerQueuePool { + public: + LoggerQueuePool(size_t num_workers) + : logger_queue_(kDefaultLoggerQueueSize), + num_workers_(num_workers), + is_running_(false) {} + + ~LoggerQueuePool() { + if (is_running_ == true) + Shutdown(); + } + + void Startup() { + for (size_t i = 0; i < num_workers_; i++) { + loggers_.emplace_back(LoggerFunc, &is_running_, &logger_queue_); + } + is_running_ = true; + } + + void Shutdown() { + for (auto &logger : loggers_) { + logger.join(); + } + loggers_.clear(); + is_running_ = false; + } + + void SubmitLogBuffer(logging::LogBuffer *buffer) { + if (is_running_ == false) + Startup(); + logger_queue_.Enqueue(std::move(buffer)); + } + + + static LoggerQueuePool &GetInstance() { + static LoggerQueuePool logger_queue_pool(kDefaultLoggerPoolSize); + return logger_queue_pool; + } + + private: + LoggerQueue logger_queue_; + std::vector loggers_; + + size_t num_workers_; + std::atomic_bool is_running_; +}; + +} // namespace threadpool +} // namespace peloton diff --git a/src/logging/checkpoint_manager_factory.cpp b/src/logging/checkpoint_manager_factory.cpp deleted file mode 100644 index 4f025729a7d..00000000000 --- a/src/logging/checkpoint_manager_factory.cpp +++ /dev/null @@ -1,23 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// checkpoint_manager_factory.cpp -// -// Identification: src/logging/checkpoint_manager_factory.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#include "logging/checkpoint_manager_factory.h" - -namespace peloton { -namespace logging { - -CheckpointingType CheckpointManagerFactory::checkpointing_type_ = CheckpointingType::ON; -int CheckpointManagerFactory::checkpointing_thread_count_ = 1; - -} // namespace gc -} // namespace peloton diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index 0c204778a4a..49e7f024343 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -1,33 +1,81 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_buffer.cpp -// -// Identification: src/logging/log_buffer.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#include "common/macros.h" + + +#include "logging/log_record.h" #include "logging/log_buffer.h" +#include "logging/wal_logger.h" +#include "catalog/manager.h" +#include "common/container_tuple.h" +#include "storage/tile_group.h" + + +namespace peloton{ +namespace logging{ + +void LogBuffer::WriteRecord(LogRecord &record) { + + // Reserve for the frame length + size_t start = output_buffer.Position(); + output_buffer.WriteInt(0); + + LogRecordType type = record.GetType(); + output_buffer.WriteEnumInSingleByte( + static_cast(type)); + + output_buffer.WriteLong(record.GetEpochId()); + output_buffer.WriteLong(record.GetTransactionId()); + output_buffer.WriteLong(record.GetCommitId()); + -namespace peloton { -namespace logging { - -bool LogBuffer::WriteData(const char *data, size_t len) { - if (unlikely_branch(size_ + len > log_buffer_capacity_)) { - return false; - } else { - PL_ASSERT(data); - PL_ASSERT(len); - PL_MEMCPY(data_ + size_, data, len); - size_ += len; - return true; + switch (type) { + case LogRecordType::TUPLE_INSERT: { + auto &manager = catalog::Manager::GetInstance(); + auto tuple_pos = record.GetItemPointer(); + auto tg = manager.GetTileGroup(tuple_pos.block).get(); + std::vector columns; + + // Write down the database id and the table id + output_buffer.WriteLong(tg->GetDatabaseId()); + output_buffer.WriteLong(tg->GetTableId()); + + output_buffer.WriteLong(tuple_pos.block); + output_buffer.WriteLong(tuple_pos.offset); + + // Write the full tuple into the buffer + for (auto schema : tg->GetTileSchemas()) { + for (auto column : schema.GetColumns()) { + columns.push_back(column); + } + } + + ContainerTuple container_tuple(tg, tuple_pos.offset); + for (oid_t oid = 0; oid < columns.size(); oid++) { + + // TODO: check if GetValue() returns variable length fields appropriately + auto val = container_tuple.GetValue(oid); + val.SerializeTo(output_buffer); + } + break; + } + case LogRecordType::TUPLE_DELETE: { + LOG_ERROR("Delete logging not supported"); + PL_ASSERT(false); + } + case LogRecordType::TUPLE_UPDATE: { + LOG_ERROR("Update logging not supported"); + PL_ASSERT(false); + } + default: { + LOG_ERROR("Unsupported log record type"); + PL_ASSERT(false); + + } } -} + // Add the frame length + // XXX: We rely on the fact that the serializer treat a int32_t as 4 bytes + int32_t length = output_buffer.Position() - start - sizeof(int32_t); + output_buffer.WriteIntAt(start, length); } + } +} \ No newline at end of file diff --git a/src/logging/log_buffer_pool.cpp b/src/logging/log_buffer_pool.cpp deleted file mode 100644 index 8c2f5365b25..00000000000 --- a/src/logging/log_buffer_pool.cpp +++ /dev/null @@ -1,63 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_buffer_pool.cpp -// -// Identification: src/logging/log_buffer_pool.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#include "logging/log_buffer_pool.h" -#include "common/logger.h" - -namespace peloton { -namespace logging { - - // Acquire a log buffer from the buffer pool. - // This function will be blocked until there is an available buffer. - // Note that only the corresponding worker thread can call this function. - std::unique_ptr LogBufferPool::GetBuffer(size_t current_eid) { - size_t head_idx = head_ % buffer_queue_size_; - while (true) { - if (head_.load() < tail_.load() - 1) { - if (local_buffer_queue_[head_idx].get() == nullptr) { - // Not any buffer allocated now - local_buffer_queue_[head_idx].reset(new LogBuffer(thread_id_, current_eid)); - } - break; - } - - // sleep a while, and try to get a new buffer - _mm_pause(); - LOG_TRACE("Worker %d uses up its buffer", (int) thread_id_); - } - - head_.fetch_add(1, std::memory_order_relaxed); - local_buffer_queue_[head_idx]->eid_ = current_eid; - return std::move(local_buffer_queue_[head_idx]); - } - - // This function is called only by the corresponding logger. - void LogBufferPool::PutBuffer(std::unique_ptr buf) { - PL_ASSERT(buf.get() != nullptr); - PL_ASSERT(buf->GetThreadId() == thread_id_); - - size_t tail_idx = tail_ % buffer_queue_size_; - - // The buffer pool must not be full - PL_ASSERT(tail_idx != head_ % buffer_queue_size_); - // The tail pos must be null - PL_ASSERT(local_buffer_queue_[tail_idx].get() == nullptr); - // The returned buffer must be empty - PL_ASSERT(buf->Empty() == true); - local_buffer_queue_[tail_idx].reset(buf.release()); - - tail_.fetch_add(1, std::memory_order_relaxed); - } - -} -} diff --git a/src/logging/log_manager_factory.cpp b/src/logging/log_manager_factory.cpp deleted file mode 100644 index e537c5dcbc8..00000000000 --- a/src/logging/log_manager_factory.cpp +++ /dev/null @@ -1,24 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_manager_factory.cpp -// -// Identification: src/logging/log_manager_factory.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#include "logging/log_manager_factory.h" - -namespace peloton { -namespace logging { - -LoggingType LogManagerFactory::logging_type_ = LoggingType::ON; - -int LogManagerFactory::logging_thread_count_ = 1; - -} // namespace gc -} // namespace peloton diff --git a/src/logging/wal_log_manager.cpp b/src/logging/wal_log_manager.cpp new file mode 100644 index 00000000000..b336a0e8c2b --- /dev/null +++ b/src/logging/wal_log_manager.cpp @@ -0,0 +1,11 @@ +// +// Created by Gandeevan R on 2/20/18. +// + +namespace peloton{ +namespace logging{ + + + +} +} \ No newline at end of file diff --git a/src/logging/wal_logger.cpp b/src/logging/wal_logger.cpp new file mode 100644 index 00000000000..793fe5b577f --- /dev/null +++ b/src/logging/wal_logger.cpp @@ -0,0 +1,22 @@ +// +// Created by Gandeevan R on 2/12/18. +// + +#include "logging/wal_logger.h" +#include "catalog/manager.h" +#include "common/container_tuple.h" +#include "storage/tile_group.h" + + + +namespace peloton{ +namespace logging{ + + + + + + + +} +} \ No newline at end of file diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 8a8fede95c6..291b616083b 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -26,7 +26,6 @@ #include "concurrency/transaction_manager_factory.h" #include "gc/gc_manager_factory.h" #include "index/index.h" -#include "logging/log_manager.h" #include "storage/abstract_table.h" #include "storage/data_table.h" #include "storage/database.h" diff --git a/src/storage/tile_group_header.cpp b/src/storage/tile_group_header.cpp index 3bdd0cb4abd..a24d7e4bb89 100644 --- a/src/storage/tile_group_header.cpp +++ b/src/storage/tile_group_header.cpp @@ -22,7 +22,6 @@ #include "common/printable.h" #include "concurrency/transaction_manager_factory.h" #include "gc/gc_manager.h" -#include "logging/log_manager.h" #include "storage/backend_manager.h" #include "type/value.h" #include "storage/tuple.h" diff --git a/src/threadpool/logger_queue_pool.cpp b/src/threadpool/logger_queue_pool.cpp new file mode 100644 index 00000000000..95b36ef2713 --- /dev/null +++ b/src/threadpool/logger_queue_pool.cpp @@ -0,0 +1,34 @@ +#include "threadpool/logger_queue_pool.h" +#include "common/container/lock_free_queue.h" + + + +namespace peloton{ +namespace threadpool{ + +void LoggerFunc(std::atomic_bool *is_running, LoggerQueue *logger_queue) { + constexpr auto kMinPauseTime = std::chrono::microseconds(1); + constexpr auto kMaxPauseTime = std::chrono::microseconds(1000); + auto pause_time = kMinPauseTime; + + logging::WalLogger logger; + + while (is_running->load() || !logger_queue->IsEmpty()) { + + logging::LogBuffer *log_buffer; + + if (!logger_queue->Dequeue(log_buffer)) { + // Polling with exponential backoff + std::this_thread::sleep_for(pause_time); + pause_time = std::min(pause_time * 2, kMaxPauseTime); + } else { + LOG_DEBUG("Logger dequeued a log buffer %p", (void *) log_buffer); + + pause_time = kMinPauseTime; + } + } + +} + +} +} \ No newline at end of file From e318f532ae234dedcdbaa5db9ccc54912796beaf Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Wed, 21 Feb 2018 20:12:29 -0500 Subject: [PATCH 02/81] insert logging refactoring --- src/common/init.cpp | 9 +++ ...timestamp_ordering_transaction_manager.cpp | 4 +- src/include/concurrency/transaction_context.h | 6 +- src/include/logging/log_buffer.h | 27 +++++--- src/include/logging/wal_log_manager.h | 65 +++++++++++++++++++ src/include/logging/wal_logger.h | 36 +++++----- src/logging/log_buffer.cpp | 26 ++++---- src/logging/wal_log_manager.cpp | 11 ---- src/logging/wal_logger.cpp | 65 +++++++++++++++++++ src/threadpool/logger_queue_pool.cpp | 13 +++- 10 files changed, 206 insertions(+), 56 deletions(-) create mode 100644 src/include/logging/wal_log_manager.h delete mode 100644 src/logging/wal_log_manager.cpp diff --git a/src/common/init.cpp b/src/common/init.cpp index 17e7606cf8a..369287e96f8 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -24,6 +24,7 @@ #include "gc/gc_manager_factory.h" #include "settings/settings_manager.h" #include "threadpool/mono_queue_pool.h" +#include "logging/wal_log_manager.h" namespace peloton { @@ -88,6 +89,14 @@ void PelotonInit::Initialize() { // Initialize the Statement Cache Manager StatementCacheManager::Init(); + + // TODO(gandeevan): start logger thread + if(logging::LogManager::GetInstance().init("/tmp")) { + LOG_DEBUG("logging enabled"); + } else { + LOG_DEBUG("logging disabled"); + } + } void PelotonInit::Shutdown() { diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index c4e6db38b4d..4c954557286 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -502,7 +502,7 @@ void TimestampOrderingTransactionManager::PerformInsert( current_txn->GetLogBuffer()->WriteRecord(record); - if(current_txn->GetLogBuffer()->GetSize() >= logging::LogBuffer::GetThreshold()) { + if(current_txn->GetLogBuffer()->HasThresholdExceeded()) { LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); @@ -788,7 +788,7 @@ void TimestampOrderingTransactionManager::PerformDelete( ResultType TimestampOrderingTransactionManager::CommitTransaction( TransactionContext *const current_txn) { - LOG_TRACE("Committing peloton txn : %" PRId64, current_txn->GetTransactionId()); + LOG_DEBUG("Committing peloton txn : %" PRId64, current_txn->GetTransactionId()); ////////////////////////////////////////////////////////// //// handle READ_ONLY diff --git a/src/include/concurrency/transaction_context.h b/src/include/concurrency/transaction_context.h index 1e6565afd3f..e9a376a0bdf 100644 --- a/src/include/concurrency/transaction_context.h +++ b/src/include/concurrency/transaction_context.h @@ -24,7 +24,7 @@ #include "common/printable.h" #include "common/internal_types.h" #include "logging/log_buffer.h" - +#include "logging/wal_log_manager.h" namespace peloton { @@ -82,7 +82,9 @@ class TransactionContext : public Printable { inline const std::vector& GetQueryStrings() const { return query_strings_; } - inline void ResetLogBuffer() { log_buffer_ = new logging::LogBuffer(); } + inline void ResetLogBuffer() { + log_buffer_ = new logging::LogBuffer(logging::LogManager::GetInstance().GetTransactionBufferSize()); + } inline void SetCommitId(const cid_t commit_id) { commit_id_ = commit_id; } diff --git a/src/include/logging/log_buffer.h b/src/include/logging/log_buffer.h index 99938b65a25..2934767b131 100644 --- a/src/include/logging/log_buffer.h +++ b/src/include/logging/log_buffer.h @@ -7,24 +7,32 @@ namespace peloton{ namespace logging{ -class LogBuffer{ -private: - //TODO(gandeevan): Tune this size - const static size_t log_buffer_threshold_ = 100; // 4 KB +typedef void (*CallbackFunction)(void *); // function pointer type + +class LogBuffer{ public: - LogBuffer() : task_callback_(nullptr), - task_callback_arg_(nullptr) {} + LogBuffer(size_t threshold) + : log_buffer_threshold_(threshold), + task_callback_(nullptr), + task_callback_arg_(nullptr) {} ~LogBuffer() {} void WriteRecord(LogRecord &record); - size_t GetSize() { return output_buffer.Size(); } + inline const char *GetData() { return log_buffer_.Data(); } + inline size_t GetSize() { return log_buffer_.Size(); } + + + inline CopySerializeOutput &GetCopySerializedOutput() { return log_buffer_; } - static size_t GetThreshold() { return log_buffer_threshold_; } + inline bool HasThresholdExceeded() { return log_buffer_.Size() >= log_buffer_threshold_; } + inline size_t GetThreshold() { return log_buffer_threshold_; } + inline CallbackFunction GetCallback() { return task_callback_; } + inline void* GetCallbackArgs() { return task_callback_arg_; } void SetTaskCallback(void (*task_callback)(void *), void *task_callback_arg) { task_callback_ = task_callback; @@ -32,7 +40,8 @@ class LogBuffer{ } private: - CopySerializeOutput output_buffer; + CopySerializeOutput log_buffer_; + size_t log_buffer_threshold_; // The current callback to be invoked after logging completes. void (*task_callback_)(void *); diff --git a/src/include/logging/wal_log_manager.h b/src/include/logging/wal_log_manager.h new file mode 100644 index 00000000000..33fef8f7a2d --- /dev/null +++ b/src/include/logging/wal_log_manager.h @@ -0,0 +1,65 @@ + + +#pragma once + +#include + +namespace peloton{ +namespace logging{ + +class LogManager{ +public: + + LogManager() {} + ~LogManager() {} + + inline bool init(std::string logging_dir) { + directory_ = logging_dir; + + if(!logger_ofstream_.is_open()){ + logger_ofstream_.open(logging_dir+"/log", std::ofstream::out | std::ofstream::app); + + if(!logger_ofstream_.fail()) { + enable_logging_ = true; + return true; + } + + } + + enable_logging_ = false; + return false; + } + + inline static LogManager &GetInstance(){ + static LogManager log_manager; + return log_manager; + } + + inline static size_t GetTransactionBufferSize() { return transaction_buffer_threshold_; } + inline static size_t GetLoggerBufferSize() { return logger_buffer_threshold_; } + + inline bool IsLoggingEnabled() { return enable_logging_; } + inline std::string GetDirectory() { return directory_; } + inline std::string GetLogFilePath() { return directory_+"/tmp"; } + inline std::ofstream *GetFileStream() { return &logger_ofstream_; } + +// TODO(gandeevan): support StartLogging and StopLogging +// static void StartLogging() {} +// static void StopLogging() {} + + +private: + + const static size_t logger_buffer_threshold_ = 1024 * 512; // 512 KB + const static size_t transaction_buffer_threshold_ = 1024 * 16; // 16 KB + + // NOTE: ofstream is not thread safe, might need to change it if more than one logger thread is used + bool enable_logging_; + std::string directory_; + std::ofstream logger_ofstream_; + + +}; + +} +} \ No newline at end of file diff --git a/src/include/logging/wal_logger.h b/src/include/logging/wal_logger.h index e2f4d6eba51..ae39598091d 100644 --- a/src/include/logging/wal_logger.h +++ b/src/include/logging/wal_logger.h @@ -4,7 +4,11 @@ #pragma once +#include + +#include "logging/log_buffer.h" #include "logging/log_record.h" +#include "logging/wal_log_manager.h" #include "type/serializeio.h" @@ -16,32 +20,30 @@ namespace storage { namespace logging { -#define TRANSACTION_BUFFER_LIMIT 2048; -class WalLogger { - public: -// WalLogger(const size_t &log_id, const std::string &log_dir) -// : log_id_(log_id), log_dir_(log_dir) {} +class WalLogger { +private: + struct logger_callback{ - WalLogger() {} + }; - ~WalLogger() {} +public: + WalLogger() { + disk_buffer_ = new LogBuffer(LogManager::GetLoggerBufferSize()); + } + ~WalLogger() {} -// private: -// std::string GetLogFileFullPath() { -// return log_dir_ + "/" + logging_filename_prefix_ + "_" + -// std::to_string(log_id_); -// } + bool IsFlushNeeded(bool pending_buffers); + void FlushToDisk(); + void AppendLogBuffer(LogBuffer *buffer); +private: + LogBuffer *disk_buffer_; + std::deque> callbacks_; -// private: -// size_t log_id_; -// std::string log_dir_; -// -// const std::string logging_filename_prefix_ = "logfile"; }; } } diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index 49e7f024343..e2e86bd1060 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -14,16 +14,16 @@ namespace logging{ void LogBuffer::WriteRecord(LogRecord &record) { // Reserve for the frame length - size_t start = output_buffer.Position(); - output_buffer.WriteInt(0); + size_t start = log_buffer_.Position(); + log_buffer_.WriteInt(0); LogRecordType type = record.GetType(); - output_buffer.WriteEnumInSingleByte( + log_buffer_.WriteEnumInSingleByte( static_cast(type)); - output_buffer.WriteLong(record.GetEpochId()); - output_buffer.WriteLong(record.GetTransactionId()); - output_buffer.WriteLong(record.GetCommitId()); + log_buffer_.WriteLong(record.GetEpochId()); + log_buffer_.WriteLong(record.GetTransactionId()); + log_buffer_.WriteLong(record.GetCommitId()); switch (type) { @@ -34,11 +34,11 @@ void LogBuffer::WriteRecord(LogRecord &record) { std::vector columns; // Write down the database id and the table id - output_buffer.WriteLong(tg->GetDatabaseId()); - output_buffer.WriteLong(tg->GetTableId()); + log_buffer_.WriteLong(tg->GetDatabaseId()); + log_buffer_.WriteLong(tg->GetTableId()); - output_buffer.WriteLong(tuple_pos.block); - output_buffer.WriteLong(tuple_pos.offset); + log_buffer_.WriteLong(tuple_pos.block); + log_buffer_.WriteLong(tuple_pos.offset); // Write the full tuple into the buffer for (auto schema : tg->GetTileSchemas()) { @@ -52,7 +52,7 @@ void LogBuffer::WriteRecord(LogRecord &record) { // TODO: check if GetValue() returns variable length fields appropriately auto val = container_tuple.GetValue(oid); - val.SerializeTo(output_buffer); + val.SerializeTo(log_buffer_); } break; } @@ -73,8 +73,8 @@ void LogBuffer::WriteRecord(LogRecord &record) { // Add the frame length // XXX: We rely on the fact that the serializer treat a int32_t as 4 bytes - int32_t length = output_buffer.Position() - start - sizeof(int32_t); - output_buffer.WriteIntAt(start, length); + int32_t length = log_buffer_.Position() - start - sizeof(int32_t); + log_buffer_.WriteIntAt(start, length); } } diff --git a/src/logging/wal_log_manager.cpp b/src/logging/wal_log_manager.cpp deleted file mode 100644 index b336a0e8c2b..00000000000 --- a/src/logging/wal_log_manager.cpp +++ /dev/null @@ -1,11 +0,0 @@ -// -// Created by Gandeevan R on 2/20/18. -// - -namespace peloton{ -namespace logging{ - - - -} -} \ No newline at end of file diff --git a/src/logging/wal_logger.cpp b/src/logging/wal_logger.cpp index 793fe5b577f..dc37a1aeb7c 100644 --- a/src/logging/wal_logger.cpp +++ b/src/logging/wal_logger.cpp @@ -2,7 +2,9 @@ // Created by Gandeevan R on 2/12/18. // +#include #include "logging/wal_logger.h" +#include "logging/log_buffer.h" #include "catalog/manager.h" #include "common/container_tuple.h" #include "storage/tile_group.h" @@ -12,6 +14,69 @@ namespace peloton{ namespace logging{ +bool WalLogger::IsFlushNeeded(bool pending_buffers){ + + + + /* if the disk buffer is full then we flush to the disk + * irrespective of whether there are pending callbacks + */ + if(disk_buffer_->HasThresholdExceeded()) + return true; + + /* if the disk buffer is not full and there are pending buffers + * in the queue then we don't flush to the disk now. We try to + * consolidate the pending buffers to achieve group commit. + */ + if(pending_buffers) + return false; + + /* if there are no pending buffers (at the time of check), but + * there are pending callbacks, we go ahead and flush the + * buffers. + */ + if(!callbacks_.empty()) + return true; + + return false; +} + + +void WalLogger::FlushToDisk(){ + std::ofstream *stream = LogManager::GetInstance().GetFileStream(); + stream->write(disk_buffer_->GetData(), disk_buffer_->GetSize()); + stream->flush(); + disk_buffer_->GetCopySerializedOutput().Reset(); + + /* send out the callbacks */ + while(!callbacks_.empty()){ + auto callback_pair = callbacks_.front(); + CallbackFunction callback_func = callback_pair.first; + void *callback_args = callback_pair.second; + + callback_func(callback_args); + } +} + + +void WalLogger::AppendLogBuffer(LogBuffer *log_buffer){ + + if(nullptr==log_buffer) + PL_ASSERT(false); + + disk_buffer_->GetCopySerializedOutput() + .WriteBytes(log_buffer->GetData(), + log_buffer->GetSize()); + + CallbackFunction callback = log_buffer->GetCallback(); + void *callback_args = log_buffer->GetCallbackArgs(); + + if(nullptr != callback){ + callbacks_.emplace_back(callback, callback_args); + } + + +} diff --git a/src/threadpool/logger_queue_pool.cpp b/src/threadpool/logger_queue_pool.cpp index 95b36ef2713..46b496e93a4 100644 --- a/src/threadpool/logger_queue_pool.cpp +++ b/src/threadpool/logger_queue_pool.cpp @@ -1,11 +1,13 @@ #include "threadpool/logger_queue_pool.h" #include "common/container/lock_free_queue.h" - +#include "logging/wal_logger.h" namespace peloton{ namespace threadpool{ + + void LoggerFunc(std::atomic_bool *is_running, LoggerQueue *logger_queue) { constexpr auto kMinPauseTime = std::chrono::microseconds(1); constexpr auto kMaxPauseTime = std::chrono::microseconds(1000); @@ -22,7 +24,14 @@ void LoggerFunc(std::atomic_bool *is_running, LoggerQueue *logger_queue) { std::this_thread::sleep_for(pause_time); pause_time = std::min(pause_time * 2, kMaxPauseTime); } else { - LOG_DEBUG("Logger dequeued a log buffer %p", (void *) log_buffer); + + logger.AppendLogBuffer(log_buffer); + + if(logger.IsFlushNeeded(logger_queue->IsEmpty())){ + logger.FlushToDisk(); + } + + // TODO(gandeevan): free log buffers pause_time = kMinPauseTime; } From a214b5735b2faabd35c6377ffa6a3337d2623a25 Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Wed, 21 Mar 2018 17:12:34 -0400 Subject: [PATCH 03/81] stored single statement txn status in transaction context --- .../timestamp_ordering_transaction_manager.cpp | 6 ++++++ src/concurrency/transaction_context.cpp | 2 ++ src/include/concurrency/transaction_context.h | 11 +++++++++++ src/traffic_cop/traffic_cop.cpp | 3 +++ 4 files changed, 22 insertions(+) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 89bcea023a0..1c421eea9ff 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -740,6 +740,12 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( TransactionContext *const current_txn) { LOG_TRACE("Committing peloton txn : %" PRId64, current_txn->GetTransactionId()); + if(current_txn->IsSingleStatementTxn()){ + LOG_INFO("Committing single stmt txn peloton txn : %" PRId64, current_txn->GetTransactionId()); + } else { + LOG_INFO("Committing multi stmt txn peloton txn : %" PRId64, current_txn->GetTransactionId()); + } + ////////////////////////////////////////////////////////// //// handle READ_ONLY ////////////////////////////////////////////////////////// diff --git a/src/concurrency/transaction_context.cpp b/src/concurrency/transaction_context.cpp index 30a9141a179..a1525b4bfd2 100644 --- a/src/concurrency/transaction_context.cpp +++ b/src/concurrency/transaction_context.cpp @@ -95,6 +95,8 @@ void TransactionContext::Init(const size_t thread_id, insert_count_ = 0; + single_statement_txn_ = true; + rw_set_.Clear(); gc_set_.reset(new GCSet()); gc_object_set_.reset(new GCObjectSet()); diff --git a/src/include/concurrency/transaction_context.h b/src/include/concurrency/transaction_context.h index 61e3db64f21..9a97ed609df 100644 --- a/src/include/concurrency/transaction_context.h +++ b/src/include/concurrency/transaction_context.h @@ -155,6 +155,14 @@ class TransactionContext : public Printable { return isolation_level_; } + inline bool IsSingleStatementTxn(){ + return single_statement_txn_; + } + + inline void SetSingleStatementTxn(bool single_statement_txn){ + single_statement_txn_ = single_statement_txn; + } + // cache for table catalog objects catalog::CatalogCache catalog_cache; @@ -163,6 +171,9 @@ class TransactionContext : public Printable { // Data members //===--------------------------------------------------------------------===// + //single statement txn + bool single_statement_txn_; + // transaction id txn_id_t txn_id_; diff --git a/src/traffic_cop/traffic_cop.cpp b/src/traffic_cop/traffic_cop.cpp index 713fbc85e35..0105107d242 100644 --- a/src/traffic_cop/traffic_cop.cpp +++ b/src/traffic_cop/traffic_cop.cpp @@ -275,6 +275,7 @@ std::shared_ptr TrafficCop::PrepareStatement( // --multi-statements except BEGIN in a transaction if (!tcop_txn_state_.empty()) { single_statement_txn_ = false; + tcop_txn_state_.top().first->SetSingleStatementTxn(single_statement_txn_); // multi-statment txn has been aborted, just skip this query, // and do not need to parse or execute this query anymore. // Do not return nullptr in case that 'COMMIT' cannot be execute, @@ -297,6 +298,8 @@ std::shared_ptr TrafficCop::PrepareStatement( single_statement_txn_ = true; } auto txn = txn_manager.BeginTransaction(thread_id); + txn->SetSingleStatementTxn(single_statement_txn_); + // this shouldn't happen if (txn == nullptr) { LOG_TRACE("Begin txn failed"); From 943cb232a146e4b52d2bbcc7177ebe95c36c171d Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Wed, 21 Mar 2018 18:37:32 -0400 Subject: [PATCH 04/81] single statement transaction - pushed commit to worker thread --- src/network/postgres_protocol_handler.cpp | 4 +++- src/traffic_cop/traffic_cop.cpp | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/network/postgres_protocol_handler.cpp b/src/network/postgres_protocol_handler.cpp index f3e573c1630..c21acdc96db 100644 --- a/src/network/postgres_protocol_handler.cpp +++ b/src/network/postgres_protocol_handler.cpp @@ -932,7 +932,9 @@ void PostgresProtocolHandler::ExecExecuteMessageGetResult(ResultType status) { } // namespace network void PostgresProtocolHandler::GetResult() { - traffic_cop_->ExecuteStatementPlanGetResult(); + +// traffic_cop_->ExecuteStatementPlanGetResult(); + auto status = traffic_cop_->ExecuteStatementGetResult(); switch (protocol_type_) { case NetworkProtocolType::POSTGRES_JDBC: diff --git a/src/traffic_cop/traffic_cop.cpp b/src/traffic_cop/traffic_cop.cpp index 0105107d242..8ba43f5f8d0 100644 --- a/src/traffic_cop/traffic_cop.cpp +++ b/src/traffic_cop/traffic_cop.cpp @@ -180,13 +180,17 @@ executor::ExecutionResult TrafficCop::ExecuteHelper( return p_status_; } - auto on_complete = [&result, this](executor::ExecutionResult p_status, + auto on_complete = [&result, this, &txn](executor::ExecutionResult p_status, std::vector &&values) { this->p_status_ = p_status; // TODO (Tianyi) I would make a decision on keeping one of p_status or // error_message in my next PR this->error_message_ = std::move(p_status.m_error_message); result = std::move(values); + + // COMMIT single statement transaction + this->ExecuteStatementPlanGetResult(); + task_callback_(task_callback_arg_); }; @@ -592,6 +596,7 @@ ResultType TrafficCop::ExecuteStatement( ExecuteHelper(statement->GetPlanTree(), params, result, result_format, thread_id); + if (GetQueuing()) { return ResultType::QUEUING; } else { From cacdf546182a0e3e6b7195b9e2323bdd08f0eefe Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Wed, 21 Mar 2018 21:34:08 -0400 Subject: [PATCH 05/81] Basic starter for getting values from codegen --- src/codegen/operator/update_translator.cpp | 11 +++- src/codegen/proxy/delta_proxy.cpp | 9 ++++ src/codegen/proxy/target_proxy.cpp | 1 + src/codegen/updater.cpp | 60 +++++++++++++++++++++- src/include/codegen/proxy/delta_proxy.h | 19 +++++++ src/include/codegen/proxy/target_proxy.h | 7 +++ src/include/codegen/updater.h | 6 ++- src/include/codegen/value.h | 2 + src/include/common/internal_types.h | 2 +- src/include/logging/log_record.h | 21 ++++++++ src/threadpool/logger_queue_pool.cpp | 2 +- 11 files changed, 133 insertions(+), 7 deletions(-) create mode 100644 src/codegen/proxy/delta_proxy.cpp create mode 100644 src/include/codegen/proxy/delta_proxy.h diff --git a/src/codegen/operator/update_translator.cpp b/src/codegen/operator/update_translator.cpp index 097b7b74cfd..621ccb718ec 100644 --- a/src/codegen/operator/update_translator.cpp +++ b/src/codegen/operator/update_translator.cpp @@ -91,6 +91,7 @@ void UpdateTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { static_cast(target_list.size() + direct_map_list.size()); auto &ais = update_plan_.GetAttributeInfos(); + std::vector diff; // Collect all the column values std::vector values; for (uint32_t i = 0, target_id = 0; i < column_num; i++) { @@ -99,6 +100,8 @@ void UpdateTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { // Set the value for the update const auto &derived_attribute = target_list[target_id].second; val = row.DeriveValue(codegen, *derived_attribute.expr); + LOG_INFO("Pushing %d and %p", i, val.GetValue()); + diff.push_back(std::make_pair(i, val)); target_id++; } else { val = row.DeriveValue(codegen, ais[i]); @@ -106,6 +109,12 @@ void UpdateTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { values.push_back(val); } + llvm::Value *diff_ptr = codegen->CreateIntToPtr( + codegen.Const64((int64_t)diff.data()), + DeltaProxy::GetType(codegen)->getPointerTo()); + llvm::Value *diff_size = + codegen.Const32((int32_t)diff.size()); + // Get the tuple pointer from the updater llvm::Value *updater = LoadStatePtr(updater_state_id_); llvm::Value *tuple_ptr; @@ -129,7 +138,7 @@ void UpdateTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { table_storage_.StoreValues(codegen, tuple_ptr, values, pool_ptr); // Finally, update with help from the Updater - std::vector update_args = {updater}; + std::vector update_args = {updater, diff_ptr, diff_size}; if (update_plan_.GetUpdatePrimaryKey() == false) { codegen.Call(UpdaterProxy::Update, update_args); } else { diff --git a/src/codegen/proxy/delta_proxy.cpp b/src/codegen/proxy/delta_proxy.cpp new file mode 100644 index 00000000000..14cea86b105 --- /dev/null +++ b/src/codegen/proxy/delta_proxy.cpp @@ -0,0 +1,9 @@ +//#include "codegen/proxy/delta_proxy.h" +// +//namespace peloton { +//namespace codegen { +// +//DEFINE_TYPE(Diff, "peloton::codegen::Delta", MEMBER(opaque)); +// +//} // namespace codegen +//} // namespace peloton diff --git a/src/codegen/proxy/target_proxy.cpp b/src/codegen/proxy/target_proxy.cpp index ffcb864e46f..7d25f8849e9 100644 --- a/src/codegen/proxy/target_proxy.cpp +++ b/src/codegen/proxy/target_proxy.cpp @@ -16,6 +16,7 @@ namespace peloton { namespace codegen { DEFINE_TYPE(Target, "peloton::Target", MEMBER(opaque)); +DEFINE_TYPE(Delta, "peloton::codegen::Delta", MEMBER(opaque)); } // namespace codegen } // namespace peloton diff --git a/src/codegen/updater.cpp b/src/codegen/updater.cpp index 014ef4f7479..766b2ff7a55 100644 --- a/src/codegen/updater.cpp +++ b/src/codegen/updater.cpp @@ -23,6 +23,13 @@ #include "type/abstract_pool.h" #include "common/internal_types.h" #include "type/value.h" +#include "threadpool/mono_queue_pool.h" +#include "logging/log_record.h" +#include "logging/log_buffer.h" +#include "logging/wal_logger.h" +#include "threadpool/logger_queue_pool.h" +#include "../include/type/value.h" +#include "../include/codegen/value.h" namespace peloton { namespace codegen { @@ -112,7 +119,7 @@ peloton::type::AbstractPool *Updater::GetPool() { return tile_->GetPool(); } -void Updater::Update() { +void Updater::Update(peloton::codegen::Delta *diff_array, uint32_t diff_size) { PL_ASSERT(table_ != nullptr && executor_context_ != nullptr); LOG_TRACE("Updating tuple <%u, %u> from table '%s' (db ID: %u, table ID: %u)", old_location_.block, old_location_.offset, @@ -143,10 +150,39 @@ void Updater::Update() { return; } txn_manager.PerformUpdate(txn, old_location_, new_location_); + + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_UPDATE, new_location_, txn->GetEpochId(), + txn->GetTransactionId(), txn->GetCommitId()); + record.SetOldItemPointer(old_location_); + record.SetDiffVector(diff_array, diff_size); + + LOG_INFO("The diff array is %p and the size is %u", diff_array, diff_size); +// std::vector *delta_changes = new std::vector(diff_array, +// diff_array + diff_size); + peloton::codegen::Delta *p = reinterpret_cast(diff_array); + for (unsigned int i = 0; i < diff_size; i++) { + LOG_INFO("column offset %u, %p", p->first, p->second.GetValue()); + p++; + } + txn->GetLogBuffer()->WriteRecord(record); + + if(txn->GetLogBuffer()->HasThresholdExceeded()) { + + LOG_DEBUG("Submitting log buffer %p", txn->GetLogBuffer()); + + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer(txn->GetLogBuffer()); + + /* allocate a new buffer for the current transaction */ + txn->ResetLogBuffer(); + } + executor_context_->num_processed++; } -void Updater::UpdatePK() { +void Updater::UpdatePK(peloton::codegen::Delta *diff_array, uint32_t diff_size) { PL_ASSERT(table_ != nullptr && executor_context_ != nullptr); LOG_TRACE("Updating tuple <%u, %u> from table '%s' (db ID: %u, table ID: %u)", old_location_.block, old_location_.offset, @@ -166,6 +202,26 @@ void Updater::UpdatePK() { return; } txn_manager.PerformInsert(txn, new_location_, index_entry_ptr); + + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_UPDATE, new_location_, txn->GetEpochId(), + txn->GetTransactionId(), txn->GetCommitId()); + record.SetOldItemPointer(old_location_); + record.SetDiffVector(diff_array, diff_size); + + txn->GetLogBuffer()->WriteRecord(record); + + if(txn->GetLogBuffer()->HasThresholdExceeded()) { + + LOG_DEBUG("Submitting log buffer %p", txn->GetLogBuffer()); + + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer(txn->GetLogBuffer()); + + /* allocate a new buffer for the current transaction */ + txn->ResetLogBuffer(); + } executor_context_->num_processed++; } diff --git a/src/include/codegen/proxy/delta_proxy.h b/src/include/codegen/proxy/delta_proxy.h new file mode 100644 index 00000000000..df16c634594 --- /dev/null +++ b/src/include/codegen/proxy/delta_proxy.h @@ -0,0 +1,19 @@ +//#pragma once +// +//#include "codegen/proxy/proxy.h" +//#include "codegen/proxy/type_builder.h" +//#include "common/internal_types.h" +//#include "codegen/value.h" +// +//namespace peloton { +//namespace codegen { +// +//PROXY(Diff) { +// DECLARE_MEMBER(0, char[sizeof(peloton::codegen::Delta)], opaque); +// DECLARE_TYPE; +//}; +// +//TYPE_BUILDER(Diff, peloton::codegen::Delta); +// +//} // namespace codegen +//} // namespace pelotonv \ No newline at end of file diff --git a/src/include/codegen/proxy/target_proxy.h b/src/include/codegen/proxy/target_proxy.h index 2b0e8924d7e..4212f9cff31 100644 --- a/src/include/codegen/proxy/target_proxy.h +++ b/src/include/codegen/proxy/target_proxy.h @@ -16,6 +16,7 @@ #include "codegen/proxy/type_builder.h" #include "common/internal_types.h" #include "planner/project_info.h" +#include "codegen/value.h" namespace peloton { namespace codegen { @@ -27,5 +28,11 @@ PROXY(Target) { TYPE_BUILDER(Target, peloton::Target); +PROXY(Delta) { + DECLARE_MEMBER(0, char[sizeof(peloton::codegen::Delta)], opaque); + DECLARE_TYPE; +}; + +TYPE_BUILDER(Delta, peloton::codegen::Delta); } // namespace codegen } // namespace peloton diff --git a/src/include/codegen/updater.h b/src/include/codegen/updater.h index f3da82cbe74..c68cfaf2044 100644 --- a/src/include/codegen/updater.h +++ b/src/include/codegen/updater.h @@ -15,6 +15,8 @@ #include "common/item_pointer.h" #include "executor/executor_context.h" #include "common/internal_types.h" +#include "../type/value.h" +#include "codegen/value.h" namespace peloton { @@ -52,10 +54,10 @@ class Updater { peloton::type::AbstractPool *GetPool(); // Update a tuple - void Update(); + void Update(peloton::codegen::Delta *diff_array, uint32_t diff_size); // Update a tuple with primary key - void UpdatePK(); + void UpdatePK(peloton::codegen::Delta *diff_array, uint32_t diff_size); // Finalize the instance void TearDown(); diff --git a/src/include/codegen/value.h b/src/include/codegen/value.h index 269016a3828..8c966615726 100644 --- a/src/include/codegen/value.h +++ b/src/include/codegen/value.h @@ -144,5 +144,7 @@ class Value { llvm::Value *null_; }; +typedef std::pair Delta; + } // namespace codegen } // namespace peloton diff --git a/src/include/common/internal_types.h b/src/include/common/internal_types.h index 331b3acc00f..144fab97921 100644 --- a/src/include/common/internal_types.h +++ b/src/include/common/internal_types.h @@ -29,6 +29,7 @@ #include "common/logger.h" #include "common/macros.h" + namespace peloton { // For all of the enums defined in this header, we will @@ -1275,7 +1276,6 @@ struct DerivedAttribute; } typedef std::pair Target; - typedef std::vector TargetList; /** diff --git a/src/include/logging/log_record.h b/src/include/logging/log_record.h index 30af29cb84b..07f85972034 100644 --- a/src/include/logging/log_record.h +++ b/src/include/logging/log_record.h @@ -15,6 +15,8 @@ #include "common/internal_types.h" #include "common/item_pointer.h" #include "common/macros.h" +#include "type/value.h" +#include "codegen/value.h" namespace peloton { namespace logging { @@ -41,18 +43,31 @@ class LogRecord { inline void SetItemPointer(const ItemPointer &pos) { tuple_pos_ = pos; } + inline void SetOldItemPointer(const ItemPointer &pos) { old_tuple_pos_ = pos; } + inline void SetEpochId(const eid_t epoch_id) { eid_ = epoch_id; } inline void SetCommitId(const cid_t commit_id) { cid_ = commit_id; } inline void SetTransactionId(const txn_id_t txn_id) { txn_id_ = txn_id; } + inline void SetDiffVector(peloton::codegen::Delta *diff_array, uint32_t diff_size) { + diff_array_ = diff_array; + diff_size_ = diff_size; + } + inline const ItemPointer &GetItemPointer() { return tuple_pos_; } + inline const ItemPointer &GetOldItemPointer() { return old_tuple_pos_; } + inline eid_t GetEpochId() { return eid_; } inline cid_t GetCommitId() { return cid_; } + inline peloton::codegen::Delta *GetDiffArray() { return diff_array_; } + + inline uint32_t GetDiffSize() { return diff_size_; } + inline txn_id_t GetTransactionId() { return txn_id_; } private: @@ -60,11 +75,17 @@ class LogRecord { ItemPointer tuple_pos_; + ItemPointer old_tuple_pos_; + eid_t eid_; txn_id_t txn_id_; cid_t cid_; + + peloton::codegen::Delta *diff_array_; + + uint32_t diff_size_; }; diff --git a/src/threadpool/logger_queue_pool.cpp b/src/threadpool/logger_queue_pool.cpp index 46b496e93a4..66ed6e0b30b 100644 --- a/src/threadpool/logger_queue_pool.cpp +++ b/src/threadpool/logger_queue_pool.cpp @@ -17,7 +17,7 @@ void LoggerFunc(std::atomic_bool *is_running, LoggerQueue *logger_queue) { while (is_running->load() || !logger_queue->IsEmpty()) { - logging::LogBuffer *log_buffer; + logging::LogBuffer *log_buffer = nullptr; if (!logger_queue->Dequeue(log_buffer)) { // Polling with exponential backoff From fb02b983be258cabbf6d6a49bd2288cf0c7661d4 Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Thu, 22 Mar 2018 21:59:30 -0400 Subject: [PATCH 06/81] Updates getting info from codegen --- src/codegen/operator/update_translator.cpp | 52 +++++++++++++++++++--- src/codegen/proxy/delta_proxy.cpp | 9 ---- src/codegen/proxy/target_proxy.cpp | 1 - src/codegen/updater.cpp | 25 +++++++---- src/include/codegen/proxy/delta_proxy.h | 19 -------- src/include/codegen/proxy/target_proxy.h | 7 --- src/include/codegen/updater.h | 6 +-- src/include/codegen/value.h | 2 - src/include/logging/log_record.h | 6 +-- 9 files changed, 66 insertions(+), 61 deletions(-) delete mode 100644 src/codegen/proxy/delta_proxy.cpp delete mode 100644 src/include/codegen/proxy/delta_proxy.h diff --git a/src/codegen/operator/update_translator.cpp b/src/codegen/operator/update_translator.cpp index 621ccb718ec..47f0d289813 100644 --- a/src/codegen/operator/update_translator.cpp +++ b/src/codegen/operator/update_translator.cpp @@ -14,8 +14,11 @@ #include "codegen/proxy/storage_manager_proxy.h" #include "codegen/proxy/target_proxy.h" #include "codegen/proxy/updater_proxy.h" +#include "codegen/proxy/value_proxy.h" +#include "codegen/proxy/values_runtime_proxy.h" #include "codegen/operator/update_translator.h" #include "codegen/table_storage.h" +#include "codegen/type/sql_type.h" #include "planner/update_plan.h" #include "storage/data_table.h" @@ -91,7 +94,11 @@ void UpdateTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { static_cast(target_list.size() + direct_map_list.size()); auto &ais = update_plan_.GetAttributeInfos(); - std::vector diff; + auto *diff = codegen.AllocateBuffer( + ValueProxy::GetType(codegen), static_cast(target_list.size()), + "diff"); + diff = + codegen->CreatePointerCast(diff, codegen.CharPtrType()); // Collect all the column values std::vector values; for (uint32_t i = 0, target_id = 0; i < column_num; i++) { @@ -101,7 +108,41 @@ void UpdateTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { const auto &derived_attribute = target_list[target_id].second; val = row.DeriveValue(codegen, *derived_attribute.expr); LOG_INFO("Pushing %d and %p", i, val.GetValue()); - diff.push_back(std::make_pair(i, val)); + const auto &sql_type = val.GetType().GetSqlType(); + + // Check if it's NULL + Value null_val; + lang::If val_is_null{codegen, val.IsNull(codegen)}; + { + // If the value is NULL (i.e., has the NULL bit set), produce the NULL + // value for the given type. + null_val = sql_type.GetNullValue(codegen); + } + val_is_null.EndIf(); + val = val_is_null.BuildPHI(null_val, val); + + // Output the value using the type's output function + auto *output_func = sql_type.GetOutputFunction(codegen, val.GetType()); + + // Setup the function arguments + std::vector args = {diff, codegen.Const32(i), + val.GetValue()}; + // If the value is a string, push back the length + if (val.GetLength() != nullptr) { + args.push_back(val.GetLength()); + } + + // If the value is a boolean, push back the NULL bit. We don't do that for + // the other data types because we have special values for NULL. Booleans + // in codegen are 1-bit types, as opposed to 1-byte types in the rest of the + // system. Since, we cannot have a special value for NULL in a 1-bit boolean + // system, we pass along the NULL bit during output. + if (sql_type.TypeId() == peloton::type::TypeId::BOOLEAN) { + args.push_back(val.IsNull(codegen)); + } + + // Call the function + codegen.CallFunc(output_func, args); target_id++; } else { val = row.DeriveValue(codegen, ais[i]); @@ -109,11 +150,8 @@ void UpdateTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { values.push_back(val); } - llvm::Value *diff_ptr = codegen->CreateIntToPtr( - codegen.Const64((int64_t)diff.data()), - DeltaProxy::GetType(codegen)->getPointerTo()); llvm::Value *diff_size = - codegen.Const32((int32_t)diff.size()); + codegen.Const32((int32_t)target_list.size()); // Get the tuple pointer from the updater llvm::Value *updater = LoadStatePtr(updater_state_id_); @@ -138,7 +176,7 @@ void UpdateTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { table_storage_.StoreValues(codegen, tuple_ptr, values, pool_ptr); // Finally, update with help from the Updater - std::vector update_args = {updater, diff_ptr, diff_size}; + std::vector update_args = {updater, diff, diff_size}; if (update_plan_.GetUpdatePrimaryKey() == false) { codegen.Call(UpdaterProxy::Update, update_args); } else { diff --git a/src/codegen/proxy/delta_proxy.cpp b/src/codegen/proxy/delta_proxy.cpp deleted file mode 100644 index 14cea86b105..00000000000 --- a/src/codegen/proxy/delta_proxy.cpp +++ /dev/null @@ -1,9 +0,0 @@ -//#include "codegen/proxy/delta_proxy.h" -// -//namespace peloton { -//namespace codegen { -// -//DEFINE_TYPE(Diff, "peloton::codegen::Delta", MEMBER(opaque)); -// -//} // namespace codegen -//} // namespace peloton diff --git a/src/codegen/proxy/target_proxy.cpp b/src/codegen/proxy/target_proxy.cpp index 7d25f8849e9..ffcb864e46f 100644 --- a/src/codegen/proxy/target_proxy.cpp +++ b/src/codegen/proxy/target_proxy.cpp @@ -16,7 +16,6 @@ namespace peloton { namespace codegen { DEFINE_TYPE(Target, "peloton::Target", MEMBER(opaque)); -DEFINE_TYPE(Delta, "peloton::codegen::Delta", MEMBER(opaque)); } // namespace codegen } // namespace peloton diff --git a/src/codegen/updater.cpp b/src/codegen/updater.cpp index 766b2ff7a55..f1650d0115c 100644 --- a/src/codegen/updater.cpp +++ b/src/codegen/updater.cpp @@ -9,7 +9,7 @@ // Copyright (c) 2015-17, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// - +#include "codegen/buffering_consumer.h" #include "codegen/updater.h" #include "codegen/transaction_runtime.h" #include "common/container_tuple.h" @@ -28,8 +28,11 @@ #include "logging/log_buffer.h" #include "logging/wal_logger.h" #include "threadpool/logger_queue_pool.h" +#include "type/serializeio.h" +#include "type/value_peeker.h" #include "../include/type/value.h" #include "../include/codegen/value.h" +#include "../include/codegen/updater.h" namespace peloton { namespace codegen { @@ -119,7 +122,7 @@ peloton::type::AbstractPool *Updater::GetPool() { return tile_->GetPool(); } -void Updater::Update(peloton::codegen::Delta *diff_array, uint32_t diff_size) { +void Updater::Update(char *diff_array, uint32_t diff_size) { PL_ASSERT(table_ != nullptr && executor_context_ != nullptr); LOG_TRACE("Updating tuple <%u, %u> from table '%s' (db ID: %u, table ID: %u)", old_location_.block, old_location_.offset, @@ -159,13 +162,17 @@ void Updater::Update(peloton::codegen::Delta *diff_array, uint32_t diff_size) { record.SetDiffVector(diff_array, diff_size); LOG_INFO("The diff array is %p and the size is %u", diff_array, diff_size); -// std::vector *delta_changes = new std::vector(diff_array, -// diff_array + diff_size); - peloton::codegen::Delta *p = reinterpret_cast(diff_array); - for (unsigned int i = 0; i < diff_size; i++) { - LOG_INFO("column offset %u, %p", p->first, p->second.GetValue()); - p++; + std::vector diff_vector; + diff_vector.emplace_back( + reinterpret_cast(diff_array), diff_size); + uint32_t offset = 0; + for (const auto &it : diff_vector) { + for (const auto &itr : it.tuple_) { + LOG_INFO("offset = %u, value = %d", ((*target_list_)[offset]).first, peloton::type::ValuePeeker::PeekInteger(itr)); + offset++; + } } + txn->GetLogBuffer()->WriteRecord(record); if(txn->GetLogBuffer()->HasThresholdExceeded()) { @@ -182,7 +189,7 @@ void Updater::Update(peloton::codegen::Delta *diff_array, uint32_t diff_size) { executor_context_->num_processed++; } -void Updater::UpdatePK(peloton::codegen::Delta *diff_array, uint32_t diff_size) { +void Updater::UpdatePK(char *diff_array, uint32_t diff_size) { PL_ASSERT(table_ != nullptr && executor_context_ != nullptr); LOG_TRACE("Updating tuple <%u, %u> from table '%s' (db ID: %u, table ID: %u)", old_location_.block, old_location_.offset, diff --git a/src/include/codegen/proxy/delta_proxy.h b/src/include/codegen/proxy/delta_proxy.h deleted file mode 100644 index df16c634594..00000000000 --- a/src/include/codegen/proxy/delta_proxy.h +++ /dev/null @@ -1,19 +0,0 @@ -//#pragma once -// -//#include "codegen/proxy/proxy.h" -//#include "codegen/proxy/type_builder.h" -//#include "common/internal_types.h" -//#include "codegen/value.h" -// -//namespace peloton { -//namespace codegen { -// -//PROXY(Diff) { -// DECLARE_MEMBER(0, char[sizeof(peloton::codegen::Delta)], opaque); -// DECLARE_TYPE; -//}; -// -//TYPE_BUILDER(Diff, peloton::codegen::Delta); -// -//} // namespace codegen -//} // namespace pelotonv \ No newline at end of file diff --git a/src/include/codegen/proxy/target_proxy.h b/src/include/codegen/proxy/target_proxy.h index 4212f9cff31..dee8c4d3e72 100644 --- a/src/include/codegen/proxy/target_proxy.h +++ b/src/include/codegen/proxy/target_proxy.h @@ -27,12 +27,5 @@ PROXY(Target) { }; TYPE_BUILDER(Target, peloton::Target); - -PROXY(Delta) { - DECLARE_MEMBER(0, char[sizeof(peloton::codegen::Delta)], opaque); - DECLARE_TYPE; -}; - -TYPE_BUILDER(Delta, peloton::codegen::Delta); } // namespace codegen } // namespace peloton diff --git a/src/include/codegen/updater.h b/src/include/codegen/updater.h index c68cfaf2044..1596263b697 100644 --- a/src/include/codegen/updater.h +++ b/src/include/codegen/updater.h @@ -15,8 +15,6 @@ #include "common/item_pointer.h" #include "executor/executor_context.h" #include "common/internal_types.h" -#include "../type/value.h" -#include "codegen/value.h" namespace peloton { @@ -54,10 +52,10 @@ class Updater { peloton::type::AbstractPool *GetPool(); // Update a tuple - void Update(peloton::codegen::Delta *diff_array, uint32_t diff_size); + void Update(char *diff_array, uint32_t diff_size); // Update a tuple with primary key - void UpdatePK(peloton::codegen::Delta *diff_array, uint32_t diff_size); + void UpdatePK(char *diff_array, uint32_t diff_size); // Finalize the instance void TearDown(); diff --git a/src/include/codegen/value.h b/src/include/codegen/value.h index 8c966615726..269016a3828 100644 --- a/src/include/codegen/value.h +++ b/src/include/codegen/value.h @@ -144,7 +144,5 @@ class Value { llvm::Value *null_; }; -typedef std::pair Delta; - } // namespace codegen } // namespace peloton diff --git a/src/include/logging/log_record.h b/src/include/logging/log_record.h index 07f85972034..5769592efc8 100644 --- a/src/include/logging/log_record.h +++ b/src/include/logging/log_record.h @@ -51,7 +51,7 @@ class LogRecord { inline void SetTransactionId(const txn_id_t txn_id) { txn_id_ = txn_id; } - inline void SetDiffVector(peloton::codegen::Delta *diff_array, uint32_t diff_size) { + inline void SetDiffVector(char *diff_array, uint32_t diff_size) { diff_array_ = diff_array; diff_size_ = diff_size; } @@ -64,7 +64,7 @@ class LogRecord { inline cid_t GetCommitId() { return cid_; } - inline peloton::codegen::Delta *GetDiffArray() { return diff_array_; } + inline char *GetDiffArray() { return diff_array_; } inline uint32_t GetDiffSize() { return diff_size_; } @@ -83,7 +83,7 @@ class LogRecord { cid_t cid_; - peloton::codegen::Delta *diff_array_; + char *diff_array_; uint32_t diff_size_; }; From fa8ed5a33467ef2caece97eb9463c1bcd3854c8b Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Fri, 23 Mar 2018 03:14:28 -0400 Subject: [PATCH 07/81] Working updates with codegen --- src/codegen/operator/update_translator.cpp | 1 - src/codegen/updater.cpp | 19 ++----------- src/include/logging/log_record.h | 24 ++++++++++------ src/logging/log_buffer.cpp | 32 ++++++++++++++++++++-- 4 files changed, 49 insertions(+), 27 deletions(-) diff --git a/src/codegen/operator/update_translator.cpp b/src/codegen/operator/update_translator.cpp index 47f0d289813..9a7577ae883 100644 --- a/src/codegen/operator/update_translator.cpp +++ b/src/codegen/operator/update_translator.cpp @@ -107,7 +107,6 @@ void UpdateTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { // Set the value for the update const auto &derived_attribute = target_list[target_id].second; val = row.DeriveValue(codegen, *derived_attribute.expr); - LOG_INFO("Pushing %d and %p", i, val.GetValue()); const auto &sql_type = val.GetType().GetSqlType(); // Check if it's NULL diff --git a/src/codegen/updater.cpp b/src/codegen/updater.cpp index f1650d0115c..835d8e55c93 100644 --- a/src/codegen/updater.cpp +++ b/src/codegen/updater.cpp @@ -28,8 +28,6 @@ #include "logging/log_buffer.h" #include "logging/wal_logger.h" #include "threadpool/logger_queue_pool.h" -#include "type/serializeio.h" -#include "type/value_peeker.h" #include "../include/type/value.h" #include "../include/codegen/value.h" #include "../include/codegen/updater.h" @@ -159,19 +157,8 @@ void Updater::Update(char *diff_array, uint32_t diff_size) { LogRecordType::TUPLE_UPDATE, new_location_, txn->GetEpochId(), txn->GetTransactionId(), txn->GetCommitId()); record.SetOldItemPointer(old_location_); - record.SetDiffVector(diff_array, diff_size); - - LOG_INFO("The diff array is %p and the size is %u", diff_array, diff_size); - std::vector diff_vector; - diff_vector.emplace_back( - reinterpret_cast(diff_array), diff_size); - uint32_t offset = 0; - for (const auto &it : diff_vector) { - for (const auto &itr : it.tuple_) { - LOG_INFO("offset = %u, value = %d", ((*target_list_)[offset]).first, peloton::type::ValuePeeker::PeekInteger(itr)); - offset++; - } - } + record.SetValuesArray(diff_array, diff_size); + record.SetOffsetsArray(target_list_); txn->GetLogBuffer()->WriteRecord(record); @@ -215,7 +202,7 @@ void Updater::UpdatePK(char *diff_array, uint32_t diff_size) { LogRecordType::TUPLE_UPDATE, new_location_, txn->GetEpochId(), txn->GetTransactionId(), txn->GetCommitId()); record.SetOldItemPointer(old_location_); - record.SetDiffVector(diff_array, diff_size); + record.SetValuesArray(diff_array, diff_size); txn->GetLogBuffer()->WriteRecord(record); diff --git a/src/include/logging/log_record.h b/src/include/logging/log_record.h index 5769592efc8..d8142a29368 100644 --- a/src/include/logging/log_record.h +++ b/src/include/logging/log_record.h @@ -16,7 +16,7 @@ #include "common/item_pointer.h" #include "common/macros.h" #include "type/value.h" -#include "codegen/value.h" +#include "../common/internal_types.h" namespace peloton { namespace logging { @@ -51,9 +51,13 @@ class LogRecord { inline void SetTransactionId(const txn_id_t txn_id) { txn_id_ = txn_id; } - inline void SetDiffVector(char *diff_array, uint32_t diff_size) { - diff_array_ = diff_array; - diff_size_ = diff_size; + inline void SetValuesArray(char *diff_array, uint32_t num_values) { + values_ = diff_array; + num_values_ = num_values; + } + + inline void SetOffsetsArray(TargetList *arr) { + offsets_ = arr; } inline const ItemPointer &GetItemPointer() { return tuple_pos_; } @@ -64,9 +68,11 @@ class LogRecord { inline cid_t GetCommitId() { return cid_; } - inline char *GetDiffArray() { return diff_array_; } + inline char *GetValuesArray() { return values_; } - inline uint32_t GetDiffSize() { return diff_size_; } + inline uint32_t GetNumValues() { return num_values_; } + + inline TargetList *GetOffsets() { return offsets_; } inline txn_id_t GetTransactionId() { return txn_id_; } @@ -83,9 +89,11 @@ class LogRecord { cid_t cid_; - char *diff_array_; + char *values_; + + TargetList *offsets_; - uint32_t diff_size_; + uint32_t num_values_; }; diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index e2e86bd1060..2367be87f16 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -6,6 +6,10 @@ #include "catalog/manager.h" #include "common/container_tuple.h" #include "storage/tile_group.h" +#include "../include/type/value.h" +#include "../include/logging/log_buffer.h" +#include "../include/common/internal_types.h" +#include "type/value_peeker.h" namespace peloton{ @@ -61,8 +65,32 @@ void LogBuffer::WriteRecord(LogRecord &record) { PL_ASSERT(false); } case LogRecordType::TUPLE_UPDATE: { - LOG_ERROR("Update logging not supported"); - PL_ASSERT(false); + auto &manager = catalog::Manager::GetInstance(); + auto tuple_pos = record.GetItemPointer(); + auto old_tuple_pos = record.GetOldItemPointer(); + auto tg = manager.GetTileGroup(tuple_pos.block).get(); + + // Write down the database id and the table id + log_buffer_.WriteLong(tg->GetDatabaseId()); + log_buffer_.WriteLong(tg->GetTableId()); + + log_buffer_.WriteLong(old_tuple_pos.block); + log_buffer_.WriteLong(old_tuple_pos.offset); + + log_buffer_.WriteLong(tuple_pos.block); + log_buffer_.WriteLong(tuple_pos.offset); + + peloton::type::Value *values_array = reinterpret_cast(record.GetValuesArray()); + TargetList *offsets = record.GetOffsets(); + for (uint32_t i = 0; i < record.GetNumValues(); i++) { + //TODO(akanjani): Check if just copying the offset info will perform better + LOG_INFO("Offset %u updated value %d", ((*offsets)[i]).first, + peloton::type::ValuePeeker::PeekInteger(values_array[i])); + log_buffer_.WriteInt(((*offsets)[i]).first); + values_array[i].SerializeTo(log_buffer_); + } + + break; } default: { LOG_ERROR("Unsupported log record type"); From d681bf7f7485ed8bd08cd7a4429273c026a25f11 Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Sun, 25 Mar 2018 19:32:19 -0400 Subject: [PATCH 08/81] multi statement transaction - pushed commit to worker thread --- ...timestamp_ordering_transaction_manager.cpp | 6 ----- src/traffic_cop/traffic_cop.cpp | 23 ++++++++++++++++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 1c421eea9ff..89bcea023a0 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -740,12 +740,6 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( TransactionContext *const current_txn) { LOG_TRACE("Committing peloton txn : %" PRId64, current_txn->GetTransactionId()); - if(current_txn->IsSingleStatementTxn()){ - LOG_INFO("Committing single stmt txn peloton txn : %" PRId64, current_txn->GetTransactionId()); - } else { - LOG_INFO("Committing multi stmt txn peloton txn : %" PRId64, current_txn->GetTransactionId()); - } - ////////////////////////////////////////////////////////// //// handle READ_ONLY ////////////////////////////////////////////////////////// diff --git a/src/traffic_cop/traffic_cop.cpp b/src/traffic_cop/traffic_cop.cpp index 8ba43f5f8d0..8b248c5c4e5 100644 --- a/src/traffic_cop/traffic_cop.cpp +++ b/src/traffic_cop/traffic_cop.cpp @@ -210,6 +210,8 @@ executor::ExecutionResult TrafficCop::ExecuteHelper( void TrafficCop::ExecuteStatementPlanGetResult() { if (p_status_.m_result == ResultType::FAILURE) return; + if(tcop_txn_state_.empty()) return; + auto txn_result = GetCurrentTxnState().first->GetResult(); if (single_statement_txn_ || txn_result == ResultType::FAILURE) { LOG_TRACE("About to commit/abort: single stmt: %d,txn_result: %s", @@ -217,7 +219,6 @@ void TrafficCop::ExecuteStatementPlanGetResult() { switch (txn_result) { case ResultType::SUCCESS: // Commit single statement - LOG_TRACE("Commit Transaction"); p_status_.m_result = CommitQueryHelper(); break; @@ -576,10 +577,26 @@ ResultType TrafficCop::ExecuteStatement( return BeginQueryHelper(thread_id); } case QueryType::QUERY_COMMIT: { - return CommitQueryHelper(); + this->is_queuing_ = true; + auto &pool = threadpool::MonoQueuePool::GetInstance(); + + pool.SubmitTask([this] { + this->CommitQueryHelper(); + task_callback_(task_callback_arg_); + }); + + return ResultType::QUEUING; } case QueryType::QUERY_ROLLBACK: { - return AbortQueryHelper(); + this->is_queuing_ = true; + auto &pool = threadpool::MonoQueuePool::GetInstance(); + + pool.SubmitTask([this] { + this->AbortQueryHelper(); + task_callback_(task_callback_arg_); + }); + + return ResultType::QUEUING; } default: // The statement may be out of date From 0df5f992d7f41cd7c0fe1eb9f31c48608dfafa5d Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Sun, 25 Mar 2018 19:32:58 -0400 Subject: [PATCH 09/81] added order by clause to query_logger_test --- test/brain/query_logger_test.cpp | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/test/brain/query_logger_test.cpp b/test/brain/query_logger_test.cpp index b2a65eff085..c86ac71acff 100644 --- a/test/brain/query_logger_test.cpp +++ b/test/brain/query_logger_test.cpp @@ -21,13 +21,22 @@ namespace test { class QueryLoggerTests : public PelotonTest { protected: + + bool SortByFingerprint(std::string x, std::string y){ + + std::string x_query = x.substr(0, x.find_first_of("|")); + std::string y_query = y.substr(0, y.find_first_of("|")); + + return x_query < y_query; + } + void SetUp() override { settings::SettingsManager::SetBool(settings::SettingId::brain, true); PelotonInit::Initialize(); // query to check that logging is done select_query_ = - "SELECT query_string, fingerprint FROM pg_catalog.pg_query_history;"; + "SELECT query_string, fingerprint FROM pg_catalog.pg_query_history order by query_string;"; brain::QueryLogger::Fingerprint fingerprint{select_query_}; select_query_fingerprint_ = fingerprint.GetFingerprint(); @@ -51,6 +60,10 @@ class QueryLoggerTests : public PelotonTest { // give some time to actually log this query sleep(wait_time_); + //order by query_string + std::sort(expected_result.begin(), expected_result.end(), + [this] (std::string x, std::string y) { return SortByFingerprint(x, y); }); + TestingSQLUtil::ExecuteSQLQueryAndCheckResult(select_query_.c_str(), expected_result, true); @@ -86,6 +99,11 @@ class QueryLoggerTests : public PelotonTest { temporary_expected_result.begin(), temporary_expected_result.end()); temporary_expected_result.clear(); + + //order by query_string + std::sort(expected_result.begin(), expected_result.end(), + [this] (std::string x, std::string y) { return SortByFingerprint(x, y); }); + TestingSQLUtil::ExecuteSQLQueryAndCheckResult(select_query_.c_str(), expected_result, true); @@ -124,9 +142,10 @@ TEST_F(QueryLoggerTests, QueriesTest) { TestSimpleUtil("INSERT INTO test VALUES (1);", expected_result); TestSimpleUtil("INSERT INTO test VALUES (2);", expected_result); + // the select_query_ done at the end of above test + // won't be logged till the txn below commits expected_result - .pop_back(); // the select_query_ done at the end of above test - // won't be logged till the txn below commits + .pop_back(); // check if the queries are logged only when the transaction actually commits TestTransactionUtil("BEGIN;", expected_result, false); From 7cf00b6e459252010f77ebb9e03f801ebe92470d Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Tue, 27 Mar 2018 16:21:46 -0400 Subject: [PATCH 10/81] Inserts and updates working with codegen --- src/codegen/buffering_consumer.cpp | 76 ++++++++++--------- src/codegen/inserter.cpp | 25 +++++- src/codegen/operator/insert_translator.cpp | 28 ++++++- src/codegen/operator/update_translator.cpp | 67 ++++++---------- src/codegen/updater.cpp | 4 +- ...timestamp_ordering_transaction_manager.cpp | 20 ----- src/include/codegen/buffering_consumer.h | 3 + src/include/codegen/inserter.h | 2 +- src/logging/log_buffer.cpp | 20 ++--- 9 files changed, 124 insertions(+), 121 deletions(-) diff --git a/src/codegen/buffering_consumer.cpp b/src/codegen/buffering_consumer.cpp index dbb7f90fdcc..f6b7cc7ae7d 100644 --- a/src/codegen/buffering_consumer.cpp +++ b/src/codegen/buffering_consumer.cpp @@ -78,6 +78,46 @@ void BufferingConsumer::Prepare(CompilationContext &ctx) { runtime_state.RegisterState("consumerState", codegen.CharPtrType()); } +void BufferingConsumer::AddToTupleBuffer(Value &val, CodeGen &codegen, llvm::Value *tuple_buffer, size_t &i) { + const auto &sql_type = val.GetType().GetSqlType(); + + // Check if it's NULL + Value null_val; + lang::If val_is_null{codegen, val.IsNull(codegen)}; + { + // If the value is NULL (i.e., has the NULL bit set), produce the NULL + // value for the given type. + null_val = sql_type.GetNullValue(codegen); + } + val_is_null.EndIf(); + val = val_is_null.BuildPHI(null_val, val); + + // Output the value using the type's output function + auto *output_func = sql_type.GetOutputFunction(codegen, val.GetType()); + + // Setup the function arguments + std::vector args = {tuple_buffer, codegen.Const32(i), + val.GetValue()}; + // If the value is a string, push back the length + if (val.GetLength() != nullptr) { + args.push_back(val.GetLength()); + } + + // If the value is a boolean, push back the NULL bit. We don't do that for + // the other data types because we have special values for NULL. Booleans + // in codegen are 1-bit types, as opposed to 1-byte types in the rest of the + // system. Since, we cannot have a special value for NULL in a 1-bit boolean + // system, we pass along the NULL bit during output. + if (sql_type.TypeId() == peloton::type::TypeId::BOOLEAN) { + args.push_back(val.IsNull(codegen)); + } + + // Call the function + codegen.CallFunc(output_func, args); +} + + + // For each output attribute, we write out the attribute's value into the // currently active output tuple. When all attributes have been written, we // call BufferTuple(...) to append the currently active tuple into the output. @@ -95,41 +135,7 @@ void BufferingConsumer::ConsumeResult(ConsumerContext &ctx, Value val = row.DeriveValue(codegen, output_ais_[i]); PL_ASSERT(output_ais_[i]->type == val.GetType()); - const auto &sql_type = val.GetType().GetSqlType(); - - // Check if it's NULL - Value null_val; - lang::If val_is_null{codegen, val.IsNull(codegen)}; - { - // If the value is NULL (i.e., has the NULL bit set), produce the NULL - // value for the given type. - null_val = sql_type.GetNullValue(codegen); - } - val_is_null.EndIf(); - val = val_is_null.BuildPHI(null_val, val); - - // Output the value using the type's output function - auto *output_func = sql_type.GetOutputFunction(codegen, val.GetType()); - - // Setup the function arguments - std::vector args = {tuple_buffer_, codegen.Const32(i), - val.GetValue()}; - // If the value is a string, push back the length - if (val.GetLength() != nullptr) { - args.push_back(val.GetLength()); - } - - // If the value is a boolean, push back the NULL bit. We don't do that for - // the other data types because we have special values for NULL. Booleans - // in codegen are 1-bit types, as opposed to 1-byte types in the rest of the - // system. Since, we cannot have a special value for NULL in a 1-bit boolean - // system, we pass along the NULL bit during output. - if (sql_type.TypeId() == peloton::type::TypeId::BOOLEAN) { - args.push_back(val.IsNull(codegen)); - } - - // Call the function - codegen.CallFunc(output_func, args); + AddToTupleBuffer(val, codegen, tuple_buffer_, i); } // Append the tuple to the output buffer (by calling BufferTuple(...)) diff --git a/src/codegen/inserter.cpp b/src/codegen/inserter.cpp index 64e3ea032bd..68efd5979f0 100644 --- a/src/codegen/inserter.cpp +++ b/src/codegen/inserter.cpp @@ -20,6 +20,10 @@ #include "storage/data_table.h" #include "storage/tile_group.h" #include "storage/tile.h" +#include "threadpool/logger_queue_pool.h" +#include "logging/log_record.h" +#include "logging/log_buffer.h" +#include "type/value.h" namespace peloton { namespace codegen { @@ -48,7 +52,7 @@ peloton::type::AbstractPool *Inserter::GetPool() { return tile_->GetPool(); } -void Inserter::Insert() { +void Inserter::Insert(char *values_buf, uint32_t values_size) { PL_ASSERT(table_ && executor_context_ && tile_); auto *txn = executor_context_->GetTransaction(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); @@ -62,6 +66,25 @@ void Inserter::Insert() { return; } txn_manager.PerformInsert(txn, location_, index_entry_ptr); + + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_INSERT, location_, txn->GetEpochId(), + txn->GetTransactionId(), txn->GetCommitId()); + record.SetValuesArray(values_buf, values_size); + + txn->GetLogBuffer()->WriteRecord(record); + + if(txn->GetLogBuffer()->HasThresholdExceeded()) { + + LOG_DEBUG("Submitting log buffer %p", txn->GetLogBuffer()); + + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer(txn->GetLogBuffer()); + + /* allocate a new buffer for the current transaction */ + txn->ResetLogBuffer(); + } executor_context_->num_processed++; } diff --git a/src/codegen/operator/insert_translator.cpp b/src/codegen/operator/insert_translator.cpp index 3e36d8c8e68..756edbe2c23 100644 --- a/src/codegen/operator/insert_translator.cpp +++ b/src/codegen/operator/insert_translator.cpp @@ -10,11 +10,14 @@ // //===----------------------------------------------------------------------===// +#include "codegen/buffering_consumer.h" #include "codegen/proxy/inserter_proxy.h" #include "codegen/proxy/query_parameters_proxy.h" #include "codegen/proxy/storage_manager_proxy.h" #include "codegen/proxy/transaction_runtime_proxy.h" #include "codegen/proxy/tuple_proxy.h" +#include "codegen/proxy/value_proxy.h" +#include "codegen/proxy/values_runtime_proxy.h" #include "codegen/operator/insert_translator.h" #include "planner/insert_plan.h" #include "storage/data_table.h" @@ -79,15 +82,23 @@ void InsertTranslator::Produce() const { // Transform into the codegen values and store values in the tuple storage std::vector values; - for (uint32_t column_id = 0; column_id < num_columns; column_id++) { + auto *values_buf = codegen.AllocateBuffer( + ValueProxy::GetType(codegen), num_columns, + "values"); + values_buf = + codegen->CreatePointerCast(values_buf, codegen.CharPtrType()); + llvm::Value *values_size = codegen.Const32((int32_t)num_columns); + for (size_t column_id = 0; column_id < num_columns; column_id++) { auto value = parameter_cache.GetValue(column_id + tuple_idx * num_columns); values.push_back(value); + peloton::codegen::BufferingConsumer::AddToTupleBuffer(value, codegen, values_buf, column_id); } table_storage_.StoreValues(codegen, tuple_ptr, values, pool); + std::vector insert_args = {inserter, values_buf, values_size}; // Complete the insertion - codegen.Call(InserterProxy::Insert, {inserter}); + codegen.Call(InserterProxy::Insert, insert_args); } } } @@ -103,14 +114,25 @@ void InsertTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { // Generate/Materialize tuple data from row and attribute information std::vector values; auto &ais = insert_plan_.GetAttributeInfos(); + auto *values_buf = codegen.AllocateBuffer( + ValueProxy::GetType(codegen), static_cast(ais.size()), + "values"); + values_buf = + codegen->CreatePointerCast(values_buf, codegen.CharPtrType()); + llvm::Value *values_size = codegen.Const32((int32_t)ais.size()); + size_t i = 0; for (const auto *ai : ais) { codegen::Value v = row.DeriveValue(codegen, ai); values.push_back(v); + peloton::codegen::BufferingConsumer::AddToTupleBuffer(v, codegen, values_buf, i); + i++; } table_storage_.StoreValues(codegen, tuple_ptr, values, pool); + std::vector insert_args = {inserter, values_buf, values_size}; + // Call Inserter to insert the reserved tuple storage area - codegen.Call(InserterProxy::Insert, {inserter}); + codegen.Call(InserterProxy::Insert, insert_args); } void InsertTranslator::TearDownState() { diff --git a/src/codegen/operator/update_translator.cpp b/src/codegen/operator/update_translator.cpp index 9a7577ae883..299f0ee1e11 100644 --- a/src/codegen/operator/update_translator.cpp +++ b/src/codegen/operator/update_translator.cpp @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +#include "codegen/buffering_consumer.h" #include "codegen/lang/if.h" #include "codegen/proxy/storage_manager_proxy.h" #include "codegen/proxy/target_proxy.h" @@ -94,70 +95,48 @@ void UpdateTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { static_cast(target_list.size() + direct_map_list.size()); auto &ais = update_plan_.GetAttributeInfos(); - auto *diff = codegen.AllocateBuffer( - ValueProxy::GetType(codegen), static_cast(target_list.size()), - "diff"); + auto is_primary_key = update_plan_.GetUpdatePrimaryKey(); + llvm::Value *diff = nullptr, *diff_size = nullptr; + if (is_primary_key) { + diff_size = codegen.Const32((int32_t)column_num); + diff = codegen.AllocateBuffer( + ValueProxy::GetType(codegen), column_num, + "diff"); + } else { + diff_size = codegen.Const32((int32_t)target_list.size()); + diff = codegen.AllocateBuffer( + ValueProxy::GetType(codegen), static_cast(target_list.size()), + "diff"); + } + + diff = codegen->CreatePointerCast(diff, codegen.CharPtrType()); // Collect all the column values std::vector values; - for (uint32_t i = 0, target_id = 0; i < column_num; i++) { + for (size_t i = 0, target_id = 0; i < column_num; i++) { codegen::Value val; if (IsTarget(target_list, i)) { // Set the value for the update const auto &derived_attribute = target_list[target_id].second; val = row.DeriveValue(codegen, *derived_attribute.expr); - const auto &sql_type = val.GetType().GetSqlType(); - - // Check if it's NULL - Value null_val; - lang::If val_is_null{codegen, val.IsNull(codegen)}; - { - // If the value is NULL (i.e., has the NULL bit set), produce the NULL - // value for the given type. - null_val = sql_type.GetNullValue(codegen); - } - val_is_null.EndIf(); - val = val_is_null.BuildPHI(null_val, val); - - // Output the value using the type's output function - auto *output_func = sql_type.GetOutputFunction(codegen, val.GetType()); - - // Setup the function arguments - std::vector args = {diff, codegen.Const32(i), - val.GetValue()}; - // If the value is a string, push back the length - if (val.GetLength() != nullptr) { - args.push_back(val.GetLength()); - } - - // If the value is a boolean, push back the NULL bit. We don't do that for - // the other data types because we have special values for NULL. Booleans - // in codegen are 1-bit types, as opposed to 1-byte types in the rest of the - // system. Since, we cannot have a special value for NULL in a 1-bit boolean - // system, we pass along the NULL bit during output. - if (sql_type.TypeId() == peloton::type::TypeId::BOOLEAN) { - args.push_back(val.IsNull(codegen)); - } - - // Call the function - codegen.CallFunc(output_func, args); + peloton::codegen::BufferingConsumer::AddToTupleBuffer(val, codegen, diff, i); target_id++; } else { val = row.DeriveValue(codegen, ais[i]); + if (is_primary_key) { + peloton::codegen::BufferingConsumer::AddToTupleBuffer(val, codegen, diff, i); + } } values.push_back(val); } - llvm::Value *diff_size = - codegen.Const32((int32_t)target_list.size()); - // Get the tuple pointer from the updater llvm::Value *updater = LoadStatePtr(updater_state_id_); llvm::Value *tuple_ptr; std::vector prep_args = {updater, row.GetTileGroupID(), row.GetTID(codegen)}; - if (update_plan_.GetUpdatePrimaryKey() == false) { + if (is_primary_key == false) { tuple_ptr = codegen.Call(UpdaterProxy::Prepare, prep_args); } else { tuple_ptr = codegen.Call(UpdaterProxy::PreparePK, prep_args); @@ -176,7 +155,7 @@ void UpdateTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { // Finally, update with help from the Updater std::vector update_args = {updater, diff, diff_size}; - if (update_plan_.GetUpdatePrimaryKey() == false) { + if (is_primary_key == false) { codegen.Call(UpdaterProxy::Update, update_args); } else { codegen.Call(UpdaterProxy::UpdatePK, update_args); diff --git a/src/codegen/updater.cpp b/src/codegen/updater.cpp index 835d8e55c93..979f831aaf2 100644 --- a/src/codegen/updater.cpp +++ b/src/codegen/updater.cpp @@ -30,7 +30,6 @@ #include "threadpool/logger_queue_pool.h" #include "../include/type/value.h" #include "../include/codegen/value.h" -#include "../include/codegen/updater.h" namespace peloton { namespace codegen { @@ -199,9 +198,8 @@ void Updater::UpdatePK(char *diff_array, uint32_t diff_size) { logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_UPDATE, new_location_, txn->GetEpochId(), + LogRecordType::TUPLE_INSERT, new_location_, txn->GetEpochId(), txn->GetTransactionId(), txn->GetCommitId()); - record.SetOldItemPointer(old_location_); record.SetValuesArray(diff_array, diff_size); txn->GetLogBuffer()->WriteRecord(record); diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 4c954557286..11118e47afd 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -495,26 +495,6 @@ void TimestampOrderingTransactionManager::PerformInsert( // Write down the head pointer's address in tile group header tile_group_header->SetIndirection(tuple_id, index_entry_ptr); - logging::LogRecord record = - logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_INSERT, location, current_txn->GetEpochId(), - current_txn->GetTransactionId(), current_txn->GetCommitId()); - - current_txn->GetLogBuffer()->WriteRecord(record); - - if(current_txn->GetLogBuffer()->HasThresholdExceeded()) { - - LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); - - /* insert to the queue */ - threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer(current_txn->GetLogBuffer()); - - /* allocate a new buffer for the current transaction */ - current_txn->ResetLogBuffer(); - } - - - // Increment table insert op stats if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { diff --git a/src/include/codegen/buffering_consumer.h b/src/include/codegen/buffering_consumer.h index 257a9a1dd09..b559e348276 100644 --- a/src/include/codegen/buffering_consumer.h +++ b/src/include/codegen/buffering_consumer.h @@ -14,6 +14,7 @@ #include +#include "codegen/codegen.h" #include "codegen/compilation_context.h" #include "codegen/query_result_consumer.h" #include "codegen/value.h" @@ -72,6 +73,8 @@ class BufferingConsumer : public QueryResultConsumer { // Called from compiled query code to buffer the tuple static void BufferTuple(char *state, char *tuple, uint32_t num_cols); + static void AddToTupleBuffer(Value &val, CodeGen &codegen, llvm::Value *tuple_buffer, size_t &i); + //===--------------------------------------------------------------------===// // ACCESSORS //===--------------------------------------------------------------------===// diff --git a/src/include/codegen/inserter.h b/src/include/codegen/inserter.h index 94167730001..d386e71995c 100644 --- a/src/include/codegen/inserter.h +++ b/src/include/codegen/inserter.h @@ -54,7 +54,7 @@ class Inserter { peloton::type::AbstractPool *GetPool(); // Insert a tuple - void Insert(); + void Insert(char *values_buf, uint32_t values_size); // Finalize the instance void TearDown(); diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index 2367be87f16..ba0bce34d3e 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -44,20 +44,13 @@ void LogBuffer::WriteRecord(LogRecord &record) { log_buffer_.WriteLong(tuple_pos.block); log_buffer_.WriteLong(tuple_pos.offset); - // Write the full tuple into the buffer - for (auto schema : tg->GetTileSchemas()) { - for (auto column : schema.GetColumns()) { - columns.push_back(column); - } + peloton::type::Value *values_array = reinterpret_cast(record.GetValuesArray()); + for (uint32_t i = 0; i < record.GetNumValues(); i++) { + //TODO(akanjani): Check if just copying the offset info will perform better + values_array[i].SerializeTo(log_buffer_); + LOG_INFO("The value at offset %u is %d", i, type::ValuePeeker::PeekInteger(values_array[i])); } - ContainerTuple container_tuple(tg, tuple_pos.offset); - for (oid_t oid = 0; oid < columns.size(); oid++) { - - // TODO: check if GetValue() returns variable length fields appropriately - auto val = container_tuple.GetValue(oid); - val.SerializeTo(log_buffer_); - } break; } case LogRecordType::TUPLE_DELETE: { @@ -84,10 +77,9 @@ void LogBuffer::WriteRecord(LogRecord &record) { TargetList *offsets = record.GetOffsets(); for (uint32_t i = 0; i < record.GetNumValues(); i++) { //TODO(akanjani): Check if just copying the offset info will perform better - LOG_INFO("Offset %u updated value %d", ((*offsets)[i]).first, - peloton::type::ValuePeeker::PeekInteger(values_array[i])); log_buffer_.WriteInt(((*offsets)[i]).first); values_array[i].SerializeTo(log_buffer_); + LOG_INFO("The value at offset %u is %d", ((*offsets)[i]).first, type::ValuePeeker::PeekInteger(values_array[i])); } break; From 6ed8c3b38b2f2765a09480f519e7ec0b710d9ac5 Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Thu, 29 Mar 2018 21:36:26 -0400 Subject: [PATCH 11/81] updates bug fix --- src/codegen/operator/update_translator.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/codegen/operator/update_translator.cpp b/src/codegen/operator/update_translator.cpp index 299f0ee1e11..04c3924c891 100644 --- a/src/codegen/operator/update_translator.cpp +++ b/src/codegen/operator/update_translator.cpp @@ -120,7 +120,13 @@ void UpdateTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { // Set the value for the update const auto &derived_attribute = target_list[target_id].second; val = row.DeriveValue(codegen, *derived_attribute.expr); - peloton::codegen::BufferingConsumer::AddToTupleBuffer(val, codegen, diff, i); + size_t offset = 0; + if (is_primary_key) { + offset = i; + } else { + offset = target_id; + } + peloton::codegen::BufferingConsumer::AddToTupleBuffer(val, codegen, diff, offset); target_id++; } else { val = row.DeriveValue(codegen, ais[i]); From 2cd4caa35227e362c4153c15c4a08f777a22798b Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Fri, 30 Mar 2018 17:16:16 -0400 Subject: [PATCH 12/81] deletes and updates working with ItemPointer --- src/codegen/inserter.cpp | 27 +--- src/codegen/updater.cpp | 45 +------ ...timestamp_ordering_transaction_manager.cpp | 124 +++++++++++++++--- .../timestamp_ordering_transaction_manager.h | 14 +- src/include/concurrency/transaction_manager.h | 14 +- src/logging/log_buffer.cpp | 17 ++- 6 files changed, 144 insertions(+), 97 deletions(-) diff --git a/src/codegen/inserter.cpp b/src/codegen/inserter.cpp index 68efd5979f0..ed72a91a50d 100644 --- a/src/codegen/inserter.cpp +++ b/src/codegen/inserter.cpp @@ -14,16 +14,10 @@ #include "codegen/transaction_runtime.h" #include "common/container_tuple.h" #include "concurrency/transaction_manager_factory.h" -#include "executor/executor_context.h" #include "executor/logical_tile.h" #include "executor/logical_tile_factory.h" -#include "storage/data_table.h" -#include "storage/tile_group.h" #include "storage/tile.h" #include "threadpool/logger_queue_pool.h" -#include "logging/log_record.h" -#include "logging/log_buffer.h" -#include "type/value.h" namespace peloton { namespace codegen { @@ -65,26 +59,7 @@ void Inserter::Insert(char *values_buf, uint32_t values_size) { txn_manager.SetTransactionResult(txn, ResultType::FAILURE); return; } - txn_manager.PerformInsert(txn, location_, index_entry_ptr); - - logging::LogRecord record = - logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_INSERT, location_, txn->GetEpochId(), - txn->GetTransactionId(), txn->GetCommitId()); - record.SetValuesArray(values_buf, values_size); - - txn->GetLogBuffer()->WriteRecord(record); - - if(txn->GetLogBuffer()->HasThresholdExceeded()) { - - LOG_DEBUG("Submitting log buffer %p", txn->GetLogBuffer()); - - /* insert to the queue */ - threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer(txn->GetLogBuffer()); - - /* allocate a new buffer for the current transaction */ - txn->ResetLogBuffer(); - } + txn_manager.PerformInsert(txn, location_, index_entry_ptr, values_buf, values_size); executor_context_->num_processed++; } diff --git a/src/codegen/updater.cpp b/src/codegen/updater.cpp index 979f831aaf2..5e9150a7124 100644 --- a/src/codegen/updater.cpp +++ b/src/codegen/updater.cpp @@ -132,7 +132,7 @@ void Updater::Update(char *diff_array, uint32_t diff_size) { // Either update in-place if (is_owner_ == true) { - txn_manager.PerformUpdate(txn, old_location_); + txn_manager.PerformUpdate(txn, old_location_, diff_array, diff_size, target_list_); executor_context_->num_processed++; return; } @@ -149,28 +149,7 @@ void Updater::Update(char *diff_array, uint32_t diff_size) { old_location_.offset); return; } - txn_manager.PerformUpdate(txn, old_location_, new_location_); - - logging::LogRecord record = - logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_UPDATE, new_location_, txn->GetEpochId(), - txn->GetTransactionId(), txn->GetCommitId()); - record.SetOldItemPointer(old_location_); - record.SetValuesArray(diff_array, diff_size); - record.SetOffsetsArray(target_list_); - - txn->GetLogBuffer()->WriteRecord(record); - - if(txn->GetLogBuffer()->HasThresholdExceeded()) { - - LOG_DEBUG("Submitting log buffer %p", txn->GetLogBuffer()); - - /* insert to the queue */ - threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer(txn->GetLogBuffer()); - - /* allocate a new buffer for the current transaction */ - txn->ResetLogBuffer(); - } + txn_manager.PerformUpdate(txn, old_location_, new_location_, diff_array, diff_size, target_list_); executor_context_->num_processed++; } @@ -194,26 +173,8 @@ void Updater::UpdatePK(char *diff_array, uint32_t diff_size) { txn_manager.SetTransactionResult(txn, ResultType::FAILURE); return; } - txn_manager.PerformInsert(txn, new_location_, index_entry_ptr); + txn_manager.PerformInsert(txn, new_location_, index_entry_ptr, diff_array, diff_size); - logging::LogRecord record = - logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_INSERT, new_location_, txn->GetEpochId(), - txn->GetTransactionId(), txn->GetCommitId()); - record.SetValuesArray(diff_array, diff_size); - - txn->GetLogBuffer()->WriteRecord(record); - - if(txn->GetLogBuffer()->HasThresholdExceeded()) { - - LOG_DEBUG("Submitting log buffer %p", txn->GetLogBuffer()); - - /* insert to the queue */ - threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer(txn->GetLogBuffer()); - - /* allocate a new buffer for the current transaction */ - txn->ResetLogBuffer(); - } executor_context_->num_processed++; } diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 11118e47afd..e638a3af437 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -467,7 +467,7 @@ bool TimestampOrderingTransactionManager::PerformRead( void TimestampOrderingTransactionManager::PerformInsert( TransactionContext *const current_txn, const ItemPointer &location, - ItemPointer *index_entry_ptr) { + ItemPointer *index_entry_ptr, char *values_buf, uint32_t values_size) { PL_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); oid_t tile_group_id = location.block; @@ -495,6 +495,27 @@ void TimestampOrderingTransactionManager::PerformInsert( // Write down the head pointer's address in tile group header tile_group_header->SetIndirection(tuple_id, index_entry_ptr); + LOG_INFO("perfom insert"); + if (values_buf != nullptr) { + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_INSERT, location, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); + record.SetValuesArray(values_buf, values_size); + + current_txn->GetLogBuffer()->WriteRecord(record); + + if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { + + LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); + + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer(current_txn->GetLogBuffer()); + + /* allocate a new buffer for the current transaction */ + current_txn->ResetLogBuffer(); + } + } // Increment table insert op stats if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { @@ -505,7 +526,8 @@ void TimestampOrderingTransactionManager::PerformInsert( void TimestampOrderingTransactionManager::PerformUpdate( TransactionContext *const current_txn, const ItemPointer &location, - const ItemPointer &new_location) { + const ItemPointer &new_location, char *values_buf, + uint32_t values_size, TargetList *offsets) { PL_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); ItemPointer old_location = location; @@ -578,6 +600,30 @@ void TimestampOrderingTransactionManager::PerformUpdate( // Add the old tuple into the update set current_txn->RecordUpdate(old_location); + LOG_INFO("Perform Update"); + + if (values_buf != nullptr) { + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_UPDATE, new_location, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); + record.SetOldItemPointer(location); + record.SetValuesArray(values_buf, values_size); + record.SetOffsetsArray(offsets); + + current_txn->GetLogBuffer()->WriteRecord(record); + + if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { + + LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); + + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer(current_txn->GetLogBuffer()); + + /* allocate a new buffer for the current transaction */ + current_txn->ResetLogBuffer(); + } + } // Increment table update op stats if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { @@ -589,7 +635,8 @@ void TimestampOrderingTransactionManager::PerformUpdate( // NOTE: this function is deprecated. void TimestampOrderingTransactionManager::PerformUpdate( TransactionContext *const current_txn UNUSED_ATTRIBUTE, - const ItemPointer &location) { + const ItemPointer &location, char *values_buf, + uint32_t values_size, TargetList *offsets) { PL_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); oid_t tile_group_id = location.block; @@ -611,7 +658,29 @@ void TimestampOrderingTransactionManager::PerformUpdate( // transaction // is updating a version that is installed by itself. // in this case, nothing needs to be performed. + if (values_buf != nullptr) { + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_UPDATE, location, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); + record.SetOldItemPointer(location); + record.SetValuesArray(values_buf, values_size); + record.SetOffsetsArray(offsets); + + current_txn->GetLogBuffer()->WriteRecord(record); + if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { + + LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); + + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer(current_txn->GetLogBuffer()); + + /* allocate a new buffer for the current transaction */ + current_txn->ResetLogBuffer(); + } + } + LOG_INFO("Perform Update"); // Increment table update op stats if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { @@ -698,16 +767,25 @@ void TimestampOrderingTransactionManager::PerformDelete( current_txn->RecordDelete(old_location); -// logging::LogRecord record = -// logging::LogRecordFactory::CreateTupleRecord( -// LogRecordType::TUPLE_DELETE, -// old_location, current_txn->GetEpochId(), -// current_txn->GetTransactionId(), -// current_txn->GetCommitId()); -// -// logging::WalLogger::WriteRecordToBuffer(record, current_txn->GetLogBuffer()); + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_DELETE, old_location, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); + + current_txn->GetLogBuffer()->WriteRecord(record); + + if(current_txn->GetLogBuffer()->HasThresholdExceeded()) { + LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer(current_txn->GetLogBuffer()); + + /* allocate a new buffer for the current transaction */ + current_txn->ResetLogBuffer(); + } + + LOG_INFO("Perform Delete"); // Increment table delete op stats if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { @@ -744,20 +822,24 @@ void TimestampOrderingTransactionManager::PerformDelete( current_txn->RecordDelete(location); } -// logging::LogRecord record = -// logging::LogRecordFactory::CreateTupleRecord( -// LogRecordType::TUPLE_DELETE, -// location, current_txn->GetEpochId(), -// current_txn->GetTransactionId(), -// current_txn->GetCommitId()); -// -// -// -// logging::WalLogger::WriteRecordToBuffer(record, current_txn->GetLogBuffer()); + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_DELETE, location, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); + current_txn->GetLogBuffer()->WriteRecord(record); + if(current_txn->GetLogBuffer()->HasThresholdExceeded()) { + LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer(current_txn->GetLogBuffer()); + + /* allocate a new buffer for the current transaction */ + current_txn->ResetLogBuffer(); + } + LOG_INFO("Perform Delete"); // Increment table delete op stats if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { diff --git a/src/include/concurrency/timestamp_ordering_transaction_manager.h b/src/include/concurrency/timestamp_ordering_transaction_manager.h index 0c9bd6cdd8c..70b5951ec5e 100644 --- a/src/include/concurrency/timestamp_ordering_transaction_manager.h +++ b/src/include/concurrency/timestamp_ordering_transaction_manager.h @@ -80,7 +80,9 @@ class TimestampOrderingTransactionManager : public TransactionManager { // which is directly pointed by the primary index. virtual void PerformInsert(TransactionContext *const current_txn, const ItemPointer &location, - ItemPointer *index_entry_ptr = nullptr); + ItemPointer *index_entry_ptr = nullptr, + char *values_buf = nullptr, + uint32_t values_size = 0); virtual bool PerformRead(TransactionContext *const current_txn, const ItemPointer &location, @@ -88,14 +90,20 @@ class TimestampOrderingTransactionManager : public TransactionManager { virtual void PerformUpdate(TransactionContext *const current_txn, const ItemPointer &old_location, - const ItemPointer &new_location); + const ItemPointer &new_location, + char *values_buf = nullptr, + uint32_t values_size = 0, + TargetList *offsets = nullptr); virtual void PerformDelete(TransactionContext *const current_txn, const ItemPointer &old_location, const ItemPointer &new_location); virtual void PerformUpdate(TransactionContext *const current_txn, - const ItemPointer &location); + const ItemPointer &location, + char *values_buf = nullptr, + uint32_t values_size = 0, + TargetList *offsets = nullptr); virtual void PerformDelete(TransactionContext *const current_txn, const ItemPointer &location); diff --git a/src/include/concurrency/transaction_manager.h b/src/include/concurrency/transaction_manager.h index f07e232c8d5..c37a8b3acc6 100644 --- a/src/include/concurrency/transaction_manager.h +++ b/src/include/concurrency/transaction_manager.h @@ -106,7 +106,9 @@ class TransactionManager { // which is directly pointed by the primary index. virtual void PerformInsert(TransactionContext *const current_txn, const ItemPointer &location, - ItemPointer *index_entry_ptr = nullptr) = 0; + ItemPointer *index_entry_ptr = nullptr, + char *values_buf = nullptr, + uint32_t values_size = 0) = 0; virtual bool PerformRead(TransactionContext *const current_txn, const ItemPointer &location, @@ -114,14 +116,20 @@ class TransactionManager { virtual void PerformUpdate(TransactionContext *const current_txn, const ItemPointer &old_location, - const ItemPointer &new_location) = 0; + const ItemPointer &new_location, + char *values_buf = nullptr, + uint32_t values_size = 0, + TargetList *offsets = nullptr) = 0; virtual void PerformDelete(TransactionContext *const current_txn, const ItemPointer &old_location, const ItemPointer &new_location) = 0; virtual void PerformUpdate(TransactionContext *const current_txn, - const ItemPointer &location) = 0; + const ItemPointer &location, + char *values_buf = nullptr, + uint32_t values_size = 0, + TargetList *offsets = nullptr) = 0; virtual void PerformDelete(TransactionContext *const current_txn, const ItemPointer &location) = 0; diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index ba0bce34d3e..f28a0f9293a 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -32,6 +32,7 @@ void LogBuffer::WriteRecord(LogRecord &record) { switch (type) { case LogRecordType::TUPLE_INSERT: { + LOG_INFO("inserting tuple"); auto &manager = catalog::Manager::GetInstance(); auto tuple_pos = record.GetItemPointer(); auto tg = manager.GetTileGroup(tuple_pos.block).get(); @@ -54,10 +55,22 @@ void LogBuffer::WriteRecord(LogRecord &record) { break; } case LogRecordType::TUPLE_DELETE: { - LOG_ERROR("Delete logging not supported"); - PL_ASSERT(false); + LOG_INFO("Deleting tuple"); + auto &manager = catalog::Manager::GetInstance(); + auto tuple_pos = record.GetItemPointer(); + auto tg = manager.GetTileGroup(tuple_pos.block).get(); + + // Write down the database id and the table id + log_buffer_.WriteLong(tg->GetDatabaseId()); + log_buffer_.WriteLong(tg->GetTableId()); + + log_buffer_.WriteLong(tuple_pos.block); + log_buffer_.WriteLong(tuple_pos.offset); + + break; } case LogRecordType::TUPLE_UPDATE: { + LOG_INFO("Updating tuple"); auto &manager = catalog::Manager::GetInstance(); auto tuple_pos = record.GetItemPointer(); auto old_tuple_pos = record.GetOldItemPointer(); From a04e25151c7ebd824ac281f99d32e635746cbf7a Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Fri, 30 Mar 2018 17:27:08 -0400 Subject: [PATCH 13/81] Writing the size of the diff buffer into the update log record --- src/logging/log_buffer.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index f28a0f9293a..11591bca975 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -86,6 +86,8 @@ void LogBuffer::WriteRecord(LogRecord &record) { log_buffer_.WriteLong(tuple_pos.block); log_buffer_.WriteLong(tuple_pos.offset); + log_buffer_.WriteLong(record.GetNumValues()); + peloton::type::Value *values_array = reinterpret_cast(record.GetValuesArray()); TargetList *offsets = record.GetOffsets(); for (uint32_t i = 0; i < record.GetNumValues(); i++) { From ec9aa9fdbd57ca26e92c3db6b93126b1f4ab229c Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Mon, 2 Apr 2018 17:56:07 -0400 Subject: [PATCH 14/81] added support for tokenized logging --- ...timestamp_ordering_transaction_manager.cpp | 15 ++------- src/concurrency/transaction_context.cpp | 1 + .../common/container/lock_free_queue.h | 18 +++++++++++ src/include/concurrency/transaction_context.h | 5 +++ src/include/logging/wal_logger.h | 19 +++-------- src/include/threadpool/logger_queue_pool.h | 32 ++++++++++++++++--- src/logging/wal_logger.cpp | 10 +----- src/threadpool/logger_queue_pool.cpp | 2 +- 8 files changed, 61 insertions(+), 41 deletions(-) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 4c954557286..b4dedbaec5e 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -507,7 +507,9 @@ void TimestampOrderingTransactionManager::PerformInsert( LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); /* insert to the queue */ - threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer(current_txn->GetLogBuffer()); + + threadpool::LoggerQueuePool::GetInstance() + .SubmitLogBuffer(current_txn->GetLogToken(), current_txn->GetLogBuffer()); /* allocate a new buffer for the current transaction */ current_txn->ResetLogBuffer(); @@ -803,9 +805,6 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( ////////////////////////////////////////////////////////// auto &manager = catalog::Manager::GetInstance(); - //auto &log_manager = logging::LogManager::GetInstance(); - - //log_manager.StartLogging(); // generate transaction id. cid_t end_commit_id = current_txn->GetCommitId(); @@ -882,8 +881,6 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( gc_set->operator[](tile_group_id)[tuple_slot] = GCVersionType::COMMIT_UPDATE; - //log_manager.LogUpdate(new_version); - } else if (tuple_entry.second == RWType::DELETE) { ItemPointer new_version = tile_group_header->GetPrevItemPointer(tuple_slot); @@ -915,8 +912,6 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( gc_set->operator[](tile_group_id)[tuple_slot] = GCVersionType::COMMIT_DELETE; - //log_manager.LogDelete(ItemPointer(tile_group_id, tuple_slot)); - } else if (tuple_entry.second == RWType::INSERT) { PL_ASSERT(tile_group_header->GetTransactionId(tuple_slot) == current_txn->GetTransactionId()); @@ -931,8 +926,6 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( // nothing to be added to gc set. - //log_manager.LogInsert(ItemPointer(tile_group_id, tuple_slot)); - } else if (tuple_entry.second == RWType::INS_DEL) { PL_ASSERT(tile_group_header->GetTransactionId(tuple_slot) == current_txn->GetTransactionId()); @@ -957,8 +950,6 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( ResultType result = current_txn->GetResult(); - //log_manager.LogEnd(); - EndTransaction(current_txn); // Increment # txns committed metric diff --git a/src/concurrency/transaction_context.cpp b/src/concurrency/transaction_context.cpp index c459a7b2a38..c166117d1db 100644 --- a/src/concurrency/transaction_context.cpp +++ b/src/concurrency/transaction_context.cpp @@ -89,6 +89,7 @@ void TransactionContext::Init(const size_t thread_id, on_commit_triggers_.reset(); + log_token_ = threadpool::LoggerQueuePool::GetInstance().GetLogToken(); ResetLogBuffer(); } diff --git a/src/include/common/container/lock_free_queue.h b/src/include/common/container/lock_free_queue.h index fb721cabfb9..6986fb7a367 100644 --- a/src/include/common/container/lock_free_queue.h +++ b/src/include/common/container/lock_free_queue.h @@ -12,6 +12,7 @@ #pragma once +#include #include "common/macros.h" #include "concurrentqueue/concurrentqueue.h" @@ -21,6 +22,8 @@ namespace peloton { // Lock-free Queue -- Supports multiple consumers and multiple producers. //===--------------------------------------------------------------------===// +using ProducerToken = moodycamel::ProducerToken; + template class LockFreeQueue { public: @@ -32,12 +35,27 @@ class LockFreeQueue { void Enqueue(const T &item) { queue_.enqueue(item); } + void Enqueue(const ProducerToken &token, T &&item) { + queue_.enqueue(token, std::move(item)); + } + + void Enqueue(const ProducerToken &token, const T &item) { + queue_.enqueue(token, item); + } + // Dequeues one item, returning true if an item was found // or false if the queue appeared empty bool Dequeue(T &item) { return queue_.try_dequeue(item); } bool IsEmpty() { return queue_.size_approx() == 0; } + void GenerateTokens(std::vector &tokens, int num_tokens){ + for(int i=0; i queue_; diff --git a/src/include/concurrency/transaction_context.h b/src/include/concurrency/transaction_context.h index 2718b1f89b5..c57cf097e16 100644 --- a/src/include/concurrency/transaction_context.h +++ b/src/include/concurrency/transaction_context.h @@ -25,6 +25,7 @@ #include "common/internal_types.h" #include "logging/log_buffer.h" #include "logging/wal_log_manager.h" +#include "threadpool/logger_queue_pool.h" namespace peloton { @@ -77,6 +78,8 @@ class TransactionContext : public Printable { inline uint64_t GetTimestamp() const { return timestamp_; } + inline int GetLogToken() const { return log_token_; } + inline logging::LogBuffer* GetLogBuffer() const { return log_buffer_; } inline const std::vector& GetQueryStrings() const { @@ -86,6 +89,7 @@ class TransactionContext : public Printable { log_buffer_ = new logging::LogBuffer(logging::LogManager::GetInstance().GetTransactionBufferSize()); } + inline void SetCommitId(const cid_t commit_id) { commit_id_ = commit_id; } inline void SetEpochId(const eid_t epoch_id) { epoch_id_ = epoch_id; } @@ -217,6 +221,7 @@ class TransactionContext : public Printable { std::unique_ptr on_commit_triggers_; + int log_token_; logging::LogBuffer *log_buffer_; }; diff --git a/src/include/logging/wal_logger.h b/src/include/logging/wal_logger.h index ae39598091d..fb69a3d0dbd 100644 --- a/src/include/logging/wal_logger.h +++ b/src/include/logging/wal_logger.h @@ -6,6 +6,7 @@ #include +#include "common/container/lock_free_queue.h" #include "logging/log_buffer.h" #include "logging/log_record.h" #include "logging/wal_log_manager.h" @@ -13,23 +14,14 @@ namespace peloton { - -namespace storage { - class TileGroupHeader; -} - namespace logging { - +using LogToken = peloton::ProducerToken; class WalLogger { -private: - struct logger_callback{ - - }; - public: + WalLogger() { disk_buffer_ = new LogBuffer(LogManager::GetLoggerBufferSize()); } @@ -38,14 +30,11 @@ class WalLogger { bool IsFlushNeeded(bool pending_buffers); void FlushToDisk(); - void AppendLogBuffer(LogBuffer *buffer); + void PerformCompaction(LogBuffer *buffer); private: LogBuffer *disk_buffer_; std::deque> callbacks_; - }; } } - - diff --git a/src/include/threadpool/logger_queue_pool.h b/src/include/threadpool/logger_queue_pool.h index cbc1141b403..1533dd0f66f 100644 --- a/src/include/threadpool/logger_queue_pool.h +++ b/src/include/threadpool/logger_queue_pool.h @@ -11,22 +11,27 @@ //===----------------------------------------------------------------------===// #pragma once +#include #include "worker_pool.h" #include "logging/log_buffer.h" #include "logging/wal_logger.h" + // TODO: tune these variables constexpr static size_t kDefaultLoggerQueueSize = 32; constexpr static size_t kDefaultLoggerPoolSize = 1; +constexpr static size_t kDefaultNumTokens = 8; + namespace peloton { namespace threadpool { using LoggerQueue = peloton::LockFreeQueue; + void LoggerFunc(std::atomic_bool *is_running, LoggerQueue *logger_queue); - /** +/** * @class LoggerQueuePool * @brief Wrapper class for single queue and single pool * One should use this if possible. @@ -36,13 +41,21 @@ class LoggerQueuePool { LoggerQueuePool(size_t num_workers) : logger_queue_(kDefaultLoggerQueueSize), num_workers_(num_workers), - is_running_(false) {} + is_running_(false) { + + logger_queue_.GenerateTokens(log_tokens_, kDefaultNumTokens); + } ~LoggerQueuePool() { if (is_running_ == true) Shutdown(); } + inline int GetLogToken(){ + return next_token_++; + } + + void Startup() { for (size_t i = 0; i < num_workers_; i++) { loggers_.emplace_back(LoggerFunc, &is_running_, &logger_queue_); @@ -64,6 +77,14 @@ class LoggerQueuePool { logger_queue_.Enqueue(std::move(buffer)); } + void SubmitLogBuffer(int token, logging::LogBuffer *buffer) { + if (is_running_ == false) + Startup(); + + int idx = (token)%kDefaultNumTokens; + logger_queue_.Enqueue(*(log_tokens_[idx]), std::move(buffer)); + } + static LoggerQueuePool &GetInstance() { static LoggerQueuePool logger_queue_pool(kDefaultLoggerPoolSize); @@ -72,10 +93,13 @@ class LoggerQueuePool { private: LoggerQueue logger_queue_; - std::vector loggers_; - size_t num_workers_; + + std::vector loggers_; + std::vector log_tokens_; std::atomic_bool is_running_; + std::atomic next_token_; + }; } // namespace threadpool diff --git a/src/logging/wal_logger.cpp b/src/logging/wal_logger.cpp index dc37a1aeb7c..4b77b62510c 100644 --- a/src/logging/wal_logger.cpp +++ b/src/logging/wal_logger.cpp @@ -16,8 +16,6 @@ namespace logging{ bool WalLogger::IsFlushNeeded(bool pending_buffers){ - - /* if the disk buffer is full then we flush to the disk * irrespective of whether there are pending callbacks */ @@ -59,7 +57,7 @@ void WalLogger::FlushToDisk(){ } -void WalLogger::AppendLogBuffer(LogBuffer *log_buffer){ +void WalLogger::PerformCompaction(LogBuffer *log_buffer){ if(nullptr==log_buffer) PL_ASSERT(false); @@ -77,11 +75,5 @@ void WalLogger::AppendLogBuffer(LogBuffer *log_buffer){ } - - - - - - } } \ No newline at end of file diff --git a/src/threadpool/logger_queue_pool.cpp b/src/threadpool/logger_queue_pool.cpp index 46b496e93a4..09203c83c81 100644 --- a/src/threadpool/logger_queue_pool.cpp +++ b/src/threadpool/logger_queue_pool.cpp @@ -25,7 +25,7 @@ void LoggerFunc(std::atomic_bool *is_running, LoggerQueue *logger_queue) { pause_time = std::min(pause_time * 2, kMaxPauseTime); } else { - logger.AppendLogBuffer(log_buffer); + logger.PerformCompaction(log_buffer); if(logger.IsFlushNeeded(logger_queue->IsEmpty())){ logger.FlushToDisk(); From 2f8b695701578bee8f394dcf6ad11261e99ae0e7 Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Wed, 4 Apr 2018 17:15:19 -0400 Subject: [PATCH 15/81] Do not assert for commit log records --- src/include/logging/log_record.h | 3 ++- src/logging/log_buffer.cpp | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/include/logging/log_record.h b/src/include/logging/log_record.h index d8142a29368..dbdc733f8de 100644 --- a/src/include/logging/log_record.h +++ b/src/include/logging/log_record.h @@ -105,7 +105,8 @@ class LogRecordFactory { txn_id_t txn_id, cid_t current_cid) { PL_ASSERT(log_type == LogRecordType::TUPLE_INSERT || log_type == LogRecordType::TUPLE_DELETE || - log_type == LogRecordType::TUPLE_UPDATE); + log_type == LogRecordType::TUPLE_UPDATE || + log_type == LogRecordType::TRANSACTION_COMMIT); return LogRecord(log_type, pos, current_eid, txn_id, current_cid); } // static LogRecord CreateTupleRecord(const LogRecordType log_type, const ItemPointer &pos) { diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index 11591bca975..0dc5982632d 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -100,9 +100,6 @@ void LogBuffer::WriteRecord(LogRecord &record) { break; } default: { - LOG_ERROR("Unsupported log record type"); - PL_ASSERT(false); - } } From 72320a1cfee885f9912dc97012c86ec5d559da78 Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Thu, 5 Apr 2018 22:24:09 -0400 Subject: [PATCH 16/81] Adding log changes to old engine to log schema changes --- ...timestamp_ordering_transaction_manager.cpp | 22 +++++++++---------- src/executor/insert_executor.cpp | 16 ++++++++++++-- src/executor/update_executor.cpp | 10 ++++++++- src/include/common/internal_types.h | 1 + src/storage/data_table.cpp | 9 ++++++-- 5 files changed, 42 insertions(+), 16 deletions(-) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 7f71886e51c..2bfcc7242ab 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -1029,17 +1029,8 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( current_txn->GetTransactionId(), current_txn->GetCommitId()); current_txn->GetLogBuffer()->WriteRecord(record); - - if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { - - LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); - /* insert to the queue */ - threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( - current_txn->GetLogToken(), current_txn->GetLogBuffer()); - - /* allocate a new buffer for the current transaction */ - current_txn->ResetLogBuffer(); - } + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); // Increment # txns committed metric if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != @@ -1226,6 +1217,15 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( current_txn->SetResult(ResultType::ABORTED); EndTransaction(current_txn); + ItemPointer placeholder; + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TRANSACTION_ABORT, placeholder, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); + + current_txn->GetLogBuffer()->WriteRecord(record); + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); // Increment # txns aborted metric if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { diff --git a/src/executor/insert_executor.cpp b/src/executor/insert_executor.cpp index 993fb377154..b265ea10728 100644 --- a/src/executor/insert_executor.cpp +++ b/src/executor/insert_executor.cpp @@ -110,10 +110,13 @@ bool InsertExecutor::DExecute() { for (oid_t tuple_id : *logical_tile) { ContainerTuple cur_tuple(logical_tile.get(), tuple_id); + std::vector values; + // Materialize the logical tile tuple for (oid_t column_itr = 0; column_itr < column_count; column_itr++) { type::Value val = (cur_tuple.GetValue(column_itr)); tuple->SetValue(column_itr, val, executor_pool); + values.push_back(val); } // insert tuple into the table. @@ -130,7 +133,7 @@ bool InsertExecutor::DExecute() { return false; } - transaction_manager.PerformInsert(current_txn, location, index_entry_ptr); + transaction_manager.PerformInsert(current_txn, location, index_entry_ptr, reinterpret_cast(values.data()), values.size()); executor_context_->num_processed += 1; // insert one } @@ -187,9 +190,11 @@ bool InsertExecutor::DExecute() { // if we are doing a bulk insert from values not project_info if (!project_info) { + LOG_INFO("Not project info"); tuple = node.GetTuple(insert_itr); if (tuple == nullptr) { + LOG_INFO("tuple is null"); storage_tuple.reset(new storage::Tuple(schema, true)); // read from values @@ -245,7 +250,14 @@ bool InsertExecutor::DExecute() { return false; } - transaction_manager.PerformInsert(current_txn, location, index_entry_ptr); + std::vector values; + uint32_t num_columns = schema->GetColumnCount(); + for (uint32_t col_id = 0; col_id < num_columns; col_id++) { + values.push_back(new_tuple->GetValue(col_id)); + } + + LOG_INFO("Perform insert is being called"); + transaction_manager.PerformInsert(current_txn, location, index_entry_ptr, reinterpret_cast(values.data()), values.size()); LOG_TRACE("Number of tuples in table after insert: %lu", target_table->GetTupleCount()); diff --git a/src/executor/update_executor.cpp b/src/executor/update_executor.cpp index 19afb685c2b..5cf0d2c1dff 100644 --- a/src/executor/update_executor.cpp +++ b/src/executor/update_executor.cpp @@ -106,6 +106,12 @@ bool UpdateExecutor::PerformUpdatePrimaryKey( peloton::ItemPointer location = target_table_->InsertTuple(&new_tuple, current_txn, &index_entry_ptr); + std::vector values; + uint32_t num_columns = target_table_schema->GetColumnCount(); + for (uint32_t col_id = 0; col_id < num_columns; col_id++) { + values.push_back(new_tuple.GetValue(col_id)); + } + // it is possible that some concurrent transactions have inserted the // same tuple. In this case, abort the transaction. if (location.block == INVALID_OID) { @@ -135,7 +141,7 @@ bool UpdateExecutor::PerformUpdatePrimaryKey( } } - transaction_manager.PerformInsert(current_txn, location, index_entry_ptr); + transaction_manager.PerformInsert(current_txn, location, index_entry_ptr, reinterpret_cast(values.data()), values.size()); return true; } @@ -261,6 +267,7 @@ bool UpdateExecutor::DExecute() { project_info_->Evaluate(&old_tuple, &old_tuple, nullptr, executor_context_); + //TODO(Anirudh): Do we need the values buf and offset buf transaction_manager.PerformUpdate(current_txn, old_location); } } @@ -357,6 +364,7 @@ bool UpdateExecutor::DExecute() { old_location.offset); LOG_TRACE("perform update new location: %u, %u", new_location.block, new_location.offset); + //TODO(Anirudh): Do we need to create the values and offset buffer transaction_manager.PerformUpdate(current_txn, old_location, new_location); diff --git a/src/include/common/internal_types.h b/src/include/common/internal_types.h index 0ffd57507a6..4b5907f7784 100644 --- a/src/include/common/internal_types.h +++ b/src/include/common/internal_types.h @@ -928,6 +928,7 @@ enum class LogRecordType { // TransactionContext-related records TRANSACTION_BEGIN = 1, TRANSACTION_COMMIT = 2, + TRANSACTION_ABORT = 3, // Generic dml records TUPLE_INSERT = 11, diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index d90ebe35789..de90a2fea16 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -722,8 +722,13 @@ bool DataTable::CheckForeignKeySrcAndCascade(storage::Tuple *prev_tuple, return false; } - transaction_manager.PerformInsert(current_txn, location, index_entry_ptr); - + std::vector values; + uint32_t num_columns = src_table->GetSchema()->GetColumnCount(); + for (uint32_t col_id = 0; col_id < num_columns; col_id++) { + values.push_back(src_new_tuple.GetValue(col_id)); + } + transaction_manager.PerformInsert(current_txn, location, index_entry_ptr, + reinterpret_cast(values.data(), values.size())); break; } } From 8b431e9fc8bc75ab8bfaa5b68cc77d75eb334ba0 Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Fri, 6 Apr 2018 04:31:37 -0400 Subject: [PATCH 17/81] plugged in callbacks --- ...timestamp_ordering_transaction_manager.cpp | 102 ++++++++++++------ src/concurrency/transaction_manager.cpp | 1 - src/include/common/internal_types.h | 1 + .../timestamp_ordering_transaction_manager.h | 6 +- src/include/concurrency/transaction_manager.h | 11 +- src/include/logging/log_buffer.h | 24 ++--- src/include/logging/log_record.h | 42 +++++--- src/include/logging/wal_logger.h | 2 +- src/include/traffic_cop/traffic_cop.h | 8 +- src/logging/log_buffer.cpp | 7 ++ src/logging/wal_logger.cpp | 14 ++- src/threadpool/logger_queue_pool.cpp | 3 +- src/traffic_cop/traffic_cop.cpp | 70 ++++++++---- 13 files changed, 190 insertions(+), 101 deletions(-) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 7f71886e51c..7168067f3f0 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -495,7 +495,6 @@ void TimestampOrderingTransactionManager::PerformInsert( // Write down the head pointer's address in tile group header tile_group_header->SetIndirection(tuple_id, index_entry_ptr); - LOG_INFO("perfom insert"); if (values_buf != nullptr) { logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( @@ -600,13 +599,13 @@ void TimestampOrderingTransactionManager::PerformUpdate( // Add the old tuple into the update set current_txn->RecordUpdate(old_location); - LOG_INFO("Perform Update"); if (values_buf != nullptr) { logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_UPDATE, new_location, current_txn->GetEpochId(), + LogRecordType::TUPLE_UPDATE, location, new_location, current_txn->GetEpochId(), current_txn->GetTransactionId(), current_txn->GetCommitId()); + record.SetOldItemPointer(location); record.SetValuesArray(values_buf, values_size); record.SetOffsetsArray(offsets); @@ -615,7 +614,6 @@ void TimestampOrderingTransactionManager::PerformUpdate( if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { - LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); /* insert to the queue */ threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( @@ -635,9 +633,10 @@ void TimestampOrderingTransactionManager::PerformUpdate( // NOTE: this function is deprecated. void TimestampOrderingTransactionManager::PerformUpdate( - TransactionContext *const current_txn UNUSED_ATTRIBUTE, + TransactionContext *const current_txn, const ItemPointer &location, char *values_buf, uint32_t values_size, TargetList *offsets) { + PL_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); oid_t tile_group_id = location.block; @@ -662,8 +661,10 @@ void TimestampOrderingTransactionManager::PerformUpdate( if (values_buf != nullptr) { logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_UPDATE, location, current_txn->GetEpochId(), - current_txn->GetTransactionId(), current_txn->GetCommitId()); + LogRecordType::TUPLE_UPDATE, location, INVALID_ITEMPOINTER, + current_txn->GetEpochId(), current_txn->GetTransactionId(), + current_txn->GetCommitId()); + record.SetOldItemPointer(location); record.SetValuesArray(values_buf, values_size); record.SetOffsetsArray(offsets); @@ -672,8 +673,6 @@ void TimestampOrderingTransactionManager::PerformUpdate( if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { - LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); - /* insert to the queue */ threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( current_txn->GetLogToken(), current_txn->GetLogBuffer()); @@ -682,7 +681,7 @@ void TimestampOrderingTransactionManager::PerformUpdate( current_txn->ResetLogBuffer(); } } - LOG_INFO("Perform Update"); + // Increment table update op stats if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { @@ -853,7 +852,8 @@ void TimestampOrderingTransactionManager::PerformDelete( } ResultType TimestampOrderingTransactionManager::CommitTransaction( - TransactionContext *const current_txn) { + TransactionContext *const current_txn, std::function task_callback) { + LOG_DEBUG("Committing peloton txn : %" PRId64, current_txn->GetTransactionId()); ////////////////////////////////////////////////////////// @@ -1018,41 +1018,46 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( } } - ResultType result = current_txn->GetResult(); + // Increment # txns committed metric + if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != + StatsType::INVALID) { + stats::BackendStatsContext::GetInstance()->IncrementTxnCommitted( + database_id); + } - EndTransaction(current_txn); + /* TODO(gandeevan): add logging switch */ + if(task_callback != nullptr) { - ItemPointer placeholder; - logging::LogRecord record = - logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TRANSACTION_COMMIT, placeholder, current_txn->GetEpochId(), - current_txn->GetTransactionId(), current_txn->GetCommitId()); + ResultType result = current_txn->GetResult(); - current_txn->GetLogBuffer()->WriteRecord(record); + auto on_flush = [this, &result, task_callback, current_txn] (){ + current_txn->SetResult(result); + this->EndTransaction(current_txn); + task_callback(result); + }; - if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TRANSACTION_COMMIT, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); + + current_txn->GetLogBuffer()->WriteRecord(record); + + current_txn->GetLogBuffer()->SetLoggerCallback(on_flush); - LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); /* insert to the queue */ threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( - current_txn->GetLogToken(), current_txn->GetLogBuffer()); - - /* allocate a new buffer for the current transaction */ - current_txn->ResetLogBuffer(); - } + current_txn->GetLogToken(), current_txn->GetLogBuffer()); - // Increment # txns committed metric - if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != - StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTxnCommitted( - database_id); + current_txn->SetResult(ResultType::QUEUING); } - return result; + return current_txn->GetResult(); } ResultType TimestampOrderingTransactionManager::AbortTransaction( - TransactionContext *const current_txn) { + TransactionContext *const current_txn, std::function task_callback) { + // a pre-declared read-only transaction will never abort. PL_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); @@ -1224,7 +1229,7 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( } current_txn->SetResult(ResultType::ABORTED); - EndTransaction(current_txn); + // Increment # txns aborted metric if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != @@ -1232,7 +1237,34 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( stats::BackendStatsContext::GetInstance()->IncrementTxnAborted(database_id); } - return ResultType::ABORTED; + /* TODO(gandeevan): add logging switch */ + if(task_callback != nullptr) { + + ResultType result = current_txn->GetResult(); + + auto on_flush = [this, &result, task_callback, current_txn] (){ + current_txn->SetResult(result); + this->EndTransaction(current_txn); + task_callback(result); + }; + + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TRANSACTION_ABORT, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); + + current_txn->GetLogBuffer()->WriteRecord(record); + + current_txn->GetLogBuffer()->SetLoggerCallback(on_flush); + + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); + + current_txn->SetResult(ResultType::QUEUING); + } + + return current_txn->GetResult(); } } // namespace storage diff --git a/src/concurrency/transaction_manager.cpp b/src/concurrency/transaction_manager.cpp index 485d6e4cc23..9d98dd63ac9 100644 --- a/src/concurrency/transaction_manager.cpp +++ b/src/concurrency/transaction_manager.cpp @@ -96,7 +96,6 @@ void TransactionManager::EndTransaction(TransactionContext *current_txn) { stats::BackendStatsContext::GetInstance() ->GetTxnLatencyMetric() .RecordLatency(); - } } diff --git a/src/include/common/internal_types.h b/src/include/common/internal_types.h index 0ffd57507a6..4b5907f7784 100644 --- a/src/include/common/internal_types.h +++ b/src/include/common/internal_types.h @@ -928,6 +928,7 @@ enum class LogRecordType { // TransactionContext-related records TRANSACTION_BEGIN = 1, TRANSACTION_COMMIT = 2, + TRANSACTION_ABORT = 3, // Generic dml records TUPLE_INSERT = 11, diff --git a/src/include/concurrency/timestamp_ordering_transaction_manager.h b/src/include/concurrency/timestamp_ordering_transaction_manager.h index 70b5951ec5e..33a5407ed56 100644 --- a/src/include/concurrency/timestamp_ordering_transaction_manager.h +++ b/src/include/concurrency/timestamp_ordering_transaction_manager.h @@ -108,9 +108,11 @@ class TimestampOrderingTransactionManager : public TransactionManager { virtual void PerformDelete(TransactionContext *const current_txn, const ItemPointer &location); - virtual ResultType CommitTransaction(TransactionContext *const current_txn); + virtual ResultType CommitTransaction(TransactionContext *const current_txn, + std::function task_callback = nullptr); - virtual ResultType AbortTransaction(TransactionContext *const current_txn); + virtual ResultType AbortTransaction(TransactionContext *const current_txn, + std::function task_callback = nullptr); private: diff --git a/src/include/concurrency/transaction_manager.h b/src/include/concurrency/transaction_manager.h index c37a8b3acc6..5bfefd79d18 100644 --- a/src/include/concurrency/transaction_manager.h +++ b/src/include/concurrency/transaction_manager.h @@ -147,9 +147,16 @@ class TransactionManager { void EndTransaction(TransactionContext *current_txn); - virtual ResultType CommitTransaction(TransactionContext *const current_txn) = 0; - virtual ResultType AbortTransaction(TransactionContext *const current_txn) = 0; + virtual ResultType CommitTransaction( + TransactionContext *const current_txn, + std::function task_callback = nullptr) = 0; + + + virtual ResultType AbortTransaction( + TransactionContext *const current_txn, + std::function task_callback = nullptr) = 0; + // this function generates the maximum commit id of committed transactions. // please note that this function only returns a "safe" value instead of a diff --git a/src/include/logging/log_buffer.h b/src/include/logging/log_buffer.h index 2934767b131..cab08b6295d 100644 --- a/src/include/logging/log_buffer.h +++ b/src/include/logging/log_buffer.h @@ -7,8 +7,7 @@ namespace peloton{ namespace logging{ - -typedef void (*CallbackFunction)(void *); // function pointer type +using LoggerCallback = std::function; class LogBuffer{ @@ -16,8 +15,7 @@ class LogBuffer{ LogBuffer(size_t threshold) : log_buffer_threshold_(threshold), - task_callback_(nullptr), - task_callback_arg_(nullptr) {} + on_flush_(nullptr) {} ~LogBuffer() {} @@ -31,23 +29,21 @@ class LogBuffer{ inline bool HasThresholdExceeded() { return log_buffer_.Size() >= log_buffer_threshold_; } inline size_t GetThreshold() { return log_buffer_threshold_; } - inline CallbackFunction GetCallback() { return task_callback_; } - inline void* GetCallbackArgs() { return task_callback_arg_; } - void SetTaskCallback(void (*task_callback)(void *), void *task_callback_arg) { - task_callback_ = task_callback; - task_callback_arg_ = task_callback_arg; + + inline void SetLoggerCallback(const LoggerCallback &on_flush) { + on_flush_ = std::move(on_flush); + } + + inline LoggerCallback &GetLoggerCallback() { + return on_flush_; } private: CopySerializeOutput log_buffer_; size_t log_buffer_threshold_; - // The current callback to be invoked after logging completes. - void (*task_callback_)(void *); - void *task_callback_arg_; - - + LoggerCallback on_flush_; }; } diff --git a/src/include/logging/log_record.h b/src/include/logging/log_record.h index dbdc733f8de..f35037cc717 100644 --- a/src/include/logging/log_record.h +++ b/src/include/logging/log_record.h @@ -28,9 +28,10 @@ namespace logging { class LogRecord { friend class LogRecordFactory; private: - LogRecord(LogRecordType log_type, const ItemPointer &pos, + LogRecord(LogRecordType log_type, const ItemPointer &old_pos, const ItemPointer &pos, const eid_t epoch_id, const txn_id_t txn_id, const cid_t commit_id) - : log_record_type_(log_type), + : log_record_type_(log_type), + old_tuple_pos_(old_pos), tuple_pos_(pos), eid_(epoch_id), txn_id_(txn_id), @@ -79,10 +80,10 @@ class LogRecord { private: LogRecordType log_record_type_; - ItemPointer tuple_pos_; - ItemPointer old_tuple_pos_; + ItemPointer tuple_pos_; + eid_t eid_; txn_id_t txn_id_; @@ -104,17 +105,30 @@ class LogRecordFactory { const ItemPointer &pos, eid_t current_eid, txn_id_t txn_id, cid_t current_cid) { PL_ASSERT(log_type == LogRecordType::TUPLE_INSERT || - log_type == LogRecordType::TUPLE_DELETE || - log_type == LogRecordType::TUPLE_UPDATE || - log_type == LogRecordType::TRANSACTION_COMMIT); - return LogRecord(log_type, pos, current_eid, txn_id, current_cid); + log_type == LogRecordType::TUPLE_DELETE); + + return LogRecord(log_type, INVALID_ITEMPOINTER, pos, current_eid, txn_id, current_cid); } -// static LogRecord CreateTupleRecord(const LogRecordType log_type, const ItemPointer &pos) { -// PL_ASSERT(log_type == LogRecordType::TUPLE_INSERT || -// log_type == LogRecordType::TUPLE_DELETE || -// log_type == LogRecordType::TUPLE_UPDATE); -// return LogRecord(log_type, pos, INVALID_EID, INVALID_CID); -// } + + static LogRecord CreateTupleRecord(const LogRecordType log_type, eid_t current_eid, + txn_id_t txn_id, cid_t current_cid) { + + PL_ASSERT(log_type == LogRecordType::TRANSACTION_COMMIT || + log_type == LogRecordType::TRANSACTION_ABORT); + + return LogRecord(log_type, INVALID_ITEMPOINTER, INVALID_ITEMPOINTER, + current_eid, txn_id, current_cid); + } + + static LogRecord CreateTupleRecord(const LogRecordType log_type, const ItemPointer &old_pos, + const ItemPointer &pos, eid_t current_eid, + txn_id_t txn_id, cid_t current_cid) { + + PL_ASSERT(log_type == LogRecordType::TUPLE_UPDATE); + + return LogRecord(log_type, old_pos, pos, current_eid, txn_id, current_cid); + } + // // static LogRecord CreateTxnRecord(const LogRecordType log_type, const cid_t commit_id) { // PL_ASSERT(log_type == LogRecordType::TRANSACTION_BEGIN || diff --git a/src/include/logging/wal_logger.h b/src/include/logging/wal_logger.h index fb69a3d0dbd..d125efedbbc 100644 --- a/src/include/logging/wal_logger.h +++ b/src/include/logging/wal_logger.h @@ -34,7 +34,7 @@ class WalLogger { private: LogBuffer *disk_buffer_; - std::deque> callbacks_; + std::deque callbacks_; }; } } diff --git a/src/include/traffic_cop/traffic_cop.h b/src/include/traffic_cop/traffic_cop.h index e324b87fe82..464433be099 100644 --- a/src/include/traffic_cop/traffic_cop.h +++ b/src/include/traffic_cop/traffic_cop.h @@ -98,9 +98,11 @@ class TrafficCop { tcop_txn_state_.emplace(txn, ResultType::SUCCESS); } - ResultType CommitQueryHelper(); + ResultType CommitQueryHelper(std::function callback = nullptr); - void ExecuteStatementPlanGetResult(); + ResultType AbortQueryHelper(std::function callback = nullptr); + + ResultType ExecuteStatementPlanGetResult(); ResultType ExecuteStatementGetResult(); @@ -189,7 +191,7 @@ class TrafficCop { ResultType BeginQueryHelper(size_t thread_id); - ResultType AbortQueryHelper(); + // Get all data tables from a TableRef. // For multi-way join diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index 0dc5982632d..74f8d5b1a3f 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -99,7 +99,14 @@ void LogBuffer::WriteRecord(LogRecord &record) { break; } + + case LogRecordType::TRANSACTION_COMMIT: + case LogRecordType::TRANSACTION_ABORT: { + break; + } + default: { + PL_ASSERT(false); } } diff --git a/src/logging/wal_logger.cpp b/src/logging/wal_logger.cpp index 4b77b62510c..bb8ed187003 100644 --- a/src/logging/wal_logger.cpp +++ b/src/logging/wal_logger.cpp @@ -48,12 +48,11 @@ void WalLogger::FlushToDisk(){ /* send out the callbacks */ while(!callbacks_.empty()){ - auto callback_pair = callbacks_.front(); - CallbackFunction callback_func = callback_pair.first; - void *callback_args = callback_pair.second; - - callback_func(callback_args); + auto callback = callbacks_.front(); + callbacks_.pop_front(); + callback(); } + } @@ -66,11 +65,10 @@ void WalLogger::PerformCompaction(LogBuffer *log_buffer){ .WriteBytes(log_buffer->GetData(), log_buffer->GetSize()); - CallbackFunction callback = log_buffer->GetCallback(); - void *callback_args = log_buffer->GetCallbackArgs(); + auto callback = log_buffer->GetLoggerCallback(); if(nullptr != callback){ - callbacks_.emplace_back(callback, callback_args); + callbacks_.push_back(std::move(callback)); } diff --git a/src/threadpool/logger_queue_pool.cpp b/src/threadpool/logger_queue_pool.cpp index 234a3d531ff..99a3bcd3110 100644 --- a/src/threadpool/logger_queue_pool.cpp +++ b/src/threadpool/logger_queue_pool.cpp @@ -27,12 +27,11 @@ void LoggerFunc(std::atomic_bool *is_running, LoggerQueue *logger_queue) { logger.PerformCompaction(log_buffer); - if(logger.IsFlushNeeded(logger_queue->IsEmpty())){ + if(logger.IsFlushNeeded(!logger_queue->IsEmpty())){ logger.FlushToDisk(); } // TODO(gandeevan): free log buffers - pause_time = kMinPauseTime; } } diff --git a/src/traffic_cop/traffic_cop.cpp b/src/traffic_cop/traffic_cop.cpp index 8b248c5c4e5..174b22a5efe 100644 --- a/src/traffic_cop/traffic_cop.cpp +++ b/src/traffic_cop/traffic_cop.cpp @@ -93,7 +93,7 @@ ResultType TrafficCop::BeginQueryHelper(size_t thread_id) { return ResultType::SUCCESS; } -ResultType TrafficCop::CommitQueryHelper() { +ResultType TrafficCop::CommitQueryHelper(std::function task_callback) { // do nothing if we have no active txns if (tcop_txn_state_.empty()) return ResultType::NOOP; auto &curr_state = tcop_txn_state_.top(); @@ -104,16 +104,17 @@ ResultType TrafficCop::CommitQueryHelper() { // If this exception is caused by a query in a transaction, // I will block following queries in that transaction until 'COMMIT' or // 'ROLLBACK' After receive 'COMMIT', see if it is rollback or really commit. + if (curr_state.second != ResultType::ABORTED) { // txn committed - return txn_manager.CommitTransaction(txn); + return txn_manager.CommitTransaction(txn, task_callback); } else { // otherwise, rollback - return txn_manager.AbortTransaction(txn); + return txn_manager.AbortTransaction(txn, task_callback); } } -ResultType TrafficCop::AbortQueryHelper() { +ResultType TrafficCop::AbortQueryHelper(std::function task_callback) { // do nothing if we have no active txns if (tcop_txn_state_.empty()) return ResultType::NOOP; auto &curr_state = tcop_txn_state_.top(); @@ -122,8 +123,7 @@ ResultType TrafficCop::AbortQueryHelper() { if (curr_state.second != ResultType::ABORTED) { auto txn = curr_state.first; auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto result = txn_manager.AbortTransaction(txn); - return result; + return txn_manager.AbortTransaction(txn, task_callback); } else { delete curr_state.first; // otherwise, the txn has already been aborted @@ -189,9 +189,11 @@ executor::ExecutionResult TrafficCop::ExecuteHelper( result = std::move(values); // COMMIT single statement transaction - this->ExecuteStatementPlanGetResult(); + ResultType result = this->ExecuteStatementPlanGetResult(); - task_callback_(task_callback_arg_); + if(result != ResultType::QUEUING) { + task_callback_(task_callback_arg_); + } }; auto &pool = threadpool::MonoQueuePool::GetInstance(); @@ -207,19 +209,28 @@ executor::ExecutionResult TrafficCop::ExecuteHelper( return p_status_; } -void TrafficCop::ExecuteStatementPlanGetResult() { - if (p_status_.m_result == ResultType::FAILURE) return; +ResultType TrafficCop::ExecuteStatementPlanGetResult() { + - if(tcop_txn_state_.empty()) return; + if (p_status_.m_result == ResultType::FAILURE) return ResultType::FAILURE; + + if(tcop_txn_state_.empty()) return ResultType::NOOP; auto txn_result = GetCurrentTxnState().first->GetResult(); + if (single_statement_txn_ || txn_result == ResultType::FAILURE) { LOG_TRACE("About to commit/abort: single stmt: %d,txn_result: %s", single_statement_txn_, ResultTypeToString(txn_result).c_str()); + + auto on_complete = [this](ResultType result) { + this->p_status_.m_result = result; + this->task_callback_(this->task_callback_arg_); + }; + switch (txn_result) { case ResultType::SUCCESS: // Commit single statement - p_status_.m_result = CommitQueryHelper(); + return CommitQueryHelper(on_complete); break; case ResultType::FAILURE: @@ -228,13 +239,15 @@ void TrafficCop::ExecuteStatementPlanGetResult() { LOG_TRACE("Abort Transaction"); if (single_statement_txn_) { LOG_TRACE("Tcop_txn_state size: %lu", tcop_txn_state_.size()); - p_status_.m_result = AbortQueryHelper(); + return AbortQueryHelper(on_complete); } else { tcop_txn_state_.top().second = ResultType::ABORTED; p_status_.m_result = ResultType::ABORTED; + return ResultType::ABORTED; } } } + return ResultType::SUCCESS; } /* @@ -572,6 +585,8 @@ ResultType TrafficCop::ExecuteStatement( static_cast(statement->GetQueryType())); try { + + switch (statement->GetQueryType()) { case QueryType::QUERY_BEGIN: { return BeginQueryHelper(thread_id); @@ -580,20 +595,37 @@ ResultType TrafficCop::ExecuteStatement( this->is_queuing_ = true; auto &pool = threadpool::MonoQueuePool::GetInstance(); - pool.SubmitTask([this] { - this->CommitQueryHelper(); - task_callback_(task_callback_arg_); + auto on_commit = [this](ResultType result UNUSED_ATTRIBUTE) { + this->task_callback_(this->task_callback_arg_); + }; + + pool.SubmitTask([this, on_commit] { + ResultType result = this->CommitQueryHelper(on_commit); + + if(result!=ResultType::QUEUING) + task_callback_(task_callback_arg_); }); return ResultType::QUEUING; } + case QueryType::QUERY_ROLLBACK: { + this->is_queuing_ = true; auto &pool = threadpool::MonoQueuePool::GetInstance(); - pool.SubmitTask([this] { - this->AbortQueryHelper(); - task_callback_(task_callback_arg_); + auto on_abort = [this](ResultType result UNUSED_ATTRIBUTE) { + this->task_callback_(this->task_callback_arg_); + }; + + + pool.SubmitTask([this, on_abort] { + ResultType result = this->AbortQueryHelper(on_abort); + + if(result!=ResultType::QUEUING) { + task_callback_(task_callback_arg_); + } + }); return ResultType::QUEUING; From 48ee8677e110e1ca982b7d2f2a1277ae924610d7 Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Fri, 6 Apr 2018 11:42:36 -0400 Subject: [PATCH 18/81] fixing merge mistakes for abort --- .../timestamp_ordering_transaction_manager.cpp | 10 ---------- src/executor/insert_executor.cpp | 3 +-- src/logging/log_buffer.cpp | 6 +++--- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index b707788d187..ce1d50ac236 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -1230,16 +1230,6 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( current_txn->SetResult(ResultType::ABORTED); - - ItemPointer placeholder; - logging::LogRecord record = - logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TRANSACTION_ABORT, placeholder, current_txn->GetEpochId(), - current_txn->GetTransactionId(), current_txn->GetCommitId()); - - current_txn->GetLogBuffer()->WriteRecord(record); - threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( - current_txn->GetLogToken(), current_txn->GetLogBuffer()); // Increment # txns aborted metric if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { diff --git a/src/executor/insert_executor.cpp b/src/executor/insert_executor.cpp index b265ea10728..d8f569f01c0 100644 --- a/src/executor/insert_executor.cpp +++ b/src/executor/insert_executor.cpp @@ -190,7 +190,6 @@ bool InsertExecutor::DExecute() { // if we are doing a bulk insert from values not project_info if (!project_info) { - LOG_INFO("Not project info"); tuple = node.GetTuple(insert_itr); if (tuple == nullptr) { @@ -256,7 +255,7 @@ bool InsertExecutor::DExecute() { values.push_back(new_tuple->GetValue(col_id)); } - LOG_INFO("Perform insert is being called"); +// LOG_INFO("Perform insert is being called"); transaction_manager.PerformInsert(current_txn, location, index_entry_ptr, reinterpret_cast(values.data()), values.size()); LOG_TRACE("Number of tuples in table after insert: %lu", diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index ad814836a51..83318dd2e78 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -32,7 +32,7 @@ void LogBuffer::WriteRecord(LogRecord &record) { switch (type) { case LogRecordType::TUPLE_INSERT: { - LOG_INFO("inserting tuple"); +// LOG_INFO("inserting tuple"); auto &manager = catalog::Manager::GetInstance(); auto tuple_pos = record.GetItemPointer(); auto tg = manager.GetTileGroup(tuple_pos.block).get(); @@ -55,7 +55,7 @@ void LogBuffer::WriteRecord(LogRecord &record) { break; } case LogRecordType::TUPLE_DELETE: { - LOG_INFO("Deleting tuple"); +// LOG_INFO("Deleting tuple"); auto &manager = catalog::Manager::GetInstance(); auto tuple_pos = record.GetItemPointer(); auto tg = manager.GetTileGroup(tuple_pos.block).get(); @@ -70,7 +70,7 @@ void LogBuffer::WriteRecord(LogRecord &record) { break; } case LogRecordType::TUPLE_UPDATE: { - LOG_INFO("Updating tuple"); +// LOG_INFO("Updating tuple"); auto &manager = catalog::Manager::GetInstance(); auto tuple_pos = record.GetItemPointer(); auto old_tuple_pos = record.GetOldItemPointer(); From c2a60fbf75bdb5e8010964a67b1f20ee5bff7540 Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Tue, 10 Apr 2018 16:01:12 -0400 Subject: [PATCH 19/81] refactored CommitQuery/AbortQuery callbacks --- src/common/internal_types.cpp | 3 + ...timestamp_ordering_transaction_manager.cpp | 60 +++++++++---------- src/include/traffic_cop/traffic_cop.h | 12 +++- src/traffic_cop/traffic_cop.cpp | 41 ++++--------- 4 files changed, 56 insertions(+), 60 deletions(-) diff --git a/src/common/internal_types.cpp b/src/common/internal_types.cpp index b409a02195b..ec28ffc2871 100644 --- a/src/common/internal_types.cpp +++ b/src/common/internal_types.cpp @@ -1988,6 +1988,9 @@ std::string ResultTypeToString(ResultType type) { case ResultType::QUEUING: { return ("QUEUING"); } + case ResultType::TO_ABORT: { + return ("TO_ABORT"); + } default: { throw ConversionException( StringUtil::Format("No string conversion for ResultType value '%d'", diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 7168067f3f0..da7d5081da5 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -787,7 +787,6 @@ void TimestampOrderingTransactionManager::PerformDelete( current_txn->ResetLogBuffer(); } - LOG_INFO("Perform Delete"); // Increment table delete op stats if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { @@ -842,8 +841,8 @@ void TimestampOrderingTransactionManager::PerformDelete( /* allocate a new buffer for the current transaction */ current_txn->ResetLogBuffer(); } - LOG_INFO("Perform Delete"); - // Increment table delete op stats + + // Increment table delete op stats if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { stats::BackendStatsContext::GetInstance()->IncrementTableDeletes( @@ -1013,26 +1012,15 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( // add to gc set. gc_set->operator[](tile_group_id)[tuple_slot] = GCVersionType::COMMIT_INS_DEL; - - // no log is needed for this case } } - // Increment # txns committed metric - if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != - StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTxnCommitted( - database_id); - } + ResultType result = current_txn->GetResult(); /* TODO(gandeevan): add logging switch */ if(task_callback != nullptr) { - ResultType result = current_txn->GetResult(); - - auto on_flush = [this, &result, task_callback, current_txn] (){ - current_txn->SetResult(result); - this->EndTransaction(current_txn); + auto on_flush = [this, result, task_callback] (){ task_callback(result); }; @@ -1049,10 +1037,20 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( current_txn->GetLogToken(), current_txn->GetLogBuffer()); - current_txn->SetResult(ResultType::QUEUING); + result = ResultType::QUEUING; + + } + + this->EndTransaction(current_txn); + + // Increment # txns committed metric + if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != + StatsType::INVALID) { + stats::BackendStatsContext::GetInstance()->IncrementTxnCommitted( + database_id); } - return current_txn->GetResult(); + return result; } ResultType TimestampOrderingTransactionManager::AbortTransaction( @@ -1230,21 +1228,13 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( current_txn->SetResult(ResultType::ABORTED); + ResultType result = current_txn->GetResult(); - // Increment # txns aborted metric - if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != - StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTxnAborted(database_id); - } /* TODO(gandeevan): add logging switch */ if(task_callback != nullptr) { - ResultType result = current_txn->GetResult(); - - auto on_flush = [this, &result, task_callback, current_txn] (){ - current_txn->SetResult(result); - this->EndTransaction(current_txn); + auto on_flush = [this, result, task_callback] (){ task_callback(result); }; @@ -1261,10 +1251,20 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( current_txn->GetLogToken(), current_txn->GetLogBuffer()); - current_txn->SetResult(ResultType::QUEUING); + + result = ResultType::QUEUING; + + } + + this->EndTransaction(current_txn); + + // Increment # txns aborted metric + if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != + StatsType::INVALID) { + stats::BackendStatsContext::GetInstance()->IncrementTxnAborted(database_id); } - return current_txn->GetResult(); + return result; } } // namespace storage diff --git a/src/include/traffic_cop/traffic_cop.h b/src/include/traffic_cop/traffic_cop.h index 464433be099..a5d64c29816 100644 --- a/src/include/traffic_cop/traffic_cop.h +++ b/src/include/traffic_cop/traffic_cop.h @@ -98,9 +98,9 @@ class TrafficCop { tcop_txn_state_.emplace(txn, ResultType::SUCCESS); } - ResultType CommitQueryHelper(std::function callback = nullptr); + ResultType CommitQueryHelper(); - ResultType AbortQueryHelper(std::function callback = nullptr); + ResultType AbortQueryHelper(); ResultType ExecuteStatementPlanGetResult(); @@ -147,6 +147,14 @@ class TrafficCop { default_database_name_ = std::move(default_database_name); } + inline std::function GetOnCompleteCallback(){ + auto on_complete = [this](ResultType result) { + this->p_status_.m_result = result; + this->task_callback_(this->task_callback_arg_); + }; + return on_complete; + } + // TODO: this member variable should be in statement_ after parser part // finished std::string query_; diff --git a/src/traffic_cop/traffic_cop.cpp b/src/traffic_cop/traffic_cop.cpp index 174b22a5efe..2f75f214a2c 100644 --- a/src/traffic_cop/traffic_cop.cpp +++ b/src/traffic_cop/traffic_cop.cpp @@ -93,7 +93,7 @@ ResultType TrafficCop::BeginQueryHelper(size_t thread_id) { return ResultType::SUCCESS; } -ResultType TrafficCop::CommitQueryHelper(std::function task_callback) { +ResultType TrafficCop::CommitQueryHelper() { // do nothing if we have no active txns if (tcop_txn_state_.empty()) return ResultType::NOOP; auto &curr_state = tcop_txn_state_.top(); @@ -107,14 +107,14 @@ ResultType TrafficCop::CommitQueryHelper(std::function task_ca if (curr_state.second != ResultType::ABORTED) { // txn committed - return txn_manager.CommitTransaction(txn, task_callback); + return txn_manager.CommitTransaction(txn, GetOnCompleteCallback()); } else { // otherwise, rollback - return txn_manager.AbortTransaction(txn, task_callback); + return txn_manager.AbortTransaction(txn, GetOnCompleteCallback()); } } -ResultType TrafficCop::AbortQueryHelper(std::function task_callback) { +ResultType TrafficCop::AbortQueryHelper() { // do nothing if we have no active txns if (tcop_txn_state_.empty()) return ResultType::NOOP; auto &curr_state = tcop_txn_state_.top(); @@ -123,7 +123,7 @@ ResultType TrafficCop::AbortQueryHelper(std::function task_cal if (curr_state.second != ResultType::ABORTED) { auto txn = curr_state.first; auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - return txn_manager.AbortTransaction(txn, task_callback); + return txn_manager.AbortTransaction(txn, GetOnCompleteCallback()); } else { delete curr_state.first; // otherwise, the txn has already been aborted @@ -183,6 +183,7 @@ executor::ExecutionResult TrafficCop::ExecuteHelper( auto on_complete = [&result, this, &txn](executor::ExecutionResult p_status, std::vector &&values) { this->p_status_ = p_status; + // TODO (Tianyi) I would make a decision on keeping one of p_status or // error_message in my next PR this->error_message_ = std::move(p_status.m_error_message); @@ -206,6 +207,7 @@ executor::ExecutionResult TrafficCop::ExecuteHelper( LOG_TRACE("Check Tcop_txn_state Size After ExecuteHelper %lu", tcop_txn_state_.size()); + return p_status_; } @@ -222,24 +224,17 @@ ResultType TrafficCop::ExecuteStatementPlanGetResult() { LOG_TRACE("About to commit/abort: single stmt: %d,txn_result: %s", single_statement_txn_, ResultTypeToString(txn_result).c_str()); - auto on_complete = [this](ResultType result) { - this->p_status_.m_result = result; - this->task_callback_(this->task_callback_arg_); - }; - switch (txn_result) { case ResultType::SUCCESS: // Commit single statement - return CommitQueryHelper(on_complete); - break; + return CommitQueryHelper(); case ResultType::FAILURE: default: // Abort - LOG_TRACE("Abort Transaction"); if (single_statement_txn_) { LOG_TRACE("Tcop_txn_state size: %lu", tcop_txn_state_.size()); - return AbortQueryHelper(on_complete); + return AbortQueryHelper(); } else { tcop_txn_state_.top().second = ResultType::ABORTED; p_status_.m_result = ResultType::ABORTED; @@ -595,12 +590,8 @@ ResultType TrafficCop::ExecuteStatement( this->is_queuing_ = true; auto &pool = threadpool::MonoQueuePool::GetInstance(); - auto on_commit = [this](ResultType result UNUSED_ATTRIBUTE) { - this->task_callback_(this->task_callback_arg_); - }; - - pool.SubmitTask([this, on_commit] { - ResultType result = this->CommitQueryHelper(on_commit); + pool.SubmitTask([this] { + ResultType result = this->CommitQueryHelper(); if(result!=ResultType::QUEUING) task_callback_(task_callback_arg_); @@ -614,18 +605,12 @@ ResultType TrafficCop::ExecuteStatement( this->is_queuing_ = true; auto &pool = threadpool::MonoQueuePool::GetInstance(); - auto on_abort = [this](ResultType result UNUSED_ATTRIBUTE) { - this->task_callback_(this->task_callback_arg_); - }; - - - pool.SubmitTask([this, on_abort] { - ResultType result = this->AbortQueryHelper(on_abort); + pool.SubmitTask([this] { + ResultType result = this->AbortQueryHelper(); if(result!=ResultType::QUEUING) { task_callback_(task_callback_arg_); } - }); return ResultType::QUEUING; From 8f53eda524727e9dae2c870a1931cf65dbf5e86b Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Tue, 10 Apr 2018 17:02:40 -0400 Subject: [PATCH 20/81] 1. added support for logging BEGIN_TXN 2. removed logging for read-only TXN. --- ...timestamp_ordering_transaction_manager.cpp | 253 ++++++++++-------- src/concurrency/transaction_manager.cpp | 12 + src/include/logging/log_record.h | 3 +- src/logging/log_buffer.cpp | 1 + 4 files changed, 155 insertions(+), 114 deletions(-) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index da7d5081da5..843417a553b 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -495,26 +495,31 @@ void TimestampOrderingTransactionManager::PerformInsert( // Write down the head pointer's address in tile group header tile_group_header->SetIndirection(tuple_id, index_entry_ptr); - if (values_buf != nullptr) { - logging::LogRecord record = - logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_INSERT, location, current_txn->GetEpochId(), - current_txn->GetTransactionId(), current_txn->GetCommitId()); - record.SetValuesArray(values_buf, values_size); - - current_txn->GetLogBuffer()->WriteRecord(record); - - if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { - - LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); - /* insert to the queue */ - threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( - current_txn->GetLogToken(), current_txn->GetLogBuffer()); - - /* allocate a new buffer for the current transaction */ - current_txn->ResetLogBuffer(); + /* TODO(graghura): remove placeholder, add logging switch */ + if(true) { + if (values_buf != nullptr) { + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_INSERT, location, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); + record.SetValuesArray(values_buf, values_size); + + current_txn->GetLogBuffer()->WriteRecord(record); + + if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { + + LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); + + /* allocate a new buffer for the current transaction */ + current_txn->ResetLogBuffer(); + } } } + + // Increment table insert op stats if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { @@ -599,30 +604,34 @@ void TimestampOrderingTransactionManager::PerformUpdate( // Add the old tuple into the update set current_txn->RecordUpdate(old_location); + /* TODO(graghura): remove placeholder, add logging switch */ + if(true) { + if (values_buf != nullptr) { + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_UPDATE, location, new_location, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); - if (values_buf != nullptr) { - logging::LogRecord record = - logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_UPDATE, location, new_location, current_txn->GetEpochId(), - current_txn->GetTransactionId(), current_txn->GetCommitId()); + record.SetOldItemPointer(location); + record.SetValuesArray(values_buf, values_size); + record.SetOffsetsArray(offsets); - record.SetOldItemPointer(location); - record.SetValuesArray(values_buf, values_size); - record.SetOffsetsArray(offsets); + current_txn->GetLogBuffer()->WriteRecord(record); - current_txn->GetLogBuffer()->WriteRecord(record); + if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { - if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); - /* insert to the queue */ - threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( - current_txn->GetLogToken(), current_txn->GetLogBuffer()); - - /* allocate a new buffer for the current transaction */ - current_txn->ResetLogBuffer(); + /* allocate a new buffer for the current transaction */ + current_txn->ResetLogBuffer(); + } } } + + // Increment table update op stats if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { @@ -651,37 +660,41 @@ void TimestampOrderingTransactionManager::PerformUpdate( PL_ASSERT(tile_group_header->GetBeginCommitId(tuple_id) == MAX_CID); PL_ASSERT(tile_group_header->GetEndCommitId(tuple_id) == MAX_CID); - // no need to add the older version into the update set. - // if there exists older version, then the older version must already - // been added to the update set. - // if there does not exist an older version, then it means that the - // transaction - // is updating a version that is installed by itself. - // in this case, nothing needs to be performed. - if (values_buf != nullptr) { - logging::LogRecord record = - logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_UPDATE, location, INVALID_ITEMPOINTER, - current_txn->GetEpochId(), current_txn->GetTransactionId(), - current_txn->GetCommitId()); + /* TODO(graghura): remove placeholder, add logging switch */ + if(true) { + if (values_buf != nullptr) { + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_UPDATE, location, INVALID_ITEMPOINTER, + current_txn->GetEpochId(), current_txn->GetTransactionId(), + current_txn->GetCommitId()); - record.SetOldItemPointer(location); - record.SetValuesArray(values_buf, values_size); - record.SetOffsetsArray(offsets); + record.SetOldItemPointer(location); + record.SetValuesArray(values_buf, values_size); + record.SetOffsetsArray(offsets); - current_txn->GetLogBuffer()->WriteRecord(record); + current_txn->GetLogBuffer()->WriteRecord(record); - if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { + if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { - /* insert to the queue */ - threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( - current_txn->GetLogToken(), current_txn->GetLogBuffer()); + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); - /* allocate a new buffer for the current transaction */ - current_txn->ResetLogBuffer(); + /* allocate a new buffer for the current transaction */ + current_txn->ResetLogBuffer(); + } } } + // no need to add the older version into the update set. + // if there exists older version, then the older version must already + // been added to the update set. + // if there does not exist an older version, then it means that the + // transaction + // is updating a version that is installed by itself. + // in this case, nothing needs to be performed. + // Increment table update op stats if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { @@ -768,23 +781,28 @@ void TimestampOrderingTransactionManager::PerformDelete( current_txn->RecordDelete(old_location); - logging::LogRecord record = - logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_DELETE, old_location, current_txn->GetEpochId(), - current_txn->GetTransactionId(), current_txn->GetCommitId()); + /* TODO(graghura): remove placeholder, add logging switch */ + if(true) { - current_txn->GetLogBuffer()->WriteRecord(record); + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_DELETE, old_location, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); - if(current_txn->GetLogBuffer()->HasThresholdExceeded()) { + current_txn->GetLogBuffer()->WriteRecord(record); - LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); + if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { - /* insert to the queue */ - threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( - current_txn->GetLogToken(), current_txn->GetLogBuffer()); + LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); + + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); + + /* allocate a new buffer for the current transaction */ + current_txn->ResetLogBuffer(); + } - /* allocate a new buffer for the current transaction */ - current_txn->ResetLogBuffer(); } // Increment table delete op stats @@ -823,23 +841,26 @@ void TimestampOrderingTransactionManager::PerformDelete( current_txn->RecordDelete(location); } - logging::LogRecord record = - logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_DELETE, location, current_txn->GetEpochId(), - current_txn->GetTransactionId(), current_txn->GetCommitId()); + /* TODO(graghura): remove placeholder, add logging switch */ + if(true) { + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_DELETE, location, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); - current_txn->GetLogBuffer()->WriteRecord(record); + current_txn->GetLogBuffer()->WriteRecord(record); - if(current_txn->GetLogBuffer()->HasThresholdExceeded()) { + if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { - LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); + LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); - /* insert to the queue */ - threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( - current_txn->GetLogToken(), current_txn->GetLogBuffer()); + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); - /* allocate a new buffer for the current transaction */ - current_txn->ResetLogBuffer(); + /* allocate a new buffer for the current transaction */ + current_txn->ResetLogBuffer(); + } } // Increment table delete op stats @@ -1017,28 +1038,33 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( ResultType result = current_txn->GetResult(); - /* TODO(gandeevan): add logging switch */ - if(task_callback != nullptr) { - auto on_flush = [this, result, task_callback] (){ - task_callback(result); - }; - logging::LogRecord record = - logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TRANSACTION_COMMIT, current_txn->GetEpochId(), - current_txn->GetTransactionId(), current_txn->GetCommitId()); + /* TODO(graghura): remove placeholder, add logging switch */ + if(true) { + // no need to log read-only transactions + if (!current_txn->IsReadOnly() && task_callback != nullptr) { - current_txn->GetLogBuffer()->WriteRecord(record); + auto on_flush = [this, result, task_callback]() { + task_callback(result); + }; - current_txn->GetLogBuffer()->SetLoggerCallback(on_flush); + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TRANSACTION_COMMIT, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); - /* insert to the queue */ - threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( - current_txn->GetLogToken(), current_txn->GetLogBuffer()); + current_txn->GetLogBuffer()->WriteRecord(record); - result = ResultType::QUEUING; + current_txn->GetLogBuffer()->SetLoggerCallback(on_flush); + + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); + result = ResultType::QUEUING; + + } } this->EndTransaction(current_txn); @@ -1230,30 +1256,31 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( ResultType result = current_txn->GetResult(); + /* TODO(graghura): remove placeholder, add logging switch */ + if(true) { + // no need to log read-only transactions + if (!current_txn->IsReadOnly() && task_callback != nullptr) { - /* TODO(gandeevan): add logging switch */ - if(task_callback != nullptr) { + auto on_flush = [this, result, task_callback]() { + task_callback(result); + }; - auto on_flush = [this, result, task_callback] (){ - task_callback(result); - }; + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TRANSACTION_ABORT, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); - logging::LogRecord record = - logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TRANSACTION_ABORT, current_txn->GetEpochId(), - current_txn->GetTransactionId(), current_txn->GetCommitId()); + current_txn->GetLogBuffer()->WriteRecord(record); - current_txn->GetLogBuffer()->WriteRecord(record); - - current_txn->GetLogBuffer()->SetLoggerCallback(on_flush); - - /* insert to the queue */ - threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( - current_txn->GetLogToken(), current_txn->GetLogBuffer()); + current_txn->GetLogBuffer()->SetLoggerCallback(on_flush); + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); - result = ResultType::QUEUING; + result = ResultType::QUEUING; + } } this->EndTransaction(current_txn); diff --git a/src/concurrency/transaction_manager.cpp b/src/concurrency/transaction_manager.cpp index 9d98dd63ac9..a781a885bbc 100644 --- a/src/concurrency/transaction_manager.cpp +++ b/src/concurrency/transaction_manager.cpp @@ -72,6 +72,18 @@ TransactionContext *TransactionManager::BeginTransaction( .StartTimer(); } + + // TODO(graghura) - add logging switch here: + // placeholder for logging switch + if(true) { + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TRANSACTION_BEGIN, txn->GetEpochId(), + txn->GetTransactionId(), txn->GetCommitId()); + + txn->GetLogBuffer()->WriteRecord(record); + } + txn->SetTimestamp(function::DateFunctions::Now()); return txn; diff --git a/src/include/logging/log_record.h b/src/include/logging/log_record.h index f35037cc717..f3a18407465 100644 --- a/src/include/logging/log_record.h +++ b/src/include/logging/log_record.h @@ -114,7 +114,8 @@ class LogRecordFactory { txn_id_t txn_id, cid_t current_cid) { PL_ASSERT(log_type == LogRecordType::TRANSACTION_COMMIT || - log_type == LogRecordType::TRANSACTION_ABORT); + log_type == LogRecordType::TRANSACTION_ABORT || + log_type == LogRecordType::TRANSACTION_BEGIN); return LogRecord(log_type, INVALID_ITEMPOINTER, INVALID_ITEMPOINTER, current_eid, txn_id, current_cid); diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index 74f8d5b1a3f..cfe726fdcb2 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -100,6 +100,7 @@ void LogBuffer::WriteRecord(LogRecord &record) { break; } + case LogRecordType::TRANSACTION_BEGIN: case LogRecordType::TRANSACTION_COMMIT: case LogRecordType::TRANSACTION_ABORT: { break; From 8ceff645056f453df622030ab31c1029f82734b0 Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Wed, 11 Apr 2018 15:15:11 -0400 Subject: [PATCH 21/81] fixed logger thread join bug --- src/include/threadpool/logger_queue_pool.h | 3 +- test/logging/log_buffer_pool_test.cpp | 100 ++++++------- test/logging/log_buffer_test.cpp | 156 ++++++++++----------- test/logging/log_record_test.cpp | 122 ++++++++-------- test/logging/new_checkpointing_test.cpp | 56 ++++---- test/logging/new_logging_test.cpp | 68 ++++----- 6 files changed, 253 insertions(+), 252 deletions(-) diff --git a/src/include/threadpool/logger_queue_pool.h b/src/include/threadpool/logger_queue_pool.h index 1533dd0f66f..73b8a496c31 100644 --- a/src/include/threadpool/logger_queue_pool.h +++ b/src/include/threadpool/logger_queue_pool.h @@ -64,11 +64,12 @@ class LoggerQueuePool { } void Shutdown() { + is_running_ = false; + for (auto &logger : loggers_) { logger.join(); } loggers_.clear(); - is_running_ = false; } void SubmitLogBuffer(logging::LogBuffer *buffer) { diff --git a/test/logging/log_buffer_pool_test.cpp b/test/logging/log_buffer_pool_test.cpp index a4c4bb56cc5..68a062ec4c1 100644 --- a/test/logging/log_buffer_pool_test.cpp +++ b/test/logging/log_buffer_pool_test.cpp @@ -1,50 +1,50 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_buffer_pool_test.cpp -// -// Identification: test/logging/log_buffer_pool_test.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "logging/log_buffer_pool.h" -#include "common/harness.h" - -namespace peloton { -namespace test { - -//===--------------------------------------------------------------------===// -// Log Buffer Pool Tests -//===--------------------------------------------------------------------===// - -class LogBufferPoolTests : public PelotonTest {}; - -TEST_F(LogBufferPoolTests, PoolTest) { - - logging::LogBufferPool log_buffer_pool(1); - - size_t thread_id = log_buffer_pool.GetThreadId(); - - EXPECT_EQ(thread_id, 1); - - std::unique_ptr log_buffer(new logging::LogBuffer(1, 1)); - - log_buffer_pool.GetBuffer(1); - - size_t slot_count = log_buffer_pool.GetEmptySlotCount(); - - EXPECT_EQ(slot_count, log_buffer_pool.GetMaxSlotCount() - 1); - - log_buffer_pool.PutBuffer(std::move(log_buffer)); - - slot_count = log_buffer_pool.GetEmptySlotCount(); - - EXPECT_EQ(slot_count, log_buffer_pool.GetMaxSlotCount()); - -} - -} -} +////===----------------------------------------------------------------------===// +//// +//// Peloton +//// +//// log_buffer_pool_test.cpp +//// +//// Identification: test/logging/log_buffer_pool_test.cpp +//// +//// Copyright (c) 2015-16, Carnegie Mellon University Database Group +//// +////===----------------------------------------------------------------------===// +// +//#include "logging/log_buffer_pool.h" +//#include "common/harness.h" +// +//namespace peloton { +//namespace test { +// +////===--------------------------------------------------------------------===// +//// Log Buffer Pool Tests +////===--------------------------------------------------------------------===// +// +//class LogBufferPoolTests : public PelotonTest {}; +// +//TEST_F(LogBufferPoolTests, PoolTest) { +// +// logging::LogBufferPool log_buffer_pool(1); +// +// size_t thread_id = log_buffer_pool.GetThreadId(); +// +// EXPECT_EQ(thread_id, 1); +// +// std::unique_ptr log_buffer(new logging::LogBuffer(1, 1)); +// +// log_buffer_pool.GetBuffer(1); +// +// size_t slot_count = log_buffer_pool.GetEmptySlotCount(); +// +// EXPECT_EQ(slot_count, log_buffer_pool.GetMaxSlotCount() - 1); +// +// log_buffer_pool.PutBuffer(std::move(log_buffer)); +// +// slot_count = log_buffer_pool.GetEmptySlotCount(); +// +// EXPECT_EQ(slot_count, log_buffer_pool.GetMaxSlotCount()); +// +//} +// +//} +//} diff --git a/test/logging/log_buffer_test.cpp b/test/logging/log_buffer_test.cpp index 82625bd7cf3..b3448d5d1b2 100644 --- a/test/logging/log_buffer_test.cpp +++ b/test/logging/log_buffer_test.cpp @@ -1,78 +1,78 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_buffer_test.cpp -// -// Identification: test/logging/log_buffer_test.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "logging/log_buffer.h" -#include "common/harness.h" - -namespace peloton { -namespace test { - -//===--------------------------------------------------------------------===// -// Log Buffer Tests -//===--------------------------------------------------------------------===// - -class LogBufferTests : public PelotonTest {}; - -TEST_F(LogBufferTests, LogBufferTest) { - - logging::LogBuffer log_buffer(1, 1); - - int eid = log_buffer.GetEpochId(); - - EXPECT_EQ(eid, 1); - - int thread_id = log_buffer.GetThreadId(); - - EXPECT_EQ(thread_id, 1); - - log_buffer.Reset(); - - eid = log_buffer.GetEpochId(); - - EXPECT_EQ(eid, INVALID_EID); - - bool rt = log_buffer.Empty(); - - EXPECT_TRUE(rt); - - char *data = log_buffer.GetData(); - - int num = 99; - - rt = log_buffer.WriteData((char*)(&num), sizeof(num)); - - EXPECT_TRUE(rt); - - int num2; - - PL_MEMCPY(&num2, data, sizeof(num)); - - EXPECT_EQ(num2, 99); - - size_t size = log_buffer.GetSize(); - - EXPECT_EQ(size, sizeof(num)); - - log_buffer.Reset(); - - rt = log_buffer.Empty(); - - EXPECT_TRUE(rt); - - size = log_buffer.GetSize(); - - EXPECT_EQ(size, 0); - -} - -} -} +////===----------------------------------------------------------------------===// +//// +//// Peloton +//// +//// log_buffer_test.cpp +//// +//// Identification: test/logging/log_buffer_test.cpp +//// +//// Copyright (c) 2015-16, Carnegie Mellon University Database Group +//// +////===----------------------------------------------------------------------===// +// +//#include "logging/log_buffer.h" +//#include "common/harness.h" +// +//namespace peloton { +//namespace test { +// +////===--------------------------------------------------------------------===// +//// Log Buffer Tests +////===--------------------------------------------------------------------===// +// +//class LogBufferTests : public PelotonTest {}; +// +//TEST_F(LogBufferTests, LogBufferTest) { +// +// logging::LogBuffer log_buffer(1, 1); +// +// int eid = log_buffer.GetEpochId(); +// +// EXPECT_EQ(eid, 1); +// +// int thread_id = log_buffer.GetThreadId(); +// +// EXPECT_EQ(thread_id, 1); +// +// log_buffer.Reset(); +// +// eid = log_buffer.GetEpochId(); +// +// EXPECT_EQ(eid, INVALID_EID); +// +// bool rt = log_buffer.Empty(); +// +// EXPECT_TRUE(rt); +// +// char *data = log_buffer.GetData(); +// +// int num = 99; +// +// rt = log_buffer.WriteData((char*)(&num), sizeof(num)); +// +// EXPECT_TRUE(rt); +// +// int num2; +// +// PL_MEMCPY(&num2, data, sizeof(num)); +// +// EXPECT_EQ(num2, 99); +// +// size_t size = log_buffer.GetSize(); +// +// EXPECT_EQ(size, sizeof(num)); +// +// log_buffer.Reset(); +// +// rt = log_buffer.Empty(); +// +// EXPECT_TRUE(rt); +// +// size = log_buffer.GetSize(); +// +// EXPECT_EQ(size, 0); +// +//} +// +//} +//} diff --git a/test/logging/log_record_test.cpp b/test/logging/log_record_test.cpp index 7d674a0e922..d6f21819cef 100644 --- a/test/logging/log_record_test.cpp +++ b/test/logging/log_record_test.cpp @@ -1,66 +1,66 @@ -//===----------------------------------------------------------------------===// +////===----------------------------------------------------------------------===// +//// +//// Peloton +//// +//// log_record_test.cpp +//// +//// Identification: test/logging/log_record_test.cpp +//// +//// Copyright (c) 2015-16, Carnegie Mellon University Database Group +//// +////===----------------------------------------------------------------------===// // -// Peloton +//#include "logging/log_record.h" +//#include "common/harness.h" // -// log_record_test.cpp +//namespace peloton { +//namespace test { // -// Identification: test/logging/log_record_test.cpp +////===--------------------------------------------------------------------===// +//// Log Buffer Tests +////===--------------------------------------------------------------------===// // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +//class LogRecordTests : public PelotonTest {}; // -//===----------------------------------------------------------------------===// - -#include "logging/log_record.h" -#include "common/harness.h" - -namespace peloton { -namespace test { - -//===--------------------------------------------------------------------===// -// Log Buffer Tests -//===--------------------------------------------------------------------===// - -class LogRecordTests : public PelotonTest {}; - -TEST_F(LogRecordTests, LogRecordTest) { - - std::vector tuple_type_list = { - LogRecordType::TUPLE_INSERT, - LogRecordType::TUPLE_DELETE, - LogRecordType::TUPLE_UPDATE - }; - - for (auto type : tuple_type_list) { - logging::LogRecord tuple_record = - logging::LogRecordFactory::CreateTupleRecord(type, ItemPointer(0, 0)); - - EXPECT_EQ(tuple_record.GetType(), type); - } - - std::vector txn_type_list = { - LogRecordType::TRANSACTION_BEGIN, - LogRecordType::TRANSACTION_COMMIT - }; - - for (auto type : txn_type_list) { - logging::LogRecord txn_record = - logging::LogRecordFactory::CreateTxnRecord(type, 50); - - EXPECT_EQ(txn_record.GetType(), type); - } - - std::vector epoch_type_list = { - LogRecordType::EPOCH_BEGIN, - LogRecordType::EPOCH_END - }; - - for (auto type : epoch_type_list) { - logging::LogRecord epoch_record = - logging::LogRecordFactory::CreateEpochRecord(type, 100); - - EXPECT_EQ(epoch_record.GetType(), type); - } -} - -} -} +//TEST_F(LogRecordTests, LogRecordTest) { +// +// std::vector tuple_type_list = { +// LogRecordType::TUPLE_INSERT, +// LogRecordType::TUPLE_DELETE, +// LogRecordType::TUPLE_UPDATE +// }; +// +// for (auto type : tuple_type_list) { +// logging::LogRecord tuple_record = +// logging::LogRecordFactory::CreateTupleRecord(type, ItemPointer(0, 0)); +// +// EXPECT_EQ(tuple_record.GetType(), type); +// } +// +// std::vector txn_type_list = { +// LogRecordType::TRANSACTION_BEGIN, +// LogRecordType::TRANSACTION_COMMIT +// }; +// +// for (auto type : txn_type_list) { +// logging::LogRecord txn_record = +// logging::LogRecordFactory::CreateTxnRecord(type, 50); +// +// EXPECT_EQ(txn_record.GetType(), type); +// } +// +// std::vector epoch_type_list = { +// LogRecordType::EPOCH_BEGIN, +// LogRecordType::EPOCH_END +// }; +// +// for (auto type : epoch_type_list) { +// logging::LogRecord epoch_record = +// logging::LogRecordFactory::CreateEpochRecord(type, 100); +// +// EXPECT_EQ(epoch_record.GetType(), type); +// } +//} +// +//} +//} diff --git a/test/logging/new_checkpointing_test.cpp b/test/logging/new_checkpointing_test.cpp index b16ea56471c..7cfcb42d50e 100644 --- a/test/logging/new_checkpointing_test.cpp +++ b/test/logging/new_checkpointing_test.cpp @@ -1,33 +1,33 @@ -//===----------------------------------------------------------------------===// +////===----------------------------------------------------------------------===// +//// +//// Peloton +//// +//// new_checkpointing_test.cpp +//// +//// Identification: test/logging/new_checkpointing_test.cpp +//// +//// Copyright (c) 2015-16, Carnegie Mellon University Database Group +//// +////===----------------------------------------------------------------------===// // -// Peloton +//#include "logging/checkpoint_manager_factory.h" +//#include "common/harness.h" // -// new_checkpointing_test.cpp +//namespace peloton { +//namespace test { // -// Identification: test/logging/new_checkpointing_test.cpp +////===--------------------------------------------------------------------===// +//// Checkpointing Tests +////===--------------------------------------------------------------------===// // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +//class NewCheckpointingTests : public PelotonTest {}; // -//===----------------------------------------------------------------------===// - -#include "logging/checkpoint_manager_factory.h" -#include "common/harness.h" - -namespace peloton { -namespace test { - -//===--------------------------------------------------------------------===// -// Checkpointing Tests -//===--------------------------------------------------------------------===// - -class NewCheckpointingTests : public PelotonTest {}; - -TEST_F(NewCheckpointingTests, MyTest) { - auto &checkpoint_manager = logging::CheckpointManagerFactory::GetInstance(); - checkpoint_manager.Reset(); - - EXPECT_TRUE(true); -} - -} -} +//TEST_F(NewCheckpointingTests, MyTest) { +// auto &checkpoint_manager = logging::CheckpointManagerFactory::GetInstance(); +// checkpoint_manager.Reset(); +// +// EXPECT_TRUE(true); +//} +// +//} +//} diff --git a/test/logging/new_logging_test.cpp b/test/logging/new_logging_test.cpp index 5973af7d475..c7d1700b503 100644 --- a/test/logging/new_logging_test.cpp +++ b/test/logging/new_logging_test.cpp @@ -1,34 +1,34 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// new_logging_test.cpp -// -// Identification: test/logging/new_logging_test.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "logging/log_manager_factory.h" -#include "common/harness.h" - -namespace peloton { -namespace test { - -//===--------------------------------------------------------------------===// -// Logging Tests -//===--------------------------------------------------------------------===// - -class NewLoggingTests : public PelotonTest {}; - -TEST_F(NewLoggingTests, MyTest) { - auto &log_manager = logging::LogManagerFactory::GetInstance(); - log_manager.Reset(); - - EXPECT_TRUE(true); - -} - -} -} +////===----------------------------------------------------------------------===// +//// +//// Peloton +//// +//// new_logging_test.cpp +//// +//// Identification: test/logging/new_logging_test.cpp +//// +//// Copyright (c) 2015-16, Carnegie Mellon University Database Group +//// +////===----------------------------------------------------------------------===// +// +//#include "logging/log_manager_factory.h" +//#include "common/harness.h" +// +//namespace peloton { +//namespace test { +// +////===--------------------------------------------------------------------===// +//// Logging Tests +////===--------------------------------------------------------------------===// +// +//class NewLoggingTests : public PelotonTest {}; +// +//TEST_F(NewLoggingTests, MyTest) { +// auto &log_manager = logging::LogManagerFactory::GetInstance(); +// log_manager.Reset(); +// +// EXPECT_TRUE(true); +// +//} +// +//} +//} From 913d3f4c864a4e7bfbab76983495d298195b1083 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Wed, 11 Apr 2018 22:55:08 -0400 Subject: [PATCH 22/81] Fix macro names --- src/codegen/inserter.cpp | 2 +- .../timestamp_ordering_transaction_manager.h | 20 ------------------- src/include/logging/log_record.h | 10 +++++----- src/logging/log_buffer.cpp | 2 +- src/logging/wal_logger.cpp | 2 +- 5 files changed, 8 insertions(+), 28 deletions(-) diff --git a/src/codegen/inserter.cpp b/src/codegen/inserter.cpp index 2ebadc12e37..fe4a6d14fba 100644 --- a/src/codegen/inserter.cpp +++ b/src/codegen/inserter.cpp @@ -48,7 +48,7 @@ peloton::type::AbstractPool *Inserter::GetPool() { <<<<<<< HEAD void Inserter::Insert(char *values_buf, uint32_t values_size) { - PL_ASSERT(table_ && executor_context_ && tile_); + PELOTON_ASSERT(table_ && executor_context_ && tile_); ======= void Inserter::Insert() { PELOTON_ASSERT(table_ && executor_context_ && tile_); diff --git a/src/include/concurrency/timestamp_ordering_transaction_manager.h b/src/include/concurrency/timestamp_ordering_transaction_manager.h index 4bbcacc7768..a48ee076489 100644 --- a/src/include/concurrency/timestamp_ordering_transaction_manager.h +++ b/src/include/concurrency/timestamp_ordering_transaction_manager.h @@ -229,31 +229,11 @@ class TimestampOrderingTransactionManager : public TransactionManager { virtual void PerformDelete(TransactionContext *const current_txn, const ItemPointer &location); -<<<<<<< HEAD virtual ResultType CommitTransaction(TransactionContext *const current_txn, std::function task_callback = nullptr); virtual ResultType AbortTransaction(TransactionContext *const current_txn, std::function task_callback = nullptr); -======= - /** - * @brief Commits a transaction. - * - * @param current_txn The current transaction - * - * @return The result type - */ - virtual ResultType CommitTransaction(TransactionContext *const current_txn); - - /** - * @brief Abort a transaction - * - * @param current_txn The current transaction - * - * @return The result type - */ - virtual ResultType AbortTransaction(TransactionContext *const current_txn); ->>>>>>> master private: diff --git a/src/include/logging/log_record.h b/src/include/logging/log_record.h index b31bc0fcf04..ce39199e771 100644 --- a/src/include/logging/log_record.h +++ b/src/include/logging/log_record.h @@ -104,7 +104,7 @@ class LogRecordFactory { static LogRecord CreateTupleRecord(const LogRecordType log_type, const ItemPointer &pos, eid_t current_eid, txn_id_t txn_id, cid_t current_cid) { - PL_ASSERT(log_type == LogRecordType::TUPLE_INSERT || + PELOTON_ASSERT(log_type == LogRecordType::TUPLE_INSERT || log_type == LogRecordType::TUPLE_DELETE); return LogRecord(log_type, INVALID_ITEMPOINTER, pos, current_eid, txn_id, current_cid); @@ -113,7 +113,7 @@ class LogRecordFactory { static LogRecord CreateTupleRecord(const LogRecordType log_type, eid_t current_eid, txn_id_t txn_id, cid_t current_cid) { - PL_ASSERT(log_type == LogRecordType::TRANSACTION_COMMIT || + PELOTON_ASSERT(log_type == LogRecordType::TRANSACTION_COMMIT || log_type == LogRecordType::TRANSACTION_ABORT || log_type == LogRecordType::TRANSACTION_BEGIN); @@ -125,20 +125,20 @@ class LogRecordFactory { const ItemPointer &pos, eid_t current_eid, txn_id_t txn_id, cid_t current_cid) { - PL_ASSERT(log_type == LogRecordType::TUPLE_UPDATE); + PELOTON_ASSERT(log_type == LogRecordType::TUPLE_UPDATE); return LogRecord(log_type, old_pos, pos, current_eid, txn_id, current_cid); } // // static LogRecord CreateTxnRecord(const LogRecordType log_type, const cid_t commit_id) { -// PL_ASSERT(log_type == LogRecordType::TRANSACTION_BEGIN || +// PELOTON_ASSERT(log_type == LogRecordType::TRANSACTION_BEGIN || // log_type == LogRecordType::TRANSACTION_COMMIT); // return LogRecord(log_type, INVALID_ITEMPOINTER, INVALID_EID, commit_id); // } // // static LogRecord CreateEpochRecord(const LogRecordType log_type, const eid_t epoch_id) { -// PL_ASSERT(log_type == LogRecordType::EPOCH_BEGIN || +// PELOTON_ASSERT(log_type == LogRecordType::EPOCH_BEGIN || // log_type == LogRecordType::EPOCH_END); // return LogRecord(log_type, INVALID_ITEMPOINTER, epoch_id, INVALID_CID); // } diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index 394df577db9..50881a4b8fb 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -107,7 +107,7 @@ void LogBuffer::WriteRecord(LogRecord &record) { } default: { - PL_ASSERT(false); + PELOTON_ASSERT(false); } } diff --git a/src/logging/wal_logger.cpp b/src/logging/wal_logger.cpp index bb8ed187003..5e81bb03846 100644 --- a/src/logging/wal_logger.cpp +++ b/src/logging/wal_logger.cpp @@ -59,7 +59,7 @@ void WalLogger::FlushToDisk(){ void WalLogger::PerformCompaction(LogBuffer *log_buffer){ if(nullptr==log_buffer) - PL_ASSERT(false); + PELOTON_ASSERT(false); disk_buffer_->GetCopySerializedOutput() .WriteBytes(log_buffer->GetData(), From a09bec06ec8d38bcf0450197e4188693287c15e2 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Wed, 11 Apr 2018 23:05:12 -0400 Subject: [PATCH 23/81] Fix conflicts left --- src/codegen/inserter.cpp | 5 ----- .../timestamp_ordering_transaction_manager.cpp | 18 +----------------- src/executor/update_executor.cpp | 5 ----- src/include/concurrency/transaction_context.h | 4 ---- 4 files changed, 1 insertion(+), 31 deletions(-) diff --git a/src/codegen/inserter.cpp b/src/codegen/inserter.cpp index fe4a6d14fba..e068edce22d 100644 --- a/src/codegen/inserter.cpp +++ b/src/codegen/inserter.cpp @@ -46,13 +46,8 @@ peloton::type::AbstractPool *Inserter::GetPool() { return tile_->GetPool(); } -<<<<<<< HEAD void Inserter::Insert(char *values_buf, uint32_t values_size) { PELOTON_ASSERT(table_ && executor_context_ && tile_); -======= -void Inserter::Insert() { - PELOTON_ASSERT(table_ && executor_context_ && tile_); ->>>>>>> master auto *txn = executor_context_->GetTransaction(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 3e9cf17eadf..9acef0b2ceb 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -437,13 +437,8 @@ bool TimestampOrderingTransactionManager::PerformRead( void TimestampOrderingTransactionManager::PerformInsert( TransactionContext *const current_txn, const ItemPointer &location, -<<<<<<< HEAD ItemPointer *index_entry_ptr, char *values_buf, uint32_t values_size) { - PL_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); -======= - ItemPointer *index_entry_ptr) { PELOTON_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); ->>>>>>> master oid_t tile_group_id = location.block; oid_t tuple_id = location.offset; @@ -505,14 +500,9 @@ void TimestampOrderingTransactionManager::PerformInsert( void TimestampOrderingTransactionManager::PerformUpdate( TransactionContext *const current_txn, const ItemPointer &location, -<<<<<<< HEAD const ItemPointer &new_location, char *values_buf, uint32_t values_size, TargetList *offsets) { - PL_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); -======= - const ItemPointer &new_location) { PELOTON_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); ->>>>>>> master ItemPointer old_location = location; @@ -621,18 +611,12 @@ void TimestampOrderingTransactionManager::PerformUpdate( } void TimestampOrderingTransactionManager::PerformUpdate( -<<<<<<< HEAD TransactionContext *const current_txn, const ItemPointer &location, char *values_buf, uint32_t values_size, TargetList *offsets) { - PL_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); -======= - TransactionContext *const current_txn UNUSED_ATTRIBUTE, - const ItemPointer &location) { PELOTON_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); ->>>>>>> master - + oid_t tile_group_id = location.block; UNUSED_ATTRIBUTE oid_t tuple_id = location.offset; diff --git a/src/executor/update_executor.cpp b/src/executor/update_executor.cpp index ab9366bc290..128425974c3 100644 --- a/src/executor/update_executor.cpp +++ b/src/executor/update_executor.cpp @@ -144,13 +144,8 @@ bool UpdateExecutor::PerformUpdatePrimaryKey( } } -<<<<<<< HEAD transaction_manager.PerformInsert(current_txn, location, index_entry_ptr, reinterpret_cast(values.data()), values.size()); - -======= - transaction_manager.PerformInsert(current_txn, location, index_entry_ptr); statement_write_set_.insert(location); ->>>>>>> master return true; } diff --git a/src/include/concurrency/transaction_context.h b/src/include/concurrency/transaction_context.h index 3b07e94ce2a..4b27297a98c 100644 --- a/src/include/concurrency/transaction_context.h +++ b/src/include/concurrency/transaction_context.h @@ -120,7 +120,6 @@ class TransactionContext : public Printable { */ inline uint64_t GetTimestamp() const { return timestamp_; } -<<<<<<< HEAD inline int GetLogToken() const { return log_token_; } inline logging::LogBuffer* GetLogBuffer() const { return log_buffer_; } @@ -132,8 +131,6 @@ class TransactionContext : public Printable { log_buffer_ = new logging::LogBuffer(logging::LogManager::GetInstance().GetTransactionBufferSize()); } - -======= /** * @brief Gets the query strings. * @@ -147,7 +144,6 @@ class TransactionContext : public Printable { * * @param[in] commit_id The commit identifier */ ->>>>>>> master inline void SetCommitId(const cid_t commit_id) { commit_id_ = commit_id; } /** From 7e607215d18a3ad618767479875edd6549fc4f00 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Wed, 11 Apr 2018 23:10:58 -0400 Subject: [PATCH 24/81] Remove redundant function duplicates --- src/include/concurrency/transaction_context.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/include/concurrency/transaction_context.h b/src/include/concurrency/transaction_context.h index 4b27297a98c..a116c80e20f 100644 --- a/src/include/concurrency/transaction_context.h +++ b/src/include/concurrency/transaction_context.h @@ -124,9 +124,6 @@ class TransactionContext : public Printable { inline logging::LogBuffer* GetLogBuffer() const { return log_buffer_; } - inline const std::vector& GetQueryStrings() const { - return query_strings_; } - inline void ResetLogBuffer() { log_buffer_ = new logging::LogBuffer(logging::LogManager::GetInstance().GetTransactionBufferSize()); } From 3014398a2ec51c08171fe1509c537a7f4489af4e Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Wed, 11 Apr 2018 23:20:26 -0400 Subject: [PATCH 25/81] Remove outdated head files and test files about logging --- src/logging/log_buffer_pool.cpp | 63 --- test/logging/buffer_pool_test.cpp | 182 -------- test/logging/checkpoint_test.cpp | 248 ----------- test/logging/log_buffer_pool_test.cpp | 50 --- test/logging/log_buffer_test.cpp | 78 ---- test/logging/log_record_test.cpp | 66 --- test/logging/logging_test.cpp | 384 ----------------- test/logging/logging_util_test.cpp | 35 -- test/logging/new_checkpointing_test.cpp | 33 -- test/logging/new_logging_test.cpp | 34 -- test/logging/recovery_test.cpp | 456 --------------------- test/logging/testing_logging_util.cpp | 357 ---------------- test/logging/write_behind_logging_test.cpp | 168 -------- 13 files changed, 2154 deletions(-) delete mode 100644 src/logging/log_buffer_pool.cpp delete mode 100644 test/logging/buffer_pool_test.cpp delete mode 100644 test/logging/checkpoint_test.cpp delete mode 100644 test/logging/log_buffer_pool_test.cpp delete mode 100644 test/logging/log_buffer_test.cpp delete mode 100644 test/logging/log_record_test.cpp delete mode 100644 test/logging/logging_test.cpp delete mode 100644 test/logging/logging_util_test.cpp delete mode 100644 test/logging/new_checkpointing_test.cpp delete mode 100644 test/logging/new_logging_test.cpp delete mode 100644 test/logging/recovery_test.cpp delete mode 100644 test/logging/testing_logging_util.cpp delete mode 100644 test/logging/write_behind_logging_test.cpp diff --git a/src/logging/log_buffer_pool.cpp b/src/logging/log_buffer_pool.cpp deleted file mode 100644 index 303f30dd55f..00000000000 --- a/src/logging/log_buffer_pool.cpp +++ /dev/null @@ -1,63 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_buffer_pool.cpp -// -// Identification: src/logging/log_buffer_pool.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#include "logging/log_buffer_pool.h" -#include "common/logger.h" - -namespace peloton { -namespace logging { - - // Acquire a log buffer from the buffer pool. - // This function will be blocked until there is an available buffer. - // Note that only the corresponding worker thread can call this function. - std::unique_ptr LogBufferPool::GetBuffer(size_t current_eid) { - size_t head_idx = head_ % buffer_queue_size_; - while (true) { - if (head_.load() < tail_.load() - 1) { - if (local_buffer_queue_[head_idx].get() == nullptr) { - // Not any buffer allocated now - local_buffer_queue_[head_idx].reset(new LogBuffer(thread_id_, current_eid)); - } - break; - } - - // sleep a while, and try to get a new buffer - _mm_pause(); - LOG_TRACE("Worker %d uses up its buffer", (int) thread_id_); - } - - head_.fetch_add(1, std::memory_order_relaxed); - local_buffer_queue_[head_idx]->eid_ = current_eid; - return std::move(local_buffer_queue_[head_idx]); - } - - // This function is called only by the corresponding logger. - void LogBufferPool::PutBuffer(std::unique_ptr buf) { - PELOTON_ASSERT(buf.get() != nullptr); - PELOTON_ASSERT(buf->GetThreadId() == thread_id_); - - size_t tail_idx = tail_ % buffer_queue_size_; - - // The buffer pool must not be full - PELOTON_ASSERT(tail_idx != head_ % buffer_queue_size_); - // The tail pos must be null - PELOTON_ASSERT(local_buffer_queue_[tail_idx].get() == nullptr); - // The returned buffer must be empty - PELOTON_ASSERT(buf->Empty() == true); - local_buffer_queue_[tail_idx].reset(buf.release()); - - tail_.fetch_add(1, std::memory_order_relaxed); - } - -} -} diff --git a/test/logging/buffer_pool_test.cpp b/test/logging/buffer_pool_test.cpp deleted file mode 100644 index 89de8e6cfb8..00000000000 --- a/test/logging/buffer_pool_test.cpp +++ /dev/null @@ -1,182 +0,0 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // buffer_pool_test.cpp -// // -// // Identification: test/logging/buffer_pool_test.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - - -// #include "common/harness.h" -// #include "logging/circular_buffer_pool.h" -// #include - -// #include "executor/testing_executor_util.h" -// #include "logging/testing_logging_util.h" - -// namespace peloton { -// namespace test { - -// //===--------------------------------------------------------------------===// -// // Buffer Pool Tests -// //===--------------------------------------------------------------------===// - -// class BufferPoolTests : public PelotonTest {}; - -// void EnqueueTest(logging::CircularBufferPool *buffer_pool, unsigned int count) { -// for (unsigned int i = 0; i < count; i++) { -// std::unique_ptr buf(new logging::LogBuffer(nullptr)); -// buf->SetSize(i); -// buffer_pool->Put(std::move(buf)); -// } -// } - -// void DequeueTest(logging::CircularBufferPool *buffer_pool, unsigned int count) { -// for (unsigned int i = 0; i < count; i++) { -// auto buf = std::move(buffer_pool->Get()); -// PELOTON_ASSERT(buf); -// EXPECT_EQ(buf->GetSize(), i); -// } -// } - -// void BackendThread(logging::WriteAheadBackendLogger *logger, -// unsigned int count) { -// for (unsigned int i = 1; i <= count; i++) { -// logging::TransactionRecord begin_record(LOGRECORD_TYPE_TRANSACTION_COMMIT, -// i); -// logger->Log(&begin_record); -// } -// } - -// void FrontendThread(logging::WriteAheadFrontendLogger *logger, -// unsigned int count) { -// srand(time(NULL)); -// while (true) { -// for (int i = 0; i < 10; i++) { -// logger->CollectLogRecordsFromBackendLoggers(); -// } -// logger->FlushLogRecords(); -// auto cid = logger->GetMaxFlushedCommitId(); -// if (cid == count) { -// break; -// } -// int rand_num = rand() % 5 + 1; -// std::chrono::milliseconds sleep_time(rand_num); -// std::this_thread::sleep_for(sleep_time); -// } -// } - -// TEST_F(BufferPoolTests, BufferPoolBasicTest) { -// logging::CircularBufferPool buffer_pool0; -// EnqueueTest(&buffer_pool0, 5); -// EXPECT_EQ(buffer_pool0.GetSize(), 5); - -// DequeueTest(&buffer_pool0, 5); -// EXPECT_EQ(buffer_pool0.GetSize(), 0); - -// EnqueueTest(&buffer_pool0, BUFFER_POOL_SIZE); -// EXPECT_EQ(buffer_pool0.GetSize(), BUFFER_POOL_SIZE); - -// for (int i = 0; i < 10; i++) { -// logging::CircularBufferPool buffer_pool1; -// std::thread enqueue_thread(EnqueueTest, &buffer_pool1, BUFFER_POOL_SIZE); -// std::thread dequeue_thread(DequeueTest, &buffer_pool1, BUFFER_POOL_SIZE); -// enqueue_thread.join(); -// dequeue_thread.join(); -// } -// } - -// TEST_F(BufferPoolTests, LogBufferBasicTest) { -// size_t tile_group_size = TESTS_TUPLES_PER_TILEGROUP; -// size_t table_tile_group_count = 3; - -// std::unique_ptr recovery_table( -// TestingExecutorUtil::CreateTable(tile_group_size)); - -// // prepare tuples -// auto mutate = true; -// auto random = false; -// int num_rows = tile_group_size * table_tile_group_count; -// std::vector> tuples = -// TestingLoggingUtil::BuildTuples(recovery_table.get(), num_rows, mutate, -// random); -// std::vector records = -// TestingLoggingUtil::BuildTupleRecords(tuples, tile_group_size, -// table_tile_group_count); -// logging::LogBuffer log_buffer(0); -// size_t total_length = 0; -// for (auto record : records) { -// PELOTON_ASSERT(record.GetTuple()->GetSchema()); -// CopySerializeOutput output_buffer; -// record.Serialize(output_buffer); -// size_t len = record.GetMessageLength(); -// total_length += len; -// log_buffer.WriteRecord(&record); -// } -// EXPECT_EQ(log_buffer.GetSize(), total_length); -// } - -// TEST_F(BufferPoolTests, LargeTupleRecordTest) { -// auto testing_pool = TestingHarness::GetInstance().GetTestingPool(); - -// std::vector columns; - -// catalog::Column column1(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), -// "A", true); -// catalog::Column column2(type::TypeId::VARCHAR, 1024 * 1024 * 20, "B", false); - -// columns.push_back(column1); -// columns.push_back(column2); - -// std::unique_ptr key_schema(new catalog::Schema(columns)); -// std::unique_ptr tuple( -// new storage::Tuple(key_schema.get(), true)); -// tuple->SetValue(0, type::ValueFactory::GetIntegerValue(1), testing_pool); -// tuple->SetValue(1, type::ValueFactory::GetVarcharValue( -// std::string(1024 * 1024 * 20, 'e').c_str()), -// testing_pool); - -// logging::TupleRecord record(LOGRECORD_TYPE_WAL_TUPLE_INSERT, INITIAL_TXN_ID, -// INVALID_OID, INVALID_ITEMPOINTER, -// INVALID_ITEMPOINTER, tuple.get(), DEFAULT_DB_ID); -// record.SetTuple(tuple.get()); - -// logging::LogBuffer log_buffer(0); -// size_t total_length = 0; -// PELOTON_ASSERT(record.GetTuple()->GetSchema()); -// CopySerializeOutput output_buffer; -// record.Serialize(output_buffer); -// size_t len = record.GetMessageLength(); -// total_length += len; -// auto success = log_buffer.WriteRecord(&record); -// EXPECT_EQ(log_buffer.GetSize(), total_length); -// EXPECT_TRUE(success); -// } - -// TEST_F(BufferPoolTests, BufferPoolConcurrentTest) { -// unsigned int txn_count = 9999; - -// auto &log_manager = logging::LogManager::GetInstance(); -// logging::LogManager::GetInstance().Configure(LoggingType::NVM_WAL, true); -// log_manager.SetLoggingStatus(LoggingStatusType::LOGGING); -// log_manager.InitFrontendLoggers(); - -// logging::WriteAheadFrontendLogger *frontend_logger = -// reinterpret_cast( -// log_manager.GetFrontendLogger(0)); -// logging::WriteAheadBackendLogger *backend_logger = -// reinterpret_cast( -// log_manager.GetBackendLogger()); - -// std::thread backend_thread(BackendThread, backend_logger, txn_count); -// std::thread frontend_thread(FrontendThread, frontend_logger, txn_count); -// backend_thread.join(); -// frontend_thread.join(); -// } - -// } // namespace test -// } // namespace peloton diff --git a/test/logging/checkpoint_test.cpp b/test/logging/checkpoint_test.cpp deleted file mode 100644 index 656ec7cfaa5..00000000000 --- a/test/logging/checkpoint_test.cpp +++ /dev/null @@ -1,248 +0,0 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // checkpoint_test.cpp -// // -// // Identification: test/logging/checkpoint_test.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - -// #include - -// #include "common/harness.h" -// #include "catalog/catalog.h" -// #include "logging/checkpoint.h" - -// #include "logging/testing_logging_util.h" -// #include "logging/logging_util.h" -// #include "logging/loggers/wal_backend_logger.h" -// #include "logging/checkpoint/simple_checkpoint.h" -// #include "logging/checkpoint_manager.h" -// #include "storage/database.h" - -// #include "concurrency/transaction_manager_factory.h" -// #include "executor/logical_tile_factory.h" -// #include "index/index.h" - -// #include "executor/mock_executor.h" - -// #define DEFAULT_RECOVERY_CID 15 - -// using ::testing::NotNull; -// using ::testing::Return; -// using ::testing::InSequence; - -// namespace peloton { -// namespace test { - -// //===--------------------------------------------------------------------===// -// // Checkpoint Tests -// //===--------------------------------------------------------------------===// - -// class CheckpointTests : public PelotonTest {}; - -// oid_t GetTotalTupleCount(size_t table_tile_group_count, cid_t next_cid) { -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - -// txn_manager.SetNextCid(next_cid); - -// auto txn = txn_manager.BeginTransaction(); - -// auto &catalog_manager = catalog::Manager::GetInstance(); -// oid_t total_tuple_count = 0; -// for (size_t tile_group_id = 1; tile_group_id <= table_tile_group_count; -// tile_group_id++) { -// auto tile_group = catalog_manager.GetTileGroup(tile_group_id); -// total_tuple_count += tile_group->GetActiveTupleCount(); -// } -// txn_manager.CommitTransaction(txn); -// return total_tuple_count; -// } - -// // TEST_F(CheckpointTests, CheckpointIntegrationTest) { -// // logging::LoggingUtil::RemoveDirectory("pl_checkpoint", false); -// // auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// // auto txn = txn_manager.BeginTransaction(); - -// // // Create a table and wrap it in logical tile -// // size_t tile_group_size = TESTS_TUPLES_PER_TILEGROUP; -// // size_t table_tile_group_count = 3; - -// // oid_t default_table_oid = 13; -// // // table has 3 tile groups -// // storage::DataTable *target_table = -// // TestingExecutorUtil::CreateTable(tile_group_size, true, default_table_oid); -// // TestingExecutorUtil::PopulateTable(target_table, -// // tile_group_size * table_tile_group_count, -// // false, false, false, txn); -// // txn_manager.CommitTransaction(txn); - -// // // add table to catalog -// // auto catalog = catalog::Catalog::GetInstance(); -// // storage::Database *db(new storage::Database(DEFAULT_DB_ID)); -// // db->AddTable(target_table); -// // catalog->AddDatabase(db); - -// // // create checkpoint -// // auto &checkpoint_manager = logging::CheckpointManager::GetInstance(); -// // auto &log_manager = logging::LogManager::GetInstance(); -// // log_manager.SetGlobalMaxFlushedCommitId(txn_manager.GetNextCommitId()); -// // checkpoint_manager.Configure(CheckpointType::NORMAL, false, 1); -// // checkpoint_manager.DestroyCheckpointers(); -// // checkpoint_manager.InitCheckpointers(); -// // auto checkpointer = checkpoint_manager.GetCheckpointer(0); - -// // checkpointer->DoCheckpoint(); - -// // auto most_recent_checkpoint_cid = checkpointer->GetMostRecentCheckpointCid(); -// // EXPECT_TRUE(most_recent_checkpoint_cid != INVALID_CID); - -// // // destroy and restart -// // checkpoint_manager.DestroyCheckpointers(); -// // checkpoint_manager.InitCheckpointers(); - -// // // recovery from checkpoint -// // log_manager.PrepareRecovery(); -// // auto recovery_checkpointer = checkpoint_manager.GetCheckpointer(0); -// // recovery_checkpointer->DoRecovery(); - -// // EXPECT_EQ(db->GetTableCount(), 1); -// // EXPECT_EQ(db->GetTable(0)->GetTupleCount(), -// // tile_group_size * table_tile_group_count); -// // catalog->DropDatabaseWithOid(db->GetOid()); -// // logging::LoggingUtil::RemoveDirectory("pl_checkpoint", false); -// // } - -// // TEST_F(CheckpointTests, CheckpointScanTest) { -// // logging::LoggingUtil::RemoveDirectory("pl_checkpoint", false); - -// // auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// // auto txn = txn_manager.BeginTransaction(); - -// // // Create a table and wrap it in logical tile -// // size_t tile_group_size = TESTS_TUPLES_PER_TILEGROUP; -// // size_t table_tile_group_count = 3; - -// // // table has 3 tile groups -// // std::unique_ptr target_table( -// // TestingExecutorUtil::CreateTable(tile_group_size)); -// // TestingExecutorUtil::PopulateTable(target_table.get(), -// // tile_group_size * table_tile_group_count, -// // false, false, false, txn); -// // txn_manager.CommitTransaction(txn); - -// // auto cid = txn_manager.GetNextCommitId() - 1; -// // LOG_INFO("Scan with cid = %d. GetExpiredEpochIdCid = %d", (int)cid, -// // (int)txn_manager.GetExpiredCid()); -// // auto schema = target_table->GetSchema(); -// // std::vector column_ids; -// // column_ids.resize(schema->GetColumnCount()); -// // std::iota(column_ids.begin(), column_ids.end(), 0); - -// // // create checkpoint -// // auto &checkpoint_manager = logging::CheckpointManager::GetInstance(); -// // checkpoint_manager.Configure(CheckpointType::NORMAL, true, 1); -// // checkpoint_manager.DestroyCheckpointers(); -// // checkpoint_manager.InitCheckpointers(); -// // auto checkpointer = checkpoint_manager.GetCheckpointer(0); - -// // auto simple_checkpointer = -// // reinterpret_cast(checkpointer); - -// // simple_checkpointer->SetLogger(new logging::WriteAheadBackendLogger()); -// // simple_checkpointer->SetStartCommitId(cid); -// // simple_checkpointer->Scan(target_table.get(), DEFAULT_DB_ID); - -// // // verify results -// // auto records = simple_checkpointer->GetRecords(); -// // EXPECT_EQ(records.size(), -// // TESTS_TUPLES_PER_TILEGROUP * table_tile_group_count); -// // for (unsigned int i = 0; i < records.size(); i++) { -// // EXPECT_EQ(records[i]->GetType(), LOGRECORD_TYPE_WAL_TUPLE_INSERT); -// // } -// // } - -// TEST_F(CheckpointTests, CheckpointRecoveryTest) { -// logging::LoggingUtil::RemoveDirectory("pl_checkpoint", false); - -// size_t tile_group_size = TESTS_TUPLES_PER_TILEGROUP; -// size_t table_tile_group_count = 3; - -// std::unique_ptr recovery_table( -// TestingExecutorUtil::CreateTable(tile_group_size)); - -// // prepare tuples -// auto mutate = true; -// auto random = false; -// int num_rows = tile_group_size * table_tile_group_count; -// std::vector> tuples = -// TestingLoggingUtil::BuildTuples(recovery_table.get(), num_rows, mutate, -// random); -// std::vector records = -// TestingLoggingUtil::BuildTupleRecords(tuples, tile_group_size, -// table_tile_group_count); - -// // recovery tuples from checkpoint -// logging::SimpleCheckpoint simple_checkpoint(true); -// for (auto record : records) { -// auto tuple = record.GetTuple(); -// auto target_location = record.GetInsertLocation(); -// // recovery checkpoint from these records -// simple_checkpoint.RecoverTuple(tuple, recovery_table.get(), target_location, -// DEFAULT_RECOVERY_CID); -// } - -// // recovered tuples are visible from DEFAULT_RECOVERY_CID -// auto total_tuple_count = -// GetTotalTupleCount(table_tile_group_count, DEFAULT_RECOVERY_CID); -// EXPECT_EQ(total_tuple_count, tile_group_size * table_tile_group_count); - -// // Clean up -// for (auto &tuple : tuples) { -// tuple.reset(); -// } -// } - -// TEST_F(CheckpointTests, CheckpointModeTransitionTest) { -// logging::LoggingUtil::RemoveDirectory("pl_checkpoint", false); - -// auto &log_manager = logging::LogManager::GetInstance(); -// auto &checkpoint_manager = logging::CheckpointManager::GetInstance(); -// checkpoint_manager.DestroyCheckpointers(); - -// checkpoint_manager.Configure(CheckpointType::NORMAL, true, 1); - -// // launch checkpoint thread, wait for standby mode -// auto thread = std::thread(&logging::CheckpointManager::StartStandbyMode, -// &checkpoint_manager); - -// checkpoint_manager.WaitForModeTransition(peloton::CheckpointStatus::STANDBY, -// true); - -// // Clean up table tile state before recovery from checkpoint -// log_manager.PrepareRecovery(); - -// // Do any recovery -// checkpoint_manager.StartRecoveryMode(); - -// // Wait for standby mode -// checkpoint_manager.WaitForModeTransition(CheckpointStatus::DONE_RECOVERY, -// true); - -// // Now, enter CHECKPOINTING mode -// checkpoint_manager.SetCheckpointStatus(CheckpointStatus::CHECKPOINTING); -// auto checkpointer = checkpoint_manager.GetCheckpointer(0); -// while (checkpointer->GetCheckpointStatus() != -// CheckpointStatus::CHECKPOINTING) { -// std::chrono::milliseconds sleep_time(10); -// std::this_thread::sleep_for(sleep_time); -// } -// checkpoint_manager.SetCheckpointStatus(CheckpointStatus::INVALID); -// thread.join(); -// } - -// } // namespace test -// } // namespace peloton diff --git a/test/logging/log_buffer_pool_test.cpp b/test/logging/log_buffer_pool_test.cpp deleted file mode 100644 index 68a062ec4c1..00000000000 --- a/test/logging/log_buffer_pool_test.cpp +++ /dev/null @@ -1,50 +0,0 @@ -////===----------------------------------------------------------------------===// -//// -//// Peloton -//// -//// log_buffer_pool_test.cpp -//// -//// Identification: test/logging/log_buffer_pool_test.cpp -//// -//// Copyright (c) 2015-16, Carnegie Mellon University Database Group -//// -////===----------------------------------------------------------------------===// -// -//#include "logging/log_buffer_pool.h" -//#include "common/harness.h" -// -//namespace peloton { -//namespace test { -// -////===--------------------------------------------------------------------===// -//// Log Buffer Pool Tests -////===--------------------------------------------------------------------===// -// -//class LogBufferPoolTests : public PelotonTest {}; -// -//TEST_F(LogBufferPoolTests, PoolTest) { -// -// logging::LogBufferPool log_buffer_pool(1); -// -// size_t thread_id = log_buffer_pool.GetThreadId(); -// -// EXPECT_EQ(thread_id, 1); -// -// std::unique_ptr log_buffer(new logging::LogBuffer(1, 1)); -// -// log_buffer_pool.GetBuffer(1); -// -// size_t slot_count = log_buffer_pool.GetEmptySlotCount(); -// -// EXPECT_EQ(slot_count, log_buffer_pool.GetMaxSlotCount() - 1); -// -// log_buffer_pool.PutBuffer(std::move(log_buffer)); -// -// slot_count = log_buffer_pool.GetEmptySlotCount(); -// -// EXPECT_EQ(slot_count, log_buffer_pool.GetMaxSlotCount()); -// -//} -// -//} -//} diff --git a/test/logging/log_buffer_test.cpp b/test/logging/log_buffer_test.cpp deleted file mode 100644 index 29b8c7a5a3d..00000000000 --- a/test/logging/log_buffer_test.cpp +++ /dev/null @@ -1,78 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_buffer_test.cpp -// -// Identification: test/logging/log_buffer_test.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "logging/log_buffer.h" -#include "common/harness.h" - -namespace peloton { -namespace test { - -//===--------------------------------------------------------------------===// -// Log Buffer Tests -//===--------------------------------------------------------------------===// - -class LogBufferTests : public PelotonTest {}; - -TEST_F(LogBufferTests, LogBufferTest) { - - logging::LogBuffer log_buffer(1, 1); - - int eid = log_buffer.GetEpochId(); - - EXPECT_EQ(eid, 1); - - int thread_id = log_buffer.GetThreadId(); - - EXPECT_EQ(thread_id, 1); - - log_buffer.Reset(); - - eid = log_buffer.GetEpochId(); - - EXPECT_EQ(eid, INVALID_EID); - - bool rt = log_buffer.Empty(); - - EXPECT_TRUE(rt); - - char *data = log_buffer.GetData(); - - int num = 99; - - rt = log_buffer.WriteData((char*)(&num), sizeof(num)); - - EXPECT_TRUE(rt); - - int num2; - - PELOTON_MEMCPY(&num2, data, sizeof(num)); - - EXPECT_EQ(num2, 99); - - size_t size = log_buffer.GetSize(); - - EXPECT_EQ(size, sizeof(num)); - - log_buffer.Reset(); - - rt = log_buffer.Empty(); - - EXPECT_TRUE(rt); - - size = log_buffer.GetSize(); - - EXPECT_EQ(size, 0); - -} - -} -} diff --git a/test/logging/log_record_test.cpp b/test/logging/log_record_test.cpp deleted file mode 100644 index d6f21819cef..00000000000 --- a/test/logging/log_record_test.cpp +++ /dev/null @@ -1,66 +0,0 @@ -////===----------------------------------------------------------------------===// -//// -//// Peloton -//// -//// log_record_test.cpp -//// -//// Identification: test/logging/log_record_test.cpp -//// -//// Copyright (c) 2015-16, Carnegie Mellon University Database Group -//// -////===----------------------------------------------------------------------===// -// -//#include "logging/log_record.h" -//#include "common/harness.h" -// -//namespace peloton { -//namespace test { -// -////===--------------------------------------------------------------------===// -//// Log Buffer Tests -////===--------------------------------------------------------------------===// -// -//class LogRecordTests : public PelotonTest {}; -// -//TEST_F(LogRecordTests, LogRecordTest) { -// -// std::vector tuple_type_list = { -// LogRecordType::TUPLE_INSERT, -// LogRecordType::TUPLE_DELETE, -// LogRecordType::TUPLE_UPDATE -// }; -// -// for (auto type : tuple_type_list) { -// logging::LogRecord tuple_record = -// logging::LogRecordFactory::CreateTupleRecord(type, ItemPointer(0, 0)); -// -// EXPECT_EQ(tuple_record.GetType(), type); -// } -// -// std::vector txn_type_list = { -// LogRecordType::TRANSACTION_BEGIN, -// LogRecordType::TRANSACTION_COMMIT -// }; -// -// for (auto type : txn_type_list) { -// logging::LogRecord txn_record = -// logging::LogRecordFactory::CreateTxnRecord(type, 50); -// -// EXPECT_EQ(txn_record.GetType(), type); -// } -// -// std::vector epoch_type_list = { -// LogRecordType::EPOCH_BEGIN, -// LogRecordType::EPOCH_END -// }; -// -// for (auto type : epoch_type_list) { -// logging::LogRecord epoch_record = -// logging::LogRecordFactory::CreateEpochRecord(type, 100); -// -// EXPECT_EQ(epoch_record.GetType(), type); -// } -//} -// -//} -//} diff --git a/test/logging/logging_test.cpp b/test/logging/logging_test.cpp deleted file mode 100644 index e4bee1e32b7..00000000000 --- a/test/logging/logging_test.cpp +++ /dev/null @@ -1,384 +0,0 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // logging_test.cpp -// // -// // Identification: test/logging/logging_test.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - -// #include "catalog/catalog.h" -// #include "common/harness.h" -// #include "executor/testing_executor_util.h" -// #include "logging/testing_logging_util.h" - -// #include "concurrency/transaction_manager_factory.h" -// #include "executor/logical_tile_factory.h" -// #include "logging/loggers/wal_frontend_logger.h" -// #include "logging/logging_util.h" -// #include "storage/data_table.h" -// #include "storage/database.h" -// #include "storage/table_factory.h" -// #include "storage/tile.h" - -// #include "executor/mock_executor.h" - -// using ::testing::NotNull; -// using ::testing::Return; -// using ::testing::InSequence; - -// extern peloton::LoggingType peloton_logging_mode; - -// namespace peloton { -// namespace test { - -// //===--------------------------------------------------------------------===// -// // Logging Tests -// //===--------------------------------------------------------------------===// - -// class LoggingTests : public PelotonTest {}; - -// TEST_F(LoggingTests, BasicLoggingTest) { -// std::unique_ptr table( -// TestingExecutorUtil::CreateTable(1)); - -// auto &log_manager = logging::LogManager::GetInstance(); - -// LoggingScheduler scheduler(2, 1, &log_manager, table.get()); - -// scheduler.Init(); -// // The first txn to commit starts with cid 2 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(2); -// scheduler.BackendLogger(0, 0).Insert(2); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(3); - -// scheduler.BackendLogger(0, 0).Commit(2); -// scheduler.BackendLogger(0, 1).Insert(3); -// scheduler.BackendLogger(0, 1).Commit(3); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 0).Done(1); -// scheduler.Run(); - -// auto results = scheduler.frontend_threads[0].results; -// EXPECT_EQ(3, results[0]); -// scheduler.Cleanup(); -// } - -// TEST_F(LoggingTests, AllCommittedTest) { -// std::unique_ptr table( -// TestingExecutorUtil::CreateTable(1)); - -// auto &log_manager = logging::LogManager::GetInstance(); - -// LoggingScheduler scheduler(2, 1, &log_manager, table.get()); - -// scheduler.Init(); -// // Logger 0 is always the front end logger -// // The first txn to commit starts with cid 2 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(2); -// scheduler.BackendLogger(0, 0).Insert(2); -// scheduler.BackendLogger(0, 0).Commit(2); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(3); -// scheduler.BackendLogger(0, 1).Insert(3); -// scheduler.BackendLogger(0, 1).Commit(3); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 0).Done(1); -// scheduler.BackendLogger(0, 1).Done(1); -// scheduler.Run(); - -// auto results = scheduler.frontend_threads[0].results; -// EXPECT_EQ(3, results[0]); -// scheduler.Cleanup(); -// } - -// TEST_F(LoggingTests, LaggardTest) { -// std::unique_ptr table( -// TestingExecutorUtil::CreateTable(1)); - -// auto &log_manager = logging::LogManager::GetInstance(); - -// LoggingScheduler scheduler(2, 1, &log_manager, table.get()); - -// scheduler.Init(); -// // Logger 0 is always the front end logger -// // The first txn to commit starts with cid 2 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(2); -// scheduler.BackendLogger(0, 0).Insert(2); -// scheduler.BackendLogger(0, 0).Commit(2); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(3); -// scheduler.BackendLogger(0, 1).Insert(3); -// scheduler.BackendLogger(0, 1).Commit(3); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// // at this point everyone should be updated to 3 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(4); -// scheduler.BackendLogger(0, 0).Insert(4); -// scheduler.BackendLogger(0, 0).Commit(4); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 0).Done(1); -// scheduler.BackendLogger(0, 1).Done(1); - -// scheduler.Run(); - -// auto results = scheduler.frontend_threads[0].results; -// EXPECT_EQ(3, results[0]); -// EXPECT_EQ(3, results[1]); -// scheduler.Cleanup(); -// } - -// TEST_F(LoggingTests, FastLoggerTest) { -// std::unique_ptr table( -// TestingExecutorUtil::CreateTable(1)); - -// auto &log_manager = logging::LogManager::GetInstance(); - -// LoggingScheduler scheduler(2, 1, &log_manager, table.get()); - -// scheduler.Init(); -// // Logger 0 is always the front end logger -// // The first txn to commit starts with cid 2 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(2); -// scheduler.BackendLogger(0, 0).Insert(2); -// scheduler.BackendLogger(0, 0).Commit(2); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(3); -// scheduler.BackendLogger(0, 1).Insert(3); -// scheduler.BackendLogger(0, 1).Commit(3); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 0).Done(1); -// scheduler.BackendLogger(0, 1).Done(1); -// // at this point everyone should be updated to 3 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(4); -// scheduler.BackendLogger(0, 0).Insert(4); -// scheduler.BackendLogger(0, 0).Commit(4); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Insert(5); -// scheduler.BackendLogger(0, 1).Commit(5); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 0).Done(1); -// scheduler.BackendLogger(0, 1).Done(1); - -// scheduler.Run(); - -// auto results = scheduler.frontend_threads[0].results; -// EXPECT_EQ(3, results[0]); -// EXPECT_EQ(3, results[1]); -// scheduler.Cleanup(); -// } - -// TEST_F(LoggingTests, BothPreparingTest) { -// std::unique_ptr table( -// TestingExecutorUtil::CreateTable(1)); - -// auto &log_manager = logging::LogManager::GetInstance(); - -// LoggingScheduler scheduler(2, 1, &log_manager, table.get()); - -// scheduler.Init(); -// // Logger 0 is always the front end logger -// // The first txn to commit starts with cid 2 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(2); -// scheduler.BackendLogger(0, 0).Insert(2); -// scheduler.BackendLogger(0, 0).Commit(2); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(3); -// scheduler.BackendLogger(0, 1).Insert(3); -// scheduler.BackendLogger(0, 1).Commit(3); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// // at this point everyone should be updated to 3 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(4); -// scheduler.BackendLogger(0, 0).Insert(4); -// scheduler.BackendLogger(0, 0).Commit(4); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(5); -// scheduler.BackendLogger(0, 1).Insert(5); -// scheduler.BackendLogger(0, 1).Commit(5); -// // this prepare should still get a may commit of 3 -// scheduler.BackendLogger(0, 1).Prepare(); - -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 1).Begin(6); -// scheduler.BackendLogger(0, 1).Insert(6); -// scheduler.BackendLogger(0, 1).Commit(6); -// // this call should get a may commit of 4 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 0).Done(1); -// scheduler.BackendLogger(0, 1).Done(1); - -// scheduler.Run(); - -// auto results = scheduler.frontend_threads[0].results; -// EXPECT_EQ(3, results[0]); -// EXPECT_EQ(3, results[1]); -// EXPECT_EQ(4, results[2]); -// scheduler.Cleanup(); -// } - -// TEST_F(LoggingTests, TwoRoundTest) { -// std::unique_ptr table( -// TestingExecutorUtil::CreateTable(1)); - -// auto &log_manager = logging::LogManager::GetInstance(); - -// LoggingScheduler scheduler(2, 1, &log_manager, table.get()); - -// scheduler.Init(); -// // Logger 0 is always the front end logger -// // The first txn to commit starts with cid 2 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(2); -// scheduler.BackendLogger(0, 0).Insert(2); -// scheduler.BackendLogger(0, 0).Commit(2); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(3); -// scheduler.BackendLogger(0, 1).Insert(3); -// scheduler.BackendLogger(0, 1).Commit(3); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// // at this point everyone should be updated to 3 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(4); -// scheduler.BackendLogger(0, 0).Insert(4); -// scheduler.BackendLogger(0, 0).Commit(4); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(5); -// scheduler.BackendLogger(0, 1).Insert(5); -// scheduler.BackendLogger(0, 1).Commit(5); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 0).Done(1); -// scheduler.BackendLogger(0, 1).Done(1); - -// scheduler.Run(); - -// auto results = scheduler.frontend_threads[0].results; -// EXPECT_EQ(5, results[1]); -// scheduler.Cleanup(); -// } - -// TEST_F(LoggingTests, InsertUpdateDeleteTest) { -// std::unique_ptr table( -// TestingExecutorUtil::CreateTable(1)); - -// auto &log_manager = logging::LogManager::GetInstance(); - -// LoggingScheduler scheduler(2, 1, &log_manager, table.get()); - -// scheduler.Init(); -// // Logger 0 is always the front end logger -// // The first txn to commit starts with cid 2 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(2); -// scheduler.BackendLogger(0, 0).Insert(2); -// scheduler.BackendLogger(0, 0).Commit(2); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(3); -// scheduler.BackendLogger(0, 1).Update(3); -// scheduler.BackendLogger(0, 1).Commit(3); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// // at this point everyone should be updated to 3 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(4); -// scheduler.BackendLogger(0, 0).Delete(4); -// scheduler.BackendLogger(0, 0).Commit(4); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(5); -// scheduler.BackendLogger(0, 1).Delete(5); -// scheduler.BackendLogger(0, 1).Commit(5); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 0).Done(1); -// scheduler.BackendLogger(0, 1).Done(1); - -// scheduler.Run(); - -// auto results = scheduler.frontend_threads[0].results; -// EXPECT_EQ(5, results[1]); -// scheduler.Cleanup(); -// } - -// TEST_F(LoggingTests, BasicLogManagerTest) { -// peloton_logging_mode = LoggingType::INVALID; -// auto &log_manager = logging::LogManager::GetInstance(); -// log_manager.DropFrontendLoggers(); -// log_manager.SetLoggingStatus(LoggingStatusType::INVALID); -// // just start, write a few records and exit -// catalog::Schema *table_schema = -// new catalog::Schema({TestingExecutorUtil::GetColumnInfo(0), -// TestingExecutorUtil::GetColumnInfo(1), -// TestingExecutorUtil::GetColumnInfo(2), -// TestingExecutorUtil::GetColumnInfo(3)}); -// std::string table_name("TEST_TABLE"); - -// // Create table. -// bool own_schema = true; -// bool adapt_table = false; -// storage::DataTable *table = storage::TableFactory::GetDataTable( -// 12345, 123456, table_schema, table_name, 1, own_schema, adapt_table); - -// storage::Database *test_db = new storage::Database(12345); -// test_db->AddTable(table); -// auto catalog = catalog::Catalog::GetInstance(); -// catalog->AddDatabase(test_db); -// concurrency::TransactionManager &txn_manager = -// concurrency::TransactionManagerFactory::GetInstance(); -// auto txn = txn_manager.BeginTransaction(); -// TestingExecutorUtil::PopulateTable(table, 5, true, false, false, txn); -// txn_manager.CommitTransaction(txn); -// peloton_logging_mode = LoggingType::NVM_WAL; - -// log_manager.SetSyncCommit(true); -// EXPECT_FALSE(log_manager.ContainsFrontendLogger()); -// log_manager.StartStandbyMode(); -// log_manager.GetFrontendLogger(0)->SetTestMode(true); -// log_manager.StartRecoveryMode(); -// log_manager.WaitForModeTransition(LoggingStatusType::LOGGING, true); -// EXPECT_TRUE(log_manager.ContainsFrontendLogger()); -// log_manager.SetGlobalMaxFlushedCommitId(4); -// concurrency::TransactionContext test_txn; -// cid_t commit_id = 5; -// log_manager.PrepareLogging(); -// log_manager.LogBeginTransaction(commit_id); -// ItemPointer insert_loc(table->GetTileGroup(1)->GetTileGroupId(), 0); -// ItemPointer delete_loc(table->GetTileGroup(2)->GetTileGroupId(), 0); -// ItemPointer update_old(table->GetTileGroup(3)->GetTileGroupId(), 0); -// ItemPointer update_new(table->GetTileGroup(4)->GetTileGroupId(), 0); -// log_manager.LogInsert(commit_id, insert_loc); -// log_manager.LogUpdate(commit_id, update_old, update_new); -// log_manager.LogInsert(commit_id, delete_loc); -// log_manager.LogCommitTransaction(commit_id); - -// // since we are doing sync commit we should have reached 5 already -// EXPECT_EQ(commit_id, log_manager.GetPersistentFlushedCommitId()); -// log_manager.EndLogging(); -// } - -// } // namespace test -// } // namespace peloton -// >>>>>>> master diff --git a/test/logging/logging_util_test.cpp b/test/logging/logging_util_test.cpp deleted file mode 100644 index 91f30aa6fff..00000000000 --- a/test/logging/logging_util_test.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // logging_util_test.cpp -// // -// // Identification: test/logging/logging_util_test.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - - -// #include "common/harness.h" - -// #include "logging/logging_util.h" - -// namespace peloton { -// namespace test { - -// //===--------------------------------------------------------------------===// -// // Logging Tests -// //===--------------------------------------------------------------------===// -// class LoggingUtilTests : public PelotonTest {}; - -// TEST_F(LoggingUtilTests, BasicLoggingUtilTest) { -// auto status = logging::LoggingUtil::CreateDirectory("test_dir", 0700); -// EXPECT_TRUE(status); - -// status = logging::LoggingUtil::RemoveDirectory("test_dir", true); -// EXPECT_TRUE(status); -// } - -// } // namespace test -// } // namespace peloton diff --git a/test/logging/new_checkpointing_test.cpp b/test/logging/new_checkpointing_test.cpp deleted file mode 100644 index 7cfcb42d50e..00000000000 --- a/test/logging/new_checkpointing_test.cpp +++ /dev/null @@ -1,33 +0,0 @@ -////===----------------------------------------------------------------------===// -//// -//// Peloton -//// -//// new_checkpointing_test.cpp -//// -//// Identification: test/logging/new_checkpointing_test.cpp -//// -//// Copyright (c) 2015-16, Carnegie Mellon University Database Group -//// -////===----------------------------------------------------------------------===// -// -//#include "logging/checkpoint_manager_factory.h" -//#include "common/harness.h" -// -//namespace peloton { -//namespace test { -// -////===--------------------------------------------------------------------===// -//// Checkpointing Tests -////===--------------------------------------------------------------------===// -// -//class NewCheckpointingTests : public PelotonTest {}; -// -//TEST_F(NewCheckpointingTests, MyTest) { -// auto &checkpoint_manager = logging::CheckpointManagerFactory::GetInstance(); -// checkpoint_manager.Reset(); -// -// EXPECT_TRUE(true); -//} -// -//} -//} diff --git a/test/logging/new_logging_test.cpp b/test/logging/new_logging_test.cpp deleted file mode 100644 index c7d1700b503..00000000000 --- a/test/logging/new_logging_test.cpp +++ /dev/null @@ -1,34 +0,0 @@ -////===----------------------------------------------------------------------===// -//// -//// Peloton -//// -//// new_logging_test.cpp -//// -//// Identification: test/logging/new_logging_test.cpp -//// -//// Copyright (c) 2015-16, Carnegie Mellon University Database Group -//// -////===----------------------------------------------------------------------===// -// -//#include "logging/log_manager_factory.h" -//#include "common/harness.h" -// -//namespace peloton { -//namespace test { -// -////===--------------------------------------------------------------------===// -//// Logging Tests -////===--------------------------------------------------------------------===// -// -//class NewLoggingTests : public PelotonTest {}; -// -//TEST_F(NewLoggingTests, MyTest) { -// auto &log_manager = logging::LogManagerFactory::GetInstance(); -// log_manager.Reset(); -// -// EXPECT_TRUE(true); -// -//} -// -//} -//} diff --git a/test/logging/recovery_test.cpp b/test/logging/recovery_test.cpp deleted file mode 100644 index 8e1cd5d2a2f..00000000000 --- a/test/logging/recovery_test.cpp +++ /dev/null @@ -1,456 +0,0 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // recovery_test.cpp -// // -// // Identification: test/logging/recovery_test.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - -// #include -// #include -// #include -// #include - -// #include "catalog/catalog.h" -// #include "common/harness.h" -// #include "executor/testing_executor_util.h" -// #include "logging/testing_logging_util.h" - -// #include "concurrency/transaction_manager_factory.h" -// #include "executor/logical_tile_factory.h" -// #include "index/index.h" -// #include "logging/log_manager.h" -// #include "logging/loggers/wal_frontend_logger.h" -// #include "logging/logging_util.h" -// #include "storage/data_table.h" -// #include "storage/database.h" -// #include "storage/table_factory.h" -// #include "storage/tile.h" - -// #include "executor/mock_executor.h" - -// #define DEFAULT_RECOVERY_CID 15 - -// using ::testing::NotNull; -// using ::testing::Return; -// using ::testing::InSequence; - -// namespace peloton { -// namespace test { - -// //===--------------------------------------------------------------------===// -// // Recovery Tests -// //===--------------------------------------------------------------------===// - -// class RecoveryTests : public PelotonTest {}; - -// std::vector BuildLoggingTuples(storage::DataTable *table, -// int num_rows, bool mutate, -// bool random) { -// std::vector tuples; -// LOG_INFO("build a vector of %d tuples", num_rows); - -// // Random values -// std::srand(std::time(nullptr)); -// const catalog::Schema *schema = table->GetSchema(); -// // Ensure that the tile group is as expected. -// PELOTON_ASSERT(schema->GetColumnCount() == 4); - -// // Insert tuples into tile_group. -// const bool allocate = true; -// auto testing_pool = TestingHarness::GetInstance().GetTestingPool(); - -// for (int rowid = 0; rowid < num_rows; rowid++) { -// int populate_value = rowid; -// if (mutate) populate_value *= 3; - -// storage::Tuple *tuple = new storage::Tuple(schema, allocate); - -// // First column is unique in this case -// tuple->SetValue(0, -// type::ValueFactory::GetIntegerValue( -// TestingExecutorUtil::PopulatedValue(populate_value, 0)), -// testing_pool); - -// // In case of random, make sure this column has duplicated values -// tuple->SetValue( -// 1, -// type::ValueFactory::GetIntegerValue(TestingExecutorUtil::PopulatedValue( -// random ? std::rand() % (num_rows / 3) : populate_value, 1)), -// testing_pool); - -// tuple->SetValue(2, type::ValueFactory::GetDecimalValue( -// TestingExecutorUtil::PopulatedValue( -// random ? std::rand() : populate_value, 2)), -// testing_pool); - -// // In case of random, make sure this column has duplicated values -// auto string_value = type::ValueFactory::GetVarcharValue( -// std::to_string(TestingExecutorUtil::PopulatedValue( -// random ? std::rand() % (num_rows / 3) : populate_value, 3))); -// tuple->SetValue(3, string_value, testing_pool); -// tuples.push_back(tuple); -// } -// return tuples; -// } - -// TEST_F(RecoveryTests, RestartTest) { -// auto catalog = catalog::Catalog::GetInstance(); -// LOG_TRACE("Finish creating catalog"); -// LOG_TRACE("Creating recovery_table"); -// auto recovery_table = TestingExecutorUtil::CreateTable(1024); -// LOG_TRACE("Finish creating recovery_table"); - -// size_t tile_group_size = 5; -// size_t table_tile_group_count = 3; -// int num_files = 3; - -// auto mutate = true; -// auto random = false; -// cid_t default_commit_id = INVALID_CID; -// cid_t default_delimiter = INVALID_CID; - -// // XXX: for now hardcode for one logger (suffix 0) -// std::string dir_name = logging::WriteAheadFrontendLogger::wal_directory_path; - -// storage::Database *db = new storage::Database(DEFAULT_DB_ID); -// catalog->AddDatabase(db); -// db->AddTable(recovery_table); - -// int num_rows = tile_group_size * table_tile_group_count; -// std::vector> tuples = -// TestingLoggingUtil::BuildTuples(recovery_table, num_rows + 2, mutate, -// random); - -// std::vector records = -// TestingLoggingUtil::BuildTupleRecordsForRestartTest( -// tuples, tile_group_size, table_tile_group_count, 1, 1); - -// logging::LoggingUtil::RemoveDirectory(dir_name.c_str(), false); - -// auto status = logging::LoggingUtil::CreateDirectory(dir_name.c_str(), 0700); -// EXPECT_TRUE(status); -// logging::LogManager::GetInstance().SetLogDirectoryName("./"); - -// for (int i = 0; i < num_files; i++) { -// std::string file_name = dir_name + "/" + std::string("peloton_log_") + -// std::to_string(i) + std::string(".log"); -// FILE *fp = fopen(file_name.c_str(), "wb"); - -// // now set the first 8 bytes to 0 - this is for the max_log id in this file -// fwrite((void *)&default_commit_id, sizeof(default_commit_id), 1, fp); - -// // now set the next 8 bytes to 0 - this is for the max delimiter in this -// // file -// fwrite((void *)&default_delimiter, sizeof(default_delimiter), 1, fp); - -// // First write a begin record -// CopySerializeOutput output_buffer_begin; -// logging::TransactionRecord record_begin(LOGRECORD_TYPE_TRANSACTION_BEGIN, -// i + 2); -// record_begin.Serialize(output_buffer_begin); - -// fwrite(record_begin.GetMessage(), sizeof(char), -// record_begin.GetMessageLength(), fp); - -// // Now write 5 insert tuple records into this file -// for (int j = 0; j < (int)tile_group_size; j++) { -// int num_record = i * tile_group_size + j; -// CopySerializeOutput output_buffer; -// records[num_record].Serialize(output_buffer); - -// fwrite(records[num_record].GetMessage(), sizeof(char), -// records[num_record].GetMessageLength(), fp); -// } - -// // Now write 1 extra out of range tuple, only in file 0, which -// // is present at the second-but-last position of this list -// if (i == 0) { -// CopySerializeOutput output_buffer_extra; -// records[num_files * tile_group_size].Serialize(output_buffer_extra); - -// fwrite(records[num_files * tile_group_size].GetMessage(), sizeof(char), -// records[num_files * tile_group_size].GetMessageLength(), fp); -// } - -// // // Now write 1 extra delete tuple, only in the last file, which -// // // is present at the end of this list -// // if (i == num_files - 1) { -// // CopySerializeOutput output_buffer_delete; -// // records[num_files * tile_group_size + 1].Serialize(output_buffer_delete); - -// // fwrite(records[num_files * tile_group_size + 1].GetMessage(), -// // sizeof(char), -// // records[num_files * tile_group_size + 1].GetMessageLength(), fp); -// // } - -// // // Now write commit -// // logging::TransactionRecord record_commit(LOGRECORD_TYPE_TRANSACTION_COMMIT, -// // i + 2); - -// // CopySerializeOutput output_buffer_commit; -// // record_commit.Serialize(output_buffer_commit); - -// // fwrite(record_commit.GetMessage(), sizeof(char), -// // record_commit.GetMessageLength(), fp); - -// // // Now write delimiter -// // CopySerializeOutput output_buffer_delim; -// // logging::TransactionRecord record_delim(LOGRECORD_TYPE_ITERATION_DELIMITER, -// // i + 2); - -// // record_delim.Serialize(output_buffer_delim); - -// // fwrite(record_delim.GetMessage(), sizeof(char), -// // record_delim.GetMessageLength(), fp); - -// fclose(fp); -// } - -// LOG_INFO("All files created and written to."); -// int index_count = recovery_table->GetIndexCount(); -// LOG_INFO("Number of indexes on this table: %d", (int)index_count); - -// for (int index_itr = index_count - 1; index_itr >= 0; --index_itr) { -// auto index = recovery_table->GetIndex(index_itr); -// EXPECT_EQ(index->GetNumberOfTuples(), 0); -// } - -// logging::WriteAheadFrontendLogger wal_fel; - -// EXPECT_EQ(wal_fel.GetMaxDelimiterForRecovery(), num_files + 1); -// EXPECT_EQ(wal_fel.GetLogFileCounter(), num_files); - -// EXPECT_EQ(recovery_table->GetTupleCount(), 0); - -// LOG_TRACE("recovery_table tile group count before recovery: %ld", -// recovery_table->GetTileGroupCount()); - -// auto &log_manager = logging::LogManager::GetInstance(); -// log_manager.SetGlobalMaxFlushedIdForRecovery(num_files + 1); - -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - -// wal_fel.DoRecovery(); - -// LOG_TRACE("recovery_table tile group count after recovery: %ld", -// recovery_table->GetTileGroupCount()); - -// EXPECT_EQ(recovery_table->GetTupleCount(), -// tile_group_size * table_tile_group_count - 1); -// EXPECT_EQ(wal_fel.GetLogFileCursor(), num_files); - -// txn_manager.SetNextCid(5); -// wal_fel.RecoverIndex(); -// for (int index_itr = index_count - 1; index_itr >= 0; --index_itr) { -// auto index = recovery_table->GetIndex(index_itr); -// EXPECT_EQ(index->GetNumberOfTuples(), -// tile_group_size * table_tile_group_count - 1); -// } - -// // TODO check a few more invariants here -// wal_fel.CreateNewLogFile(false); - -// EXPECT_EQ(wal_fel.GetLogFileCounter(), num_files + 1); - -// wal_fel.TruncateLog(4); - -// for (int i = 1; i <= 2; i++) { -// struct stat stat_buf; -// EXPECT_NE(stat((dir_name + "/" + std::string("peloton_log_") + -// std::to_string(i) + std::string(".log")).c_str(), -// &stat_buf), -// 0); -// } - -// wal_fel.CreateNewLogFile(true); - -// EXPECT_EQ(wal_fel.GetLogFileCounter(), num_files + 2); - -// status = logging::LoggingUtil::RemoveDirectory(dir_name.c_str(), false); -// EXPECT_TRUE(status); - -// auto txn = txn_manager.BeginTransaction(); -// catalog->DropDatabaseWithOid(DEFAULT_DB_ID, txn); -// txn_manager.CommitTransaction(txn); -// } - -// TEST_F(RecoveryTests, BasicInsertTest) { -// auto recovery_table = TestingExecutorUtil::CreateTable(1024); -// auto catalog = catalog::Catalog::GetInstance(); -// storage::Database *db = new storage::Database(DEFAULT_DB_ID); -// catalog->AddDatabase(db); -// db->AddTable(recovery_table); - -// auto tuples = BuildLoggingTuples(recovery_table, 1, false, false); -// EXPECT_EQ(recovery_table->GetTupleCount(), 0); -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 1); -// EXPECT_EQ(tuples.size(), 1); -// logging::WriteAheadFrontendLogger fel(true); -// // auto bel = logging::WriteAheadBackendLogger::GetInstance(); -// cid_t test_commit_id = 10; - -// type::Value val0 = (tuples[0]->GetValue(0)); -// type::Value val1 = (tuples[0]->GetValue(1)); -// type::Value val2 = (tuples[0]->GetValue(2)); -// type::Value val3 = (tuples[0]->GetValue(3)); -// auto curr_rec = new logging::TupleRecord( -// LOGRECORD_TYPE_TUPLE_INSERT, test_commit_id, recovery_table->GetOid(), -// ItemPointer(100, 5), INVALID_ITEMPOINTER, tuples[0], DEFAULT_DB_ID); -// curr_rec->SetTuple(tuples[0]); -// fel.InsertTuple(curr_rec); -// delete curr_rec; - -// auto tg_header = recovery_table->GetTileGroupById(100)->GetHeader(); -// EXPECT_TRUE(tg_header->GetBeginCommitId(5) <= test_commit_id); -// EXPECT_EQ(tg_header->GetEndCommitId(5), MAX_CID); - -// type::Value rval0 = (recovery_table->GetTileGroupById(100)->GetValue(5, 0)); -// CmpBool cmp0 = (val0.CompareEquals(rval0)); -// EXPECT_TRUE(cmp0 == CmpBool::CmpTrue); -// type::Value rval1 = (recovery_table->GetTileGroupById(100)->GetValue(5, 1)); -// CmpBool cmp1 = (val1.CompareEquals(rval1)); -// EXPECT_TRUE(cmp1 == CmpBool::CmpTrue); -// type::Value rval2 = (recovery_table->GetTileGroupById(100)->GetValue(5, 2)); -// CmpBool cmp2 = (val2.CompareEquals(rval2)); -// EXPECT_TRUE(cmp2 == CmpBool::CmpTrue); -// type::Value rval3 = (recovery_table->GetTileGroupById(100)->GetValue(5, 3)); -// CmpBool cmp3 = (val3.CompareEquals(rval3)); -// EXPECT_TRUE(cmp3 == CmpBool::CmpTrue); - -// EXPECT_EQ(recovery_table->GetTupleCount(), 1); -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 2); - -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// auto txn = txn_manager.BeginTransaction(); -// catalog->DropDatabaseWithOid(DEFAULT_DB_ID, txn); -// txn_manager.CommitTransaction(txn); -// } - -// TEST_F(RecoveryTests, BasicUpdateTest) { -// auto catalog = catalog::Catalog::GetInstance(); -// auto recovery_table = TestingExecutorUtil::CreateTable(1024); -// storage::Database *db = new storage::Database(DEFAULT_DB_ID); -// catalog->AddDatabase(db); -// db->AddTable(recovery_table); - -// auto tuples = BuildLoggingTuples(recovery_table, 1, false, false); -// EXPECT_EQ(recovery_table->GetTupleCount(), 0); -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 1); -// EXPECT_EQ(tuples.size(), 1); -// logging::WriteAheadFrontendLogger fel(true); -// // auto bel = logging::WriteAheadBackendLogger::GetInstance(); -// cid_t test_commit_id = 10; - -// type::Value val0 = (tuples[0]->GetValue(0)); -// type::Value val1 = (tuples[0]->GetValue(1)); -// type::Value val2 = (tuples[0]->GetValue(2)); -// type::Value val3 = (tuples[0]->GetValue(3)); - -// auto curr_rec = new logging::TupleRecord( -// LOGRECORD_TYPE_TUPLE_UPDATE, test_commit_id, recovery_table->GetOid(), -// ItemPointer(100, 5), ItemPointer(100, 4), tuples[0], DEFAULT_DB_ID); -// curr_rec->SetTuple(tuples[0]); -// fel.UpdateTuple(curr_rec); -// delete curr_rec; - -// auto tg_header = recovery_table->GetTileGroupById(100)->GetHeader(); -// EXPECT_TRUE(tg_header->GetBeginCommitId(5) <= test_commit_id); -// EXPECT_EQ(tg_header->GetEndCommitId(5), MAX_CID); -// EXPECT_EQ(tg_header->GetEndCommitId(4), test_commit_id); - -// type::Value rval0 = (recovery_table->GetTileGroupById(100)->GetValue(5, 0)); -// CmpBool cmp0 = (val0.CompareEquals(rval0)); -// EXPECT_TRUE(cmp0 == CmpBool::CmpTrue); -// type::Value rval1 = (recovery_table->GetTileGroupById(100)->GetValue(5, 1)); -// CmpBool cmp1 = (val1.CompareEquals(rval1)); -// EXPECT_TRUE(cmp1 == CmpBool::CmpTrue); -// type::Value rval2 = (recovery_table->GetTileGroupById(100)->GetValue(5, 2)); -// CmpBool cmp2 = (val2.CompareEquals(rval2)); -// EXPECT_TRUE(cmp2 == CmpBool::CmpTrue); -// type::Value rval3 = (recovery_table->GetTileGroupById(100)->GetValue(5, 3)); -// CmpBool cmp3 = (val3.CompareEquals(rval3)); -// EXPECT_TRUE(cmp3 == CmpBool::CmpTrue); - -// EXPECT_EQ(recovery_table->GetTupleCount(), 0); -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 2); - -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// auto txn = txn_manager.BeginTransaction(); -// catalog->DropDatabaseWithOid(DEFAULT_DB_ID, txn); -// txn_manager.CommitTransaction(txn); -// } - -// /* (From Joy) TODO FIX this -// TEST_F(RecoveryTests, BasicDeleteTest) { -// auto recovery_table = ExecutorTestsUtil::CreateTable(1024); -// auto &manager = catalog::Manager::GetInstance(); -// storage::Database db(DEFAULT_DB_ID); -// manager.AddDatabase(&db); -// db.AddTable(recovery_table); - -// EXPECT_EQ(recovery_table->GetNumberOfTuples(), 0); -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 1); -// logging::WriteAheadFrontendLogger fel(true); - -// cid_t test_commit_id = 10; - -// auto curr_rec = new logging::TupleRecord( -// LOGRECORD_TYPE_TUPLE_UPDATE, test_commit_id, recovery_table->GetOid(), -// INVALID_ITEMPOINTER, ItemPointer(100, 4), nullptr, DEFAULT_DB_ID); -// fel.DeleteTuple(curr_rec); - -// delete curr_rec; - -// auto tg_header = recovery_table->GetTileGroupById(100)->GetHeader(); -// EXPECT_EQ(tg_header->GetEndCommitId(4), test_commit_id); - -// // EXPECT_EQ(recovery_table->GetNumberOfTuples(), 1); -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 2); -// }*/ - -// TEST_F(RecoveryTests, OutOfOrderCommitTest) { -// auto recovery_table = TestingExecutorUtil::CreateTable(1024); -// auto catalog = catalog::Catalog::GetInstance(); -// storage::Database *db = new storage::Database(DEFAULT_DB_ID); -// catalog->AddDatabase(db); -// db->AddTable(recovery_table); - -// auto tuples = BuildLoggingTuples(recovery_table, 1, false, false); -// EXPECT_EQ(recovery_table->GetTupleCount(), 0); -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 1); -// EXPECT_EQ(tuples.size(), 1); -// logging::WriteAheadFrontendLogger fel(true); -// // auto bel = logging::WriteAheadBackendLogger::GetInstance(); -// cid_t test_commit_id = 10; - -// auto curr_rec = new logging::TupleRecord( -// LOGRECORD_TYPE_TUPLE_UPDATE, test_commit_id + 1, recovery_table->GetOid(), -// INVALID_ITEMPOINTER, ItemPointer(100, 5), nullptr, DEFAULT_DB_ID); -// fel.DeleteTuple(curr_rec); -// delete curr_rec; - -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 2); - -// curr_rec = new logging::TupleRecord( -// LOGRECORD_TYPE_TUPLE_INSERT, test_commit_id, recovery_table->GetOid(), -// ItemPointer(100, 5), INVALID_ITEMPOINTER, tuples[0], DEFAULT_DB_ID); - -// curr_rec->SetTuple(tuples[0]); -// fel.InsertTuple(curr_rec); - -// delete curr_rec; - -// auto tg_header = recovery_table->GetTileGroupById(100)->GetHeader(); -// EXPECT_EQ(tg_header->GetEndCommitId(5), test_commit_id + 1); - -// EXPECT_EQ(recovery_table->GetTupleCount(), 0); -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 2); -// } - -// } // namespace test -// } // namespace peloton diff --git a/test/logging/testing_logging_util.cpp b/test/logging/testing_logging_util.cpp deleted file mode 100644 index e495e32556b..00000000000 --- a/test/logging/testing_logging_util.cpp +++ /dev/null @@ -1,357 +0,0 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // logging_tests_util.cpp -// // -// // Identification: test/logging/logging_tests_util.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - -// #include "logging/testing_logging_util.h" - -// #define DEFAULT_RECOVERY_CID 15 - -// namespace peloton { -// namespace test { - -// //===--------------------------------------------------------------------===// -// // LoggingTests Util -// //===--------------------------------------------------------------------===// - -// std::vector TestingLoggingUtil::BuildTupleRecords( -// std::vector> &tuples, -// size_t tile_group_size, size_t table_tile_group_count) { -// std::vector records; -// for (size_t block = 1; block <= table_tile_group_count; ++block) { -// for (size_t offset = 0; offset < tile_group_size; ++offset) { -// ItemPointer location(block, offset); -// auto &tuple = tuples[(block - 1) * tile_group_size + offset]; -// PELOTON_ASSERT(tuple->GetSchema()); -// logging::TupleRecord record( -// LOGRECORD_TYPE_WAL_TUPLE_INSERT, INITIAL_TXN_ID, INVALID_OID, -// location, INVALID_ITEMPOINTER, tuple.get(), DEFAULT_DB_ID); -// record.SetTuple(tuple.get()); -// records.push_back(record); -// } -// } -// LOG_TRACE("Built a vector of %lu tuple WAL insert records", records.size()); -// return records; -// } - -// std::vector -// TestingLoggingUtil::BuildTupleRecordsForRestartTest( -// std::vector> &tuples, -// size_t tile_group_size, size_t table_tile_group_count, -// int out_of_range_tuples, int delete_tuples) { -// auto tile_group_start_oid = -// catalog::Manager::GetInstance().GetNextTileGroupId(); -// std::vector records; -// for (size_t block = 1; block <= table_tile_group_count; ++block) { -// for (size_t offset = 0; offset < tile_group_size; ++offset) { -// ItemPointer location(block + tile_group_start_oid, offset); -// auto &tuple = tuples[(block - 1) * tile_group_size + offset]; -// PELOTON_ASSERT(tuple->GetSchema()); -// logging::TupleRecord record(LOGRECORD_TYPE_WAL_TUPLE_INSERT, block + 1, -// INVALID_OID, location, INVALID_ITEMPOINTER, -// tuple.get(), DEFAULT_DB_ID); -// record.SetTuple(tuple.get()); -// records.push_back(record); -// } -// } -// for (int i = 0; i < out_of_range_tuples; i++) { -// ItemPointer location(tile_group_size + tile_group_start_oid, -// table_tile_group_count + i); -// auto &tuple = tuples[tile_group_size * table_tile_group_count + i]; -// PELOTON_ASSERT(tuple->GetSchema()); -// logging::TupleRecord record(LOGRECORD_TYPE_WAL_TUPLE_INSERT, 1000, -// INVALID_OID, location, INVALID_ITEMPOINTER, -// tuple.get(), DEFAULT_DB_ID); -// record.SetTuple(tuple.get()); -// records.push_back(record); -// } -// for (int i = 0; i < delete_tuples; i++) { -// ItemPointer location(tile_group_start_oid + 1, 0); -// auto &tuple = tuples[tile_group_size * table_tile_group_count + -// out_of_range_tuples + i]; -// PELOTON_ASSERT(tuple->GetSchema()); -// logging::TupleRecord record(LOGRECORD_TYPE_WAL_TUPLE_DELETE, 4, INVALID_OID, -// INVALID_ITEMPOINTER, location, nullptr, -// DEFAULT_DB_ID); -// record.SetTuple(tuple.get()); -// records.push_back(record); -// } - -// LOG_TRACE("Built a vector of %lu tuple WAL insert records", records.size()); -// return records; -// } - -// std::vector> TestingLoggingUtil::BuildTuples( -// storage::DataTable *table, int num_rows, bool mutate, bool random) { -// std::vector> tuples; -// LOG_TRACE("build a vector of %d tuples", num_rows); - -// // Random values -// std::srand(std::time(nullptr)); -// const catalog::Schema *schema = table->GetSchema(); -// // Ensure that the tile group is as expected. -// PELOTON_ASSERT(schema->GetColumnCount() == 4); - -// // Insert tuples into tile_group. -// const bool allocate = true; -// auto testing_pool = TestingHarness::GetInstance().GetTestingPool(); - -// for (int rowid = 0; rowid < num_rows; rowid++) { -// int populate_value = rowid; -// if (mutate) populate_value *= 3; - -// std::shared_ptr tuple(new storage::Tuple(schema, allocate)); - -// // First column is unique in this case -// tuple->SetValue(0, -// type::ValueFactory::GetIntegerValue( -// TestingExecutorUtil::PopulatedValue(populate_value, 0)), -// testing_pool); - -// // In case of random, make sure this column has duplicated values -// tuple->SetValue( -// 1, -// type::ValueFactory::GetIntegerValue(TestingExecutorUtil::PopulatedValue( -// random ? std::rand() % (num_rows / 3) : populate_value, 1)), -// testing_pool); - -// tuple->SetValue(2, type::ValueFactory::GetDecimalValue( -// TestingExecutorUtil::PopulatedValue( -// random ? std::rand() : populate_value, 2)), -// testing_pool); - -// // In case of random, make sure this column has duplicated values -// auto string_value = type::ValueFactory::GetVarcharValue( -// std::to_string(TestingExecutorUtil::PopulatedValue( -// random ? std::rand() % (num_rows / 3) : populate_value, 3))); -// tuple->SetValue(3, string_value, testing_pool); -// PELOTON_ASSERT(tuple->GetSchema()); -// tuples.push_back(std::move(tuple)); -// } -// return tuples; -// } - -// // ======================================================================= -// // Abstract Logging Thread -// // ======================================================================= -// void AbstractLoggingThread::MainLoop() { -// while (true) { -// while (!go) { -// std::chrono::milliseconds sleep_time(1); -// std::this_thread::sleep_for(sleep_time); -// }; -// ExecuteNext(); -// if (cur_seq == (int)schedule->operations.size()) { -// go = false; -// return; -// } -// go = false; -// } -// } - -// // ======================================================================= -// // Frontend Logging Thread -// // ======================================================================= -// void FrontendLoggingThread::RunLoop() { -// frontend_logger = reinterpret_cast( -// log_manager->GetFrontendLogger(frontend_id)); - -// // Assume txns up to cid = 1 is committed -// frontend_logger->SetMaxFlushedCommitId(1); - -// MainLoop(); -// } - -// void FrontendLoggingThread::ExecuteNext() { -// // Prepare data for operation -// logging_op_type op = schedule->operations[cur_seq].op; - -// cur_seq++; - -// // Execute the operation -// switch (op) { -// case LOGGING_OP_COLLECT: { -// LOG_TRACE("Execute Collect"); -// PELOTON_ASSERT(frontend_logger); -// frontend_logger->CollectLogRecordsFromBackendLoggers(); -// break; -// } -// case LOGGING_OP_FLUSH: { -// LOG_TRACE("Execute Flush"); -// PELOTON_ASSERT(frontend_logger); -// frontend_logger->FlushLogRecords(); -// results.push_back(frontend_logger->GetMaxFlushedCommitId()); -// break; -// } -// default: { -// LOG_TRACE("Unsupported operation type!"); -// PELOTON_ASSERT(false); -// break; -// } -// } -// } - -// void BackendLoggingThread::RunLoop() { -// backend_logger = reinterpret_cast( -// log_manager->GetBackendLogger()); - -// MainLoop(); -// } - -// void BackendLoggingThread::ExecuteNext() { -// // Prepare data for operation -// logging_op_type op = schedule->operations[cur_seq].op; -// cid_t cid = schedule->operations[cur_seq].cid; - -// cur_seq++; - -// // Execute the operation -// switch (op) { -// case LOGGING_OP_PREPARE: { -// LOG_TRACE("Execute Prepare"); -// log_manager->PrepareLogging(); -// break; -// } -// case LOGGING_OP_BEGIN: { -// LOG_TRACE("Execute Begin txn %d", (int)cid); -// log_manager->LogBeginTransaction(cid); -// break; -// } -// case LOGGING_OP_INSERT: { -// LOG_TRACE("Execute Insert txn %d", (int)cid); -// auto tuple = TestingLoggingUtil::BuildTuples(table, 1, false, false)[0]; -// std::unique_ptr tuple_record( -// backend_logger->GetTupleRecord(LOGRECORD_TYPE_TUPLE_INSERT, cid, 1, -// DEFAULT_DB_ID, INVALID_ITEMPOINTER, -// INVALID_ITEMPOINTER, tuple.get())); -// backend_logger->Log(tuple_record.get()); -// tuple.reset(); -// break; -// } -// case LOGGING_OP_UPDATE: { -// LOG_TRACE("Execute Update txn %d", (int)cid); -// auto tuple = TestingLoggingUtil::BuildTuples(table, 1, false, false)[0]; -// std::unique_ptr tuple_record( -// backend_logger->GetTupleRecord(LOGRECORD_TYPE_TUPLE_UPDATE, cid, 1, -// DEFAULT_DB_ID, INVALID_ITEMPOINTER, -// INVALID_ITEMPOINTER, tuple.get())); -// backend_logger->Log(tuple_record.get()); -// tuple.reset(); -// break; -// } -// case LOGGING_OP_DELETE: { -// LOG_TRACE("Execute Delete txn %d", (int)cid); -// auto tuple = TestingLoggingUtil::BuildTuples(table, 1, false, false)[0]; -// std::unique_ptr tuple_record( -// backend_logger->GetTupleRecord(LOGRECORD_TYPE_TUPLE_DELETE, cid, 1, -// DEFAULT_DB_ID, INVALID_ITEMPOINTER, -// INVALID_ITEMPOINTER, tuple.get())); -// backend_logger->Log(tuple_record.get()); -// tuple.reset(); -// break; -// } -// case LOGGING_OP_DONE: { -// LOG_TRACE("Execute Done txn %d", (int)cid); -// log_manager->DoneLogging(); -// break; -// } -// case LOGGING_OP_COMMIT: { -// LOG_TRACE("Execute Commit txn %d", (int)cid); -// std::unique_ptr record(new logging::TransactionRecord( -// LOGRECORD_TYPE_TRANSACTION_COMMIT, cid)); -// PELOTON_ASSERT(backend_logger); -// backend_logger->Log(record.get()); -// break; -// } -// case LOGGING_OP_ABORT: { -// LOG_TRACE("Execute Abort txn %d", (int)cid); -// std::unique_ptr record(new logging::TransactionRecord( -// LOGRECORD_TYPE_TRANSACTION_ABORT, cid)); -// PELOTON_ASSERT(backend_logger); -// backend_logger->Log(record.get()); -// break; -// } -// default: { -// LOG_ERROR("Unsupported operation type!"); -// break; -// } -// } -// } - -// // ========================================================================= -// // Logging Scheduler -// // ========================================================================== -// void LoggingScheduler::Run() { -// // Run the txns according to the schedule -// if (!concurrent) { -// for (auto itr = sequence.begin(); itr != sequence.end(); itr++) { -// auto front_id = itr->second.front; -// auto backend_id = itr->second.back; -// PELOTON_ASSERT(front_id != INVALID_LOGGER_IDX); - -// // frontend logger's turn -// if (backend_id == INVALID_LOGGER_IDX) { -// LOG_TRACE("Execute Frontend Thread %d", (int)front_id); -// frontend_threads[front_id].go = true; -// while (frontend_threads[front_id].go) { -// std::chrono::milliseconds sleep_time(1); -// std::this_thread::sleep_for(sleep_time); -// } -// LOG_TRACE("Done Frontend Thread %d", (int)front_id); -// } else { -// // backend logger's turn -// LOG_TRACE("Execute Backend Thread (%d, %d)", (int)front_id, -// (int)backend_id % num_backend_logger_per_frontend); -// backend_threads[backend_id].go = true; -// while (backend_threads[backend_id].go) { -// std::chrono::milliseconds sleep_time(1); -// std::this_thread::sleep_for(sleep_time); -// } -// LOG_TRACE("Done Backend Thread (%d, %d)", (int)front_id, -// (int)backend_id % num_backend_logger_per_frontend); -// } -// } -// } -// } - -// void LoggingScheduler::Init() { -// logging::LogManager::GetInstance().Configure( -// LoggingType::NVM_WAL, true, num_frontend_logger, -// LoggerMappingStrategyType::MANUAL); -// log_manager->SetLoggingStatus(LoggingStatusType::LOGGING); -// log_manager->InitFrontendLoggers(); - -// for (unsigned int i = 0; i < num_frontend_logger; i++) { -// frontend_threads.emplace_back(&frontend_schedules[i], log_manager, i, -// table); -// } -// for (unsigned int i = 0; -// i < num_frontend_logger * num_backend_logger_per_frontend; i++) { -// backend_threads.emplace_back(&backend_schedules[i], log_manager, i, table, -// i % num_backend_logger_per_frontend); -// } - -// // Spawn frontend logger threads -// for (int i = 0; i < (int)frontend_schedules.size(); i++) { -// std::thread t = frontend_threads[i].Run(); -// t.detach(); -// } - -// // Spawn backend logger threads -// for (int i = 0; i < (int)backend_schedules.size(); i++) { -// std::thread t = backend_threads[i].Run(); -// t.detach(); -// } -// } - -// void LoggingScheduler::Cleanup() { log_manager->ResetFrontendLoggers(); } - -// } // namespace test -// } // namespace peloton diff --git a/test/logging/write_behind_logging_test.cpp b/test/logging/write_behind_logging_test.cpp deleted file mode 100644 index 18fdca7d20d..00000000000 --- a/test/logging/write_behind_logging_test.cpp +++ /dev/null @@ -1,168 +0,0 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // write_behind_logging_test.cpp -// // -// // Identification: test/logging/write_behind_logging_test.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - - -// #include "executor/testing_executor_util.h" -// #include "logging/testing_logging_util.h" -// #include "common/harness.h" - -// #include "concurrency/transaction_manager_factory.h" -// #include "executor/logical_tile_factory.h" -// #include "executor/executor_context.h" -// #include "logging/loggers/wal_frontend_logger.h" -// #include "logging/logging_util.h" -// #include "storage/data_table.h" -// #include "storage/tile.h" -// #include "storage/table_factory.h" - -// #include "executor/mock_executor.h" - -// using ::testing::NotNull; -// using ::testing::Return; -// using ::testing::InSequence; - -// namespace peloton { -// namespace test { -// class WriteBehindLoggingTests : public PelotonTest {}; - -// /* TODO: Disabled it due to arbitrary timing constraints -// void grant_thread(concurrency::TransactionManager &txn_manager){ -// for (long i = 6; i <= 20; i++){ -// std::this_thread::sleep_for(std::chrono::milliseconds(10)); -// txn_manager.SetMaxGrantCid(i); -// } -// } - -// // not sure the best way to test this, so I will spawn a new thread to bump up the grant every 10 ms -// // and ensure enough time has passed by the end of the test (we are not prematurely -// // allowing transactions to continue with unsanctioned cids -// TEST_F(WriteBehindLoggingTests, BasicGrantTest) { -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// txn_manager.SetMaxGrantCid(5); -// auto begin = std::chrono::high_resolution_clock::now(); -// std::thread granting_thread(grant_thread, std::ref(txn_manager)); -// for (int i = 0; i < 20; i++){ -// txn_manager.GetNextCommitId(); -// } -// auto end = std::chrono::high_resolution_clock::now(); -// std::chrono::milliseconds min_expected_dur(140); -// EXPECT_TRUE(end-begin > min_expected_dur); -// granting_thread.join(); - - -// } -// */ - -// int SeqScanCount(storage::DataTable *table, -// const std::vector &column_ids, -// expression::AbstractExpression *predicate) { -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// auto txn = txn_manager.BeginTransaction(); -// std::unique_ptr context( -// new executor::ExecutorContext(txn)); - -// planner::SeqScanPlan seq_scan_node(table, predicate, column_ids); -// executor::SeqScanExecutor seq_scan_executor(&seq_scan_node, context.get()); - -// EXPECT_TRUE(seq_scan_executor.Init()); -// auto tuple_cnt = 0; - -// while (seq_scan_executor.Execute()) { -// std::unique_ptr result_logical_tile( -// seq_scan_executor.GetOutput()); -// tuple_cnt += result_logical_tile->GetTupleCount(); -// } - -// txn_manager.CommitTransaction(txn); - -// return tuple_cnt; -// } - -// //check the visibility -// // TEST_F(WriteBehindLoggingTests, DirtyRangeVisibilityTest) { -// // auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// // auto &catalog_manager = catalog::Manager::GetInstance(); - -// // ItemPointer *index_entry_ptr = nullptr; - -// // std::unique_ptr table(TestingExecutorUtil::CreateTable()); -// // auto pool = TestingHarness::GetInstance().GetTestingPool(); - -// // txn_manager.SetNextCid(1); -// // auto txn = txn_manager.BeginTransaction(); -// // auto tuple = TestingExecutorUtil::GetTuple(table.get(), 1, pool); -// // index_entry_ptr = nullptr; -// // auto visible1 = table->InsertTuple(tuple.get(), txn, &index_entry_ptr); -// // txn_manager.PerformInsert(txn, visible1, index_entry_ptr); -// // txn_manager.CommitTransaction(txn); - -// // // got cid 2 -// // txn = txn_manager.BeginTransaction(); -// // tuple = TestingExecutorUtil::GetTuple(table.get(), 2, pool); -// // index_entry_ptr = nullptr; -// // auto visible2 = table->InsertTuple(tuple.get(), txn, &index_entry_ptr); -// // txn_manager.PerformInsert(txn, visible2, index_entry_ptr); -// // txn_manager.CommitTransaction(txn); - -// // // got cid 3 -// // txn = txn_manager.BeginTransaction(); -// // tuple = TestingExecutorUtil::GetTuple(table.get(), 3, pool); -// // index_entry_ptr = nullptr; -// // auto invisible1 = table->InsertTuple(tuple.get(), txn, &index_entry_ptr); -// // txn_manager.PerformInsert(txn, invisible1, index_entry_ptr); -// // txn_manager.CommitTransaction(txn); - -// // // got cid 4 -// // txn = txn_manager.BeginTransaction(); -// // tuple = TestingExecutorUtil::GetTuple(table.get(), 4, pool); -// // index_entry_ptr = nullptr; -// // auto invisible2 = table->InsertTuple(tuple.get(), txn, &index_entry_ptr); -// // txn_manager.PerformInsert(txn, invisible2, index_entry_ptr); -// // txn_manager.CommitTransaction(txn); - -// // // got cid 5 -// // txn = txn_manager.BeginTransaction(); -// // tuple = TestingExecutorUtil::GetTuple(table.get(), 5, pool); -// // index_entry_ptr = nullptr; -// // auto visible3 = table->InsertTuple(tuple.get(), txn, &index_entry_ptr); -// // txn_manager.PerformInsert(txn, visible3, index_entry_ptr); -// // txn_manager.CommitTransaction(txn); - -// // // got cid 6 -// // std::vector column_ids; -// // column_ids.push_back(0); -// // column_ids.push_back(1); -// // column_ids.push_back(2); -// // column_ids.push_back(3); - -// // txn = txn_manager.BeginTransaction(); -// // EXPECT_TRUE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(visible1.block)->GetHeader(), visible1.offset) == VisibilityType::OK); -// // EXPECT_TRUE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(visible2.block)->GetHeader(), visible2.offset) == VisibilityType::OK); -// // EXPECT_TRUE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(invisible1.block)->GetHeader(), invisible1.offset) == VisibilityType::OK); -// // EXPECT_TRUE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(invisible2.block)->GetHeader(), invisible2.offset) == VisibilityType::OK); -// // EXPECT_TRUE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(visible3.block)->GetHeader(), visible3.offset) == VisibilityType::OK); -// // txn_manager.AbortTransaction(txn); - -// // txn_manager.SetDirtyRange(std::make_pair(2, 4)); - -// // txn = txn_manager.BeginTransaction(); -// // EXPECT_TRUE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(visible1.block)->GetHeader(), visible1.offset) == VisibilityType::OK); -// // EXPECT_TRUE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(visible2.block)->GetHeader(), visible2.offset) == VisibilityType::OK); -// // EXPECT_FALSE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(invisible1.block)->GetHeader(), invisible1.offset) == VisibilityType::OK); -// // EXPECT_FALSE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(invisible2.block)->GetHeader(), invisible2.offset) == VisibilityType::OK); -// // EXPECT_TRUE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(visible3.block)->GetHeader(), visible3.offset) == VisibilityType::OK); -// // txn_manager.AbortTransaction(txn); -// // } - -// } // namespace test -// } // namespace peloton - From 28ab50654866597f4a6a1de4b84e88197aac4dbf Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Wed, 11 Apr 2018 23:23:13 -0400 Subject: [PATCH 26/81] added logger switch --- .../timestamp_ordering_transaction_manager.cpp | 14 +++++++------- src/logging/log_buffer.cpp | 2 -- src/logging/wal_logger.cpp | 18 +++++++++++++++++- src/threadpool/logger_queue_pool.cpp | 7 +++++++ src/traffic_cop/traffic_cop.cpp | 1 + 5 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 843417a553b..d5d9249a9d0 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -496,7 +496,7 @@ void TimestampOrderingTransactionManager::PerformInsert( tile_group_header->SetIndirection(tuple_id, index_entry_ptr); /* TODO(graghura): remove placeholder, add logging switch */ - if(true) { + if(logging::LogManager::GetInstance().IsLoggingEnabled()) { if (values_buf != nullptr) { logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( @@ -605,7 +605,7 @@ void TimestampOrderingTransactionManager::PerformUpdate( current_txn->RecordUpdate(old_location); /* TODO(graghura): remove placeholder, add logging switch */ - if(true) { + if(logging::LogManager::GetInstance().IsLoggingEnabled()) { if (values_buf != nullptr) { logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( @@ -661,7 +661,7 @@ void TimestampOrderingTransactionManager::PerformUpdate( PL_ASSERT(tile_group_header->GetEndCommitId(tuple_id) == MAX_CID); /* TODO(graghura): remove placeholder, add logging switch */ - if(true) { + if(logging::LogManager::GetInstance().IsLoggingEnabled()) { if (values_buf != nullptr) { logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( @@ -782,7 +782,7 @@ void TimestampOrderingTransactionManager::PerformDelete( current_txn->RecordDelete(old_location); /* TODO(graghura): remove placeholder, add logging switch */ - if(true) { + if(logging::LogManager::GetInstance().IsLoggingEnabled()) { logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( @@ -842,7 +842,7 @@ void TimestampOrderingTransactionManager::PerformDelete( } /* TODO(graghura): remove placeholder, add logging switch */ - if(true) { + if(logging::LogManager::GetInstance().IsLoggingEnabled()) { logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( LogRecordType::TUPLE_DELETE, location, current_txn->GetEpochId(), @@ -1041,7 +1041,7 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( /* TODO(graghura): remove placeholder, add logging switch */ - if(true) { + if(logging::LogManager::GetInstance().IsLoggingEnabled()) { // no need to log read-only transactions if (!current_txn->IsReadOnly() && task_callback != nullptr) { @@ -1257,7 +1257,7 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( ResultType result = current_txn->GetResult(); /* TODO(graghura): remove placeholder, add logging switch */ - if(true) { + if(logging::LogManager::GetInstance().IsLoggingEnabled()) { // no need to log read-only transactions if (!current_txn->IsReadOnly() && task_callback != nullptr) { diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index cfe726fdcb2..c6e6bcf1508 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -49,7 +49,6 @@ void LogBuffer::WriteRecord(LogRecord &record) { for (uint32_t i = 0; i < record.GetNumValues(); i++) { //TODO(akanjani): Check if just copying the offset info will perform better values_array[i].SerializeTo(log_buffer_); - LOG_INFO("The value at offset %u is %d", i, type::ValuePeeker::PeekInteger(values_array[i])); } break; @@ -94,7 +93,6 @@ void LogBuffer::WriteRecord(LogRecord &record) { //TODO(akanjani): Check if just copying the offset info will perform better log_buffer_.WriteInt(((*offsets)[i]).first); values_array[i].SerializeTo(log_buffer_); - LOG_INFO("The value at offset %u is %d", ((*offsets)[i]).first, type::ValuePeeker::PeekInteger(values_array[i])); } break; diff --git a/src/logging/wal_logger.cpp b/src/logging/wal_logger.cpp index bb8ed187003..0352e76b997 100644 --- a/src/logging/wal_logger.cpp +++ b/src/logging/wal_logger.cpp @@ -8,7 +8,7 @@ #include "catalog/manager.h" #include "common/container_tuple.h" #include "storage/tile_group.h" - +#include "../include/logging/wal_logger.h" namespace peloton{ @@ -41,11 +41,27 @@ bool WalLogger::IsFlushNeeded(bool pending_buffers){ void WalLogger::FlushToDisk(){ + + if(disk_buffer_->GetSize()==0) + return; + std::ofstream *stream = LogManager::GetInstance().GetFileStream(); stream->write(disk_buffer_->GetData(), disk_buffer_->GetSize()); + + if(stream->fail()){ + PL_ASSERT(false); + } + stream->flush(); + + if(stream->fail()){ + PL_ASSERT(false); + } + + disk_buffer_->GetCopySerializedOutput().Reset(); + /* send out the callbacks */ while(!callbacks_.empty()){ auto callback = callbacks_.front(); diff --git a/src/threadpool/logger_queue_pool.cpp b/src/threadpool/logger_queue_pool.cpp index 99a3bcd3110..a643d624b66 100644 --- a/src/threadpool/logger_queue_pool.cpp +++ b/src/threadpool/logger_queue_pool.cpp @@ -2,6 +2,8 @@ #include "common/container/lock_free_queue.h" #include "logging/wal_logger.h" +#include + namespace peloton{ namespace threadpool{ @@ -15,6 +17,7 @@ void LoggerFunc(std::atomic_bool *is_running, LoggerQueue *logger_queue) { logging::WalLogger logger; + while (is_running->load() || !logger_queue->IsEmpty()) { logging::LogBuffer *log_buffer = nullptr; @@ -36,6 +39,10 @@ void LoggerFunc(std::atomic_bool *is_running, LoggerQueue *logger_queue) { } } + + + logger.FlushToDisk(); + } } diff --git a/src/traffic_cop/traffic_cop.cpp b/src/traffic_cop/traffic_cop.cpp index 2f75f214a2c..f9ade1ffaca 100644 --- a/src/traffic_cop/traffic_cop.cpp +++ b/src/traffic_cop/traffic_cop.cpp @@ -193,6 +193,7 @@ executor::ExecutionResult TrafficCop::ExecuteHelper( ResultType result = this->ExecuteStatementPlanGetResult(); if(result != ResultType::QUEUING) { + p_status_.m_result = result; task_callback_(task_callback_arg_); } }; From cf22bebe8b4e7fe4418e531c551bde0bd8cb5620 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Wed, 11 Apr 2018 23:43:57 -0400 Subject: [PATCH 27/81] Fix macro names --- src/logging/wal_logger.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/logging/wal_logger.cpp b/src/logging/wal_logger.cpp index 66855bbf271..af78ec29d9b 100644 --- a/src/logging/wal_logger.cpp +++ b/src/logging/wal_logger.cpp @@ -49,13 +49,13 @@ void WalLogger::FlushToDisk(){ stream->write(disk_buffer_->GetData(), disk_buffer_->GetSize()); if(stream->fail()){ - PL_ASSERT(false); + PELOTON_ASSERT(false); } stream->flush(); if(stream->fail()){ - PL_ASSERT(false); + PELOTON_ASSERT(false); } From ec7a306719c949f47e6b4ed5a790e6eba5882032 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Thu, 12 Apr 2018 09:14:39 -0400 Subject: [PATCH 28/81] Remove finished TODO --- src/concurrency/timestamp_ordering_transaction_manager.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 3ee14fd9837..9462ba6e557 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -465,7 +465,6 @@ void TimestampOrderingTransactionManager::PerformInsert( // Write down the head pointer's address in tile group header tile_group_header->SetIndirection(tuple_id, index_entry_ptr); - /* TODO(graghura): remove placeholder, add logging switch */ if(logging::LogManager::GetInstance().IsLoggingEnabled()) { if (values_buf != nullptr) { logging::LogRecord record = @@ -574,7 +573,6 @@ void TimestampOrderingTransactionManager::PerformUpdate( // Add the old tuple into the update set current_txn->RecordUpdate(old_location); - /* TODO(graghura): remove placeholder, add logging switch */ if(logging::LogManager::GetInstance().IsLoggingEnabled()) { if (values_buf != nullptr) { logging::LogRecord record = @@ -629,7 +627,6 @@ void TimestampOrderingTransactionManager::PerformUpdate( PELOTON_ASSERT(tile_group_header->GetBeginCommitId(tuple_id) == MAX_CID); PELOTON_ASSERT(tile_group_header->GetEndCommitId(tuple_id) == MAX_CID); - /* TODO(graghura): remove placeholder, add logging switch */ if(logging::LogManager::GetInstance().IsLoggingEnabled()) { if (values_buf != nullptr) { logging::LogRecord record = @@ -750,7 +747,6 @@ void TimestampOrderingTransactionManager::PerformDelete( current_txn->RecordDelete(old_location); - /* TODO(graghura): remove placeholder, add logging switch */ if(logging::LogManager::GetInstance().IsLoggingEnabled()) { logging::LogRecord record = @@ -810,7 +806,6 @@ void TimestampOrderingTransactionManager::PerformDelete( current_txn->RecordDelete(location); } - /* TODO(graghura): remove placeholder, add logging switch */ if(logging::LogManager::GetInstance().IsLoggingEnabled()) { logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( @@ -1009,7 +1004,6 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( - /* TODO(graghura): remove placeholder, add logging switch */ if(logging::LogManager::GetInstance().IsLoggingEnabled()) { // no need to log read-only transactions if (!current_txn->IsReadOnly() && task_callback != nullptr) { @@ -1225,7 +1219,6 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( ResultType result = current_txn->GetResult(); - /* TODO(graghura): remove placeholder, add logging switch */ if(logging::LogManager::GetInstance().IsLoggingEnabled()) { // no need to log read-only transactions if (!current_txn->IsReadOnly() && task_callback != nullptr) { From ae63217569c3179f94eadcdf5ff880d2f4802da7 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Thu, 12 Apr 2018 09:32:01 -0400 Subject: [PATCH 29/81] Add missing switch replacement --- src/concurrency/transaction_manager.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/concurrency/transaction_manager.cpp b/src/concurrency/transaction_manager.cpp index f95a39508c1..d51b83d57c5 100644 --- a/src/concurrency/transaction_manager.cpp +++ b/src/concurrency/transaction_manager.cpp @@ -72,10 +72,7 @@ TransactionContext *TransactionManager::BeginTransaction( .StartTimer(); } - - // TODO(graghura) - add logging switch here: - // placeholder for logging switch - if(true) { + if(logging::LogManager::GetInstance().IsLoggingEnabled()) { logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( LogRecordType::TRANSACTION_BEGIN, txn->GetEpochId(), From 087d2fe97a37e1b8e8324ef880f71a5d5d7694af Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Thu, 12 Apr 2018 15:01:45 -0400 Subject: [PATCH 30/81] Add tests for log buffers --- test/logging/log_buffer_test.cpp | 56 ++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 test/logging/log_buffer_test.cpp diff --git a/test/logging/log_buffer_test.cpp b/test/logging/log_buffer_test.cpp new file mode 100644 index 00000000000..25d61950082 --- /dev/null +++ b/test/logging/log_buffer_test.cpp @@ -0,0 +1,56 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// log_buffer_test.cpp +// +// Identification: test/logging/log_buffer_test.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "logging/log_buffer.h" +#include "common/harness.h" + +namespace peloton { +namespace test { + +//===--------------------------------------------------------------------===// +// Log Buffer Tests +//===--------------------------------------------------------------------===// + +class LogBufferTests : public PelotonTest {}; + +TEST_F(LogBufferTests, LogBufferTest) { + ItemPointer old_location(1, 8); + ItemPointer location(1, 16); + ItemPointer new_location(1, 24); + eid_t epoch_id = 3; + txn_id_t txn_id = 99; + cid_t commit_id = 98; + logging::LogBuffer log_buffer(3); + + logging::LogRecord insert_record = logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_INSERT, location, epoch_id, txn_id, commit_id); + logging::LogRecord update_record = logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_UPDATE, location, new_location, epoch_id, txn_id, + commit_id); + logging::LogRecord delete_record = logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_DELETE, old_location, epoch_id, txn_id, commit_id); + + EXPECT_EQ(log_buffer.GetThreshold(), 3); +// log_buffer.WriteRecord(insert_record); +// std::cout << log_buffer.GetSize() << std::endl; +// std::cout << log_buffer.GetData() << std::endl; +// +// log_buffer.WriteRecord(update_record); +// std::cout << log_buffer.GetSize() << std::endl; +// std::cout << log_buffer.GetData() << std::endl; +// +// log_buffer.WriteRecord(delete_record); +// std::cout << log_buffer.GetSize() << std::endl; +// std::cout << log_buffer.GetData() << std::endl; +} +} +} \ No newline at end of file From 7553a08036f3a48fbe5a9dcf9a2b3e706179e90d Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Thu, 12 Apr 2018 15:03:45 -0400 Subject: [PATCH 31/81] Create logging testing utility files --- test/include/logging/testing_logging_util.h | 253 +------------------- test/logging/testing_logging_util.cpp | 19 ++ 2 files changed, 21 insertions(+), 251 deletions(-) create mode 100644 test/logging/testing_logging_util.cpp diff --git a/test/include/logging/testing_logging_util.h b/test/include/logging/testing_logging_util.h index c2e3eb2c4b6..346f3247d6b 100644 --- a/test/include/logging/testing_logging_util.h +++ b/test/include/logging/testing_logging_util.h @@ -12,17 +12,8 @@ #include "common/harness.h" - -#include "logging/log_manager.h" -#include "common/logger.h" -#include "logging/loggers/wal_frontend_logger.h" -#include "logging/loggers/wal_backend_logger.h" -#include "logging/records/tuple_record.h" -#include "logging/records/transaction_record.h" -#include "storage/data_table.h" -#include "storage/tile.h" -#include -#include "../executor/testing_executor_util.h" +#include "logging/wal_logger.h" +#include "logging/log_buffer.h" #define INVALID_LOGGER_IDX UINT_MAX @@ -32,246 +23,6 @@ namespace test { //===--------------------------------------------------------------------===// // LoggingTests Utility //===--------------------------------------------------------------------===// -enum logging_op_type { - LOGGING_OP_PREPARE, - LOGGING_OP_BEGIN, - LOGGING_OP_INSERT, - LOGGING_OP_UPDATE, - LOGGING_OP_DELETE, - LOGGING_OP_COMMIT, - LOGGING_OP_ABORT, - LOGGING_OP_DONE, - LOGGING_OP_COLLECT, - LOGGING_OP_FLUSH, -}; - -class TestingLoggingUtil { - public: - static std::vector BuildTupleRecords( - std::vector> &tuples, - size_t tile_group_size, size_t table_tile_group_count); - - static std::vector BuildTupleRecordsForRestartTest( - std::vector> &tuples, - size_t tile_group_size, size_t table_tile_group_count, - int out_of_range_tuples, int delete_tuples); - - static std::vector> BuildTuples( - storage::DataTable *table, int num_rows, bool mutate, bool random); -}; - -// Operation of the logger -struct LoggingOperation { - logging_op_type op; - cid_t cid; - LoggingOperation(logging_op_type op_, cid_t cid_ = INVALID_CID) - : op(op_), cid(cid_){}; -}; - -// The schedule for logging execution -struct LoggingSchedule { - std::vector operations; - LoggingSchedule() {} -}; - -class LoggerId { - public: - unsigned int front; - unsigned int back; - LoggerId(unsigned int front_, unsigned int back_ = INVALID_LOGGER_IDX) - : front(front_), back(back_) {} - - LoggerId() : front(INVALID_LOGGER_IDX), back(INVALID_LOGGER_IDX) {} -}; - -// A thread wrapper that runs a frontend logger -class AbstractLoggingThread { - public: - AbstractLoggingThread(LoggingSchedule *sched, - logging::LogManager *log_manager_, - unsigned int frontend_id_, storage::DataTable *table_) - : frontend_id(frontend_id_), - schedule(sched), - log_manager(log_manager_), - cur_seq(0), - go(false), - table(table_) {} - - virtual void RunLoop() = 0; - - void MainLoop(); - - std::thread Run() { - return std::thread(&AbstractLoggingThread::RunLoop, this); - } - - virtual void ExecuteNext() = 0; - - virtual ~AbstractLoggingThread() {} - - unsigned int frontend_id = INVALID_LOGGER_IDX; - LoggingSchedule *schedule; - logging::LogManager *log_manager; - int cur_seq; - bool go; - storage::DataTable *table; -}; - -class FrontendLoggingThread : public AbstractLoggingThread { - public: - FrontendLoggingThread(LoggingSchedule *sched, - logging::LogManager *log_manager_, - unsigned int frontend_id_, storage::DataTable *table_) - : AbstractLoggingThread(sched, log_manager_, frontend_id_, table_) {} - - void RunLoop(); - - void ExecuteNext(); - - ~FrontendLoggingThread() {} - - logging::WriteAheadFrontendLogger *frontend_logger = nullptr; - - // result of committed cid. used by front end logger only - std::vector results; -}; - -class BackendLoggingThread : public AbstractLoggingThread { - public: - BackendLoggingThread(LoggingSchedule *sched, - logging::LogManager *log_manager_, - unsigned int frontend_id_, storage::DataTable *table_, - unsigned int backend_id_) - : AbstractLoggingThread(sched, log_manager_, frontend_id_, table_), - backend_id(backend_id_) {} - - void RunLoop(); - - void ExecuteNext(); - - ~BackendLoggingThread() {} - - logging::WriteAheadBackendLogger *backend_logger = nullptr; - - unsigned int backend_id = INVALID_LOGGER_IDX; -}; - -// Logging scheduler, to make life easier writing logging test -class LoggingScheduler { - public: - LoggingScheduler(size_t num_backend_logger_per_frontend, - unsigned int num_frontend_logger, - logging::LogManager *log_manager_, - storage::DataTable *table_) - : log_manager(log_manager_), - num_frontend_logger(num_frontend_logger), - num_backend_logger_per_frontend(num_backend_logger_per_frontend), - frontend_schedules(num_frontend_logger), - backend_schedules(num_backend_logger_per_frontend * - num_frontend_logger), - table(table_) {} - - void Prepare() { - backend_schedules[cur_id.back].operations.emplace_back(LOGGING_OP_PREPARE); - sequence[time++] = cur_id; - } - void Begin(cid_t cid) { - backend_schedules[cur_id.back].operations.emplace_back(LOGGING_OP_BEGIN, - cid); - sequence[time++] = cur_id; - } - void Insert(cid_t cid) { - backend_schedules[cur_id.back].operations.emplace_back(LOGGING_OP_INSERT, - cid); - sequence[time++] = cur_id; - } - void Delete(cid_t cid) { - backend_schedules[cur_id.back].operations.emplace_back(LOGGING_OP_DELETE, - cid); - sequence[time++] = cur_id; - } - void Update(cid_t cid) { - backend_schedules[cur_id.back].operations.emplace_back(LOGGING_OP_UPDATE, - cid); - sequence[time++] = cur_id; - } - void Abort(cid_t cid) { - backend_schedules[cur_id.back].operations.emplace_back(LOGGING_OP_ABORT, - cid); - sequence[time++] = cur_id; - } - void Commit(cid_t cid) { - backend_schedules[cur_id.back].operations.emplace_back(LOGGING_OP_COMMIT, - cid); - sequence[time++] = cur_id; - } - void Done(cid_t cid) { - backend_schedules[cur_id.back].operations.emplace_back(LOGGING_OP_DONE, - cid); - sequence[time++] = cur_id; - } - void Collect() { - frontend_schedules[cur_id.front].operations.emplace_back( - LOGGING_OP_COLLECT); - sequence[time++] = cur_id; - } - void Flush() { - frontend_schedules[cur_id.front].operations.emplace_back(LOGGING_OP_FLUSH); - sequence[time++] = cur_id; - } - - void Init(); - - void Cleanup(); - - void Run(); - - LoggingScheduler &BackendLogger(unsigned int frontend_idx, - unsigned int backend_idx) { - PELOTON_ASSERT(frontend_idx < frontend_schedules.size()); - PELOTON_ASSERT(backend_idx < num_backend_logger_per_frontend); - cur_id.front = frontend_idx; - cur_id.back = GetBackendLoggerId(frontend_idx, backend_idx); - return *this; - } - - LoggingScheduler &FrontendLogger(unsigned int frontend_idx) { - PELOTON_ASSERT(frontend_idx < frontend_schedules.size()); - cur_id.front = frontend_idx; - cur_id.back = INVALID_LOGGER_IDX; - return *this; - } - - int time = 0; - logging::LogManager *log_manager; - - unsigned int num_frontend_logger = 1; - unsigned int num_backend_logger_per_frontend = 2; - - // the logging schedules for frontend and backend loggers - std::vector frontend_schedules; - std::vector backend_schedules; - - // the logging threads for frontend and backend loggers - std::vector frontend_threads; - std::vector backend_threads; - - // the sequence of operation - std::map sequence; - - // current id of frontend & backend loggers - LoggerId cur_id = LoggerId(INVALID_LOGGER_IDX, INVALID_LOGGER_IDX); - - bool concurrent = false; - - storage::DataTable *table; - - private: - inline unsigned int GetBackendLoggerId(unsigned int frontend_idx, - unsigned int backend_idx) { - return frontend_idx * num_backend_logger_per_frontend + backend_idx; - } -}; } // namespace test } // namespace peloton diff --git a/test/logging/testing_logging_util.cpp b/test/logging/testing_logging_util.cpp new file mode 100644 index 00000000000..0d84ada103f --- /dev/null +++ b/test/logging/testing_logging_util.cpp @@ -0,0 +1,19 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// testing_logging_util.cpp +// +// Identification: test/index/testing_logging_util.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "logging/testing_logging_util.h" + +namespace peloton { + namespace test { + + } // namespace test +} // namespace peloton From 6da91ba4961a332b58681ecf730a5cb5ff49664a Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Thu, 12 Apr 2018 15:11:14 -0400 Subject: [PATCH 32/81] Add and fix headers --- src/include/logging/log_buffer.h | 13 ++++++++++++- src/include/logging/log_record.h | 4 +--- src/include/logging/wal_log_manager.h | 14 +++++++++++--- src/include/logging/wal_logger.h | 10 +++++++++- src/logging/log_buffer.cpp | 3 --- src/logging/wal_logger.cpp | 10 +++++++++- 6 files changed, 42 insertions(+), 12 deletions(-) diff --git a/src/include/logging/log_buffer.h b/src/include/logging/log_buffer.h index ea6cfb5c297..140e472359c 100644 --- a/src/include/logging/log_buffer.h +++ b/src/include/logging/log_buffer.h @@ -1,5 +1,16 @@ -#pragma once +//===----------------------------------------------------------------------===// +// +// Peloton +// +// log_buffer.h +// +// Identification: src/include/logging/log_buffer.h +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// +#pragma once #include "type/serializeio.h" #include "logging/log_record.h" diff --git a/src/include/logging/log_record.h b/src/include/logging/log_record.h index ce39199e771..f0ea21cc8d9 100644 --- a/src/include/logging/log_record.h +++ b/src/include/logging/log_record.h @@ -6,12 +6,10 @@ // // Identification: src/logging/log_record.h // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-18, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// - #pragma once - #include "common/internal_types.h" #include "common/item_pointer.h" #include "common/macros.h" diff --git a/src/include/logging/wal_log_manager.h b/src/include/logging/wal_log_manager.h index 33fef8f7a2d..d026c2d7ad1 100644 --- a/src/include/logging/wal_log_manager.h +++ b/src/include/logging/wal_log_manager.h @@ -1,7 +1,15 @@ - - +//===----------------------------------------------------------------------===// +// +// Peloton +// +// wal_log_manager.h +// +// Identification: src/logging/wal_log_manager.h +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// #pragma once - #include namespace peloton{ diff --git a/src/include/logging/wal_logger.h b/src/include/logging/wal_logger.h index d125efedbbc..4b44e938a8d 100644 --- a/src/include/logging/wal_logger.h +++ b/src/include/logging/wal_logger.h @@ -1,6 +1,14 @@ +//===----------------------------------------------------------------------===// // -// Created by Gandeevan R on 2/12/18. +// Peloton // +// wal_logger.h +// +// Identification: src/logging/wal_logger.h +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// #pragma once diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index 005c73b6099..9283c8286c0 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -1,7 +1,4 @@ - -#include "logging/log_record.h" -#include "logging/log_buffer.h" #include "logging/wal_logger.h" #include "catalog/manager.h" #include "common/container_tuple.h" diff --git a/src/logging/wal_logger.cpp b/src/logging/wal_logger.cpp index af78ec29d9b..d45d7620bb8 100644 --- a/src/logging/wal_logger.cpp +++ b/src/logging/wal_logger.cpp @@ -1,6 +1,14 @@ +//===----------------------------------------------------------------------===// // -// Created by Gandeevan R on 2/12/18. +// Peloton // +// wal_logger.cpp +// +// Identification: src/logging/wal_logger.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// #include #include "logging/wal_logger.h" From 9f79d880e0d9d043c67a785e552162c24f80db68 Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Fri, 13 Apr 2018 00:47:44 -0400 Subject: [PATCH 33/81] fixed the fucking race condition --- src/common/init.cpp | 3 +++ .../timestamp_ordering_transaction_manager.cpp | 1 + src/include/common/container/lock_free_queue.h | 2 +- src/include/common/notifiable_task.h | 1 + src/include/logging/wal_log_manager.h | 3 +-- src/include/threadpool/logger_queue_pool.h | 17 +++++------------ src/threadpool/logger_queue_pool.cpp | 4 ---- 7 files changed, 12 insertions(+), 19 deletions(-) diff --git a/src/common/init.cpp b/src/common/init.cpp index 5a0467551a3..e0f7d9f746b 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -25,6 +25,7 @@ #include "index/index.h" #include "settings/settings_manager.h" #include "threadpool/mono_queue_pool.h" +#include "threadpool/logger_queue_pool.h" #include "logging/wal_log_manager.h" namespace peloton { @@ -98,6 +99,8 @@ void PelotonInit::Initialize() { LOG_DEBUG("logging disabled"); } + threadpool::LoggerQueuePool::GetInstance().Startup(); + } void PelotonInit::Shutdown() { diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 513dc73cf0d..c3361f9ce6a 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -1076,6 +1076,7 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( database_id); } + return result; } diff --git a/src/include/common/container/lock_free_queue.h b/src/include/common/container/lock_free_queue.h index 6986fb7a367..53ecc0c9625 100644 --- a/src/include/common/container/lock_free_queue.h +++ b/src/include/common/container/lock_free_queue.h @@ -40,7 +40,7 @@ class LockFreeQueue { } void Enqueue(const ProducerToken &token, const T &item) { - queue_.enqueue(token, item); + queue_.enqueue(token, item) } // Dequeues one item, returning true if an item was found diff --git a/src/include/common/notifiable_task.h b/src/include/common/notifiable_task.h index 8f57df4ade2..c329a2e0caa 100644 --- a/src/include/common/notifiable_task.h +++ b/src/include/common/notifiable_task.h @@ -15,6 +15,7 @@ #include #include #include "common/event_util.h" +#include "common/logger.h" namespace peloton { diff --git a/src/include/logging/wal_log_manager.h b/src/include/logging/wal_log_manager.h index 33fef8f7a2d..c425b7f28b5 100644 --- a/src/include/logging/wal_log_manager.h +++ b/src/include/logging/wal_log_manager.h @@ -4,6 +4,7 @@ #include + namespace peloton{ namespace logging{ @@ -23,9 +24,7 @@ class LogManager{ enable_logging_ = true; return true; } - } - enable_logging_ = false; return false; } diff --git a/src/include/threadpool/logger_queue_pool.h b/src/include/threadpool/logger_queue_pool.h index 73b8a496c31..a8f48186259 100644 --- a/src/include/threadpool/logger_queue_pool.h +++ b/src/include/threadpool/logger_queue_pool.h @@ -47,7 +47,7 @@ class LoggerQueuePool { } ~LoggerQueuePool() { - if (is_running_ == true) + if(is_running_) Shutdown(); } @@ -55,14 +55,6 @@ class LoggerQueuePool { return next_token_++; } - - void Startup() { - for (size_t i = 0; i < num_workers_; i++) { - loggers_.emplace_back(LoggerFunc, &is_running_, &logger_queue_); - } - is_running_ = true; - } - void Shutdown() { is_running_ = false; @@ -74,19 +66,19 @@ class LoggerQueuePool { void SubmitLogBuffer(logging::LogBuffer *buffer) { if (is_running_ == false) - Startup(); + PL_ASSERT(false); + logger_queue_.Enqueue(std::move(buffer)); } void SubmitLogBuffer(int token, logging::LogBuffer *buffer) { if (is_running_ == false) - Startup(); + PL_ASSERT(false); int idx = (token)%kDefaultNumTokens; logger_queue_.Enqueue(*(log_tokens_[idx]), std::move(buffer)); } - static LoggerQueuePool &GetInstance() { static LoggerQueuePool logger_queue_pool(kDefaultLoggerPoolSize); return logger_queue_pool; @@ -101,6 +93,7 @@ class LoggerQueuePool { std::atomic_bool is_running_; std::atomic next_token_; + }; } // namespace threadpool diff --git a/src/threadpool/logger_queue_pool.cpp b/src/threadpool/logger_queue_pool.cpp index a643d624b66..cd4ffcb731b 100644 --- a/src/threadpool/logger_queue_pool.cpp +++ b/src/threadpool/logger_queue_pool.cpp @@ -1,7 +1,6 @@ #include "threadpool/logger_queue_pool.h" #include "common/container/lock_free_queue.h" #include "logging/wal_logger.h" - #include @@ -17,7 +16,6 @@ void LoggerFunc(std::atomic_bool *is_running, LoggerQueue *logger_queue) { logging::WalLogger logger; - while (is_running->load() || !logger_queue->IsEmpty()) { logging::LogBuffer *log_buffer = nullptr; @@ -39,8 +37,6 @@ void LoggerFunc(std::atomic_bool *is_running, LoggerQueue *logger_queue) { } } - - logger.FlushToDisk(); } From a6fb853c427c6fa1b163778c7b73f53e60afccfc Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Fri, 13 Apr 2018 01:34:39 -0400 Subject: [PATCH 34/81] fixed build issues --- src/include/common/container/lock_free_queue.h | 2 +- src/include/threadpool/logger_queue_pool.h | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/include/common/container/lock_free_queue.h b/src/include/common/container/lock_free_queue.h index 53ecc0c9625..6986fb7a367 100644 --- a/src/include/common/container/lock_free_queue.h +++ b/src/include/common/container/lock_free_queue.h @@ -40,7 +40,7 @@ class LockFreeQueue { } void Enqueue(const ProducerToken &token, const T &item) { - queue_.enqueue(token, item) + queue_.enqueue(token, item); } // Dequeues one item, returning true if an item was found diff --git a/src/include/threadpool/logger_queue_pool.h b/src/include/threadpool/logger_queue_pool.h index a8f48186259..69c2d1874a4 100644 --- a/src/include/threadpool/logger_queue_pool.h +++ b/src/include/threadpool/logger_queue_pool.h @@ -55,6 +55,15 @@ class LoggerQueuePool { return next_token_++; } + + void Startup() { + is_running_ = true; + + for (size_t i = 0; i < num_workers_; i++) { + loggers_.emplace_back(LoggerFunc, &is_running_, &logger_queue_); + } + } + void Shutdown() { is_running_ = false; From 7e59cd8f79a342b7dd49e789d101884b7fabfb51 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Fri, 13 Apr 2018 11:07:00 -0400 Subject: [PATCH 35/81] Add tile group for the log_buffer_test --- test/logging/log_buffer_test.cpp | 67 ++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 16 deletions(-) diff --git a/test/logging/log_buffer_test.cpp b/test/logging/log_buffer_test.cpp index 25d61950082..3fc5bd05504 100644 --- a/test/logging/log_buffer_test.cpp +++ b/test/logging/log_buffer_test.cpp @@ -10,6 +10,9 @@ // //===----------------------------------------------------------------------===// +#include "storage/tile_group_factory.h" +#include "catalog/schema.h" +#include "catalog/column.h" #include "logging/log_buffer.h" #include "common/harness.h" @@ -23,13 +26,44 @@ namespace test { class LogBufferTests : public PelotonTest {}; TEST_F(LogBufferTests, LogBufferTest) { - ItemPointer old_location(1, 8); - ItemPointer location(1, 16); - ItemPointer new_location(1, 24); + oid_t block = 1; + ItemPointer old_location(block, 8); + ItemPointer location(block, 16); + ItemPointer new_location(block, 24); eid_t epoch_id = 3; txn_id_t txn_id = 99; cid_t commit_id = 98; - logging::LogBuffer log_buffer(3); + oid_t database_id = 10; + oid_t table_id = 20; + oid_t tile_group_id = 30; + + logging::LogBuffer *log_buffer; + log_buffer = new logging::LogBuffer(logging::LogManager::GetInstance().GetTransactionBufferSize()); + + // TILES + std::vector tile_column_names; + std::vector> column_names; + + tile_column_names.push_back("INTEGER COL"); + column_names.push_back(tile_column_names); + + std::vector schemas; + std::vector columns; + + // SCHEMA + catalog::Column column1(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "A", true); + columns.push_back(column1); + + std::unique_ptr schema1(new catalog::Schema(columns)); + schemas.push_back(*schema1); + + std::map> column_map; + column_map[0] = std::make_pair(0, 0); + + std::shared_ptr tile_group(storage::TileGroupFactory::GetTileGroup( + database_id, table_id, tile_group_id, nullptr, schemas, column_map, 3)); + catalog::Manager::GetInstance().AddTileGroup(block, tile_group); logging::LogRecord insert_record = logging::LogRecordFactory::CreateTupleRecord( LogRecordType::TUPLE_INSERT, location, epoch_id, txn_id, commit_id); @@ -39,18 +73,19 @@ TEST_F(LogBufferTests, LogBufferTest) { logging::LogRecord delete_record = logging::LogRecordFactory::CreateTupleRecord( LogRecordType::TUPLE_DELETE, old_location, epoch_id, txn_id, commit_id); - EXPECT_EQ(log_buffer.GetThreshold(), 3); -// log_buffer.WriteRecord(insert_record); -// std::cout << log_buffer.GetSize() << std::endl; -// std::cout << log_buffer.GetData() << std::endl; -// -// log_buffer.WriteRecord(update_record); -// std::cout << log_buffer.GetSize() << std::endl; -// std::cout << log_buffer.GetData() << std::endl; -// -// log_buffer.WriteRecord(delete_record); -// std::cout << log_buffer.GetSize() << std::endl; -// std::cout << log_buffer.GetData() << std::endl; + EXPECT_EQ(log_buffer->GetThreshold(), logging::LogManager::GetInstance().GetTransactionBufferSize()); + log_buffer->WriteRecord(insert_record); + +// std::cout << log_buffer->GetSize() << std::endl; +// std::cout << log_buffer->GetData() << std::endl; + +// log_buffer->WriteRecord(update_record); +// std::cout << log_buffer->GetSize() << std::endl; +// std::cout << log_buffer->GetData() << std::endl; + +// log_buffer->WriteRecord(delete_record); +// std::cout << log_buffer->GetSize() << std::endl; +// std::cout << log_buffer->GetData() << std::endl; } } } \ No newline at end of file From dab6376f52301f9339f18581f3fecbff694ce84e Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Sun, 15 Apr 2018 21:13:28 -0400 Subject: [PATCH 36/81] Complete insert record tests --- test/logging/log_buffer_test.cpp | 47 ++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/test/logging/log_buffer_test.cpp b/test/logging/log_buffer_test.cpp index 3fc5bd05504..fe9b7a6692d 100644 --- a/test/logging/log_buffer_test.cpp +++ b/test/logging/log_buffer_test.cpp @@ -26,9 +26,9 @@ namespace test { class LogBufferTests : public PelotonTest {}; TEST_F(LogBufferTests, LogBufferTest) { - oid_t block = 1; + oid_t block = 1, offset = 16; ItemPointer old_location(block, 8); - ItemPointer location(block, 16); + ItemPointer location(block, offset); ItemPointer new_location(block, 24); eid_t epoch_id = 3; txn_id_t txn_id = 99; @@ -36,11 +36,12 @@ TEST_F(LogBufferTests, LogBufferTest) { oid_t database_id = 10; oid_t table_id = 20; oid_t tile_group_id = 30; + double value1 = 9.1, value2 = 9.2, value3 = 9.3; logging::LogBuffer *log_buffer; log_buffer = new logging::LogBuffer(logging::LogManager::GetInstance().GetTransactionBufferSize()); - // TILES +// TILES std::vector tile_column_names; std::vector> column_names; @@ -50,7 +51,7 @@ TEST_F(LogBufferTests, LogBufferTest) { std::vector schemas; std::vector columns; - // SCHEMA +// SCHEMA catalog::Column column1(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "A", true); columns.push_back(column1); @@ -62,22 +63,44 @@ TEST_F(LogBufferTests, LogBufferTest) { column_map[0] = std::make_pair(0, 0); std::shared_ptr tile_group(storage::TileGroupFactory::GetTileGroup( - database_id, table_id, tile_group_id, nullptr, schemas, column_map, 3)); + database_id, table_id, tile_group_id, nullptr, schemas, column_map, 3)); catalog::Manager::GetInstance().AddTileGroup(block, tile_group); logging::LogRecord insert_record = logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_INSERT, location, epoch_id, txn_id, commit_id); + LogRecordType::TUPLE_INSERT, location, epoch_id, txn_id, commit_id); logging::LogRecord update_record = logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_UPDATE, location, new_location, epoch_id, txn_id, - commit_id); + LogRecordType::TUPLE_UPDATE, location, new_location, epoch_id, txn_id, + commit_id); logging::LogRecord delete_record = logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_DELETE, old_location, epoch_id, txn_id, commit_id); + LogRecordType::TUPLE_DELETE, old_location, epoch_id, txn_id, commit_id); EXPECT_EQ(log_buffer->GetThreshold(), logging::LogManager::GetInstance().GetTransactionBufferSize()); - log_buffer->WriteRecord(insert_record); -// std::cout << log_buffer->GetSize() << std::endl; -// std::cout << log_buffer->GetData() << std::endl; + type::Value val1 = type::ValueFactory::GetDecimalValue(value1); + type::Value val2 = type::ValueFactory::GetDecimalValue(value2); + type::Value val3 = type::ValueFactory::GetDecimalValue(value3); + std::vector values; + values.push_back(val1); + values.push_back(val2); + values.push_back(val3); + insert_record.SetValuesArray(reinterpret_cast(values.data()), static_cast(values.size())); + log_buffer->WriteRecord(insert_record); + size_t buffer_size = sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t) * 3 + sizeof(int64_t) * 4 + + sizeof(double) * 3; + EXPECT_EQ(buffer_size, log_buffer->GetSize()); + ReferenceSerializeInput log_buffer_input(log_buffer->GetData(), log_buffer->GetSize()); + EXPECT_EQ(log_buffer->GetSize() - sizeof(int32_t), log_buffer_input.ReadInt()); + EXPECT_EQ(static_cast(LogRecordType::TUPLE_INSERT), log_buffer_input.ReadEnumInSingleByte()); + EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); + EXPECT_EQ(txn_id, log_buffer_input.ReadLong()); + EXPECT_EQ(commit_id, log_buffer_input.ReadLong()); + EXPECT_EQ(database_id, log_buffer_input.ReadLong()); + EXPECT_EQ(table_id, log_buffer_input.ReadLong()); + EXPECT_EQ(block, log_buffer_input.ReadLong()); + EXPECT_EQ(offset, log_buffer_input.ReadLong()); + EXPECT_EQ(value1, log_buffer_input.ReadDouble()); + EXPECT_EQ(value2, log_buffer_input.ReadDouble()); + EXPECT_EQ(value3, log_buffer_input.ReadDouble()); // log_buffer->WriteRecord(update_record); // std::cout << log_buffer->GetSize() << std::endl; From 196e60ca4467c087694c7d7edf93d71f46594448 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Mon, 16 Apr 2018 11:05:39 -0400 Subject: [PATCH 37/81] Complete update record tests --- test/logging/log_buffer_test.cpp | 52 ++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/test/logging/log_buffer_test.cpp b/test/logging/log_buffer_test.cpp index fe9b7a6692d..4c85b9421bc 100644 --- a/test/logging/log_buffer_test.cpp +++ b/test/logging/log_buffer_test.cpp @@ -26,10 +26,11 @@ namespace test { class LogBufferTests : public PelotonTest {}; TEST_F(LogBufferTests, LogBufferTest) { - oid_t block = 1, offset = 16; - ItemPointer old_location(block, 8); + oid_t block = 1, offset = 16, old_block = 1, old_offset = 8, new_block = 1, + new_offset = 24; + ItemPointer old_location(old_block, old_offset); ItemPointer location(block, offset); - ItemPointer new_location(block, 24); + ItemPointer new_location(new_block, new_offset); eid_t epoch_id = 3; txn_id_t txn_id = 99; cid_t commit_id = 98; @@ -37,6 +38,7 @@ TEST_F(LogBufferTests, LogBufferTest) { oid_t table_id = 20; oid_t tile_group_id = 30; double value1 = 9.1, value2 = 9.2, value3 = 9.3; + oid_t target_oid1 = 2, target_oid2 = 4, target_oid3 = 5; logging::LogBuffer *log_buffer; log_buffer = new logging::LogBuffer(logging::LogManager::GetInstance().GetTransactionBufferSize()); @@ -85,11 +87,29 @@ TEST_F(LogBufferTests, LogBufferTest) { values.push_back(val3); insert_record.SetValuesArray(reinterpret_cast(values.data()), static_cast(values.size())); log_buffer->WriteRecord(insert_record); - size_t buffer_size = sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t) * 3 + sizeof(int64_t) * 4 + + size_t buffer_size1 = sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t) * 3 + sizeof(int64_t) * 4 + sizeof(double) * 3; - EXPECT_EQ(buffer_size, log_buffer->GetSize()); + EXPECT_EQ(buffer_size1, log_buffer->GetSize()); + + expression::AbstractExpression *expr = nullptr; + planner::DerivedAttribute tmp_att(expr); + Target target1 = std::make_pair(target_oid1, tmp_att); + Target target2 = std::make_pair(target_oid2, tmp_att); + Target target3 = std::make_pair(target_oid3, tmp_att); + TargetList *target_list = new TargetList; + target_list->push_back(target1); + target_list->push_back(target2); + target_list->push_back(target3); + update_record.SetOffsetsArray(target_list); + update_record.SetValuesArray(reinterpret_cast(values.data()), static_cast(values.size())); + log_buffer->WriteRecord(update_record); + size_t buffer_size2 = sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t) * 3 + sizeof(int64_t) * 7 + + (sizeof(int32_t) + sizeof(double)) * 3; + EXPECT_EQ(buffer_size2, log_buffer->GetSize() - buffer_size1); + ReferenceSerializeInput log_buffer_input(log_buffer->GetData(), log_buffer->GetSize()); - EXPECT_EQ(log_buffer->GetSize() - sizeof(int32_t), log_buffer_input.ReadInt()); + // Insert record + EXPECT_EQ(buffer_size1 - sizeof(int32_t), log_buffer_input.ReadInt()); EXPECT_EQ(static_cast(LogRecordType::TUPLE_INSERT), log_buffer_input.ReadEnumInSingleByte()); EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); EXPECT_EQ(txn_id, log_buffer_input.ReadLong()); @@ -102,6 +122,26 @@ TEST_F(LogBufferTests, LogBufferTest) { EXPECT_EQ(value2, log_buffer_input.ReadDouble()); EXPECT_EQ(value3, log_buffer_input.ReadDouble()); + // Update Record + EXPECT_EQ(buffer_size2 - sizeof(int32_t), log_buffer_input.ReadInt()); + EXPECT_EQ(static_cast(LogRecordType::TUPLE_UPDATE), log_buffer_input.ReadEnumInSingleByte()); + EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); + EXPECT_EQ(txn_id, log_buffer_input.ReadLong()); + EXPECT_EQ(commit_id, log_buffer_input.ReadLong()); + EXPECT_EQ(database_id, log_buffer_input.ReadLong()); + EXPECT_EQ(table_id, log_buffer_input.ReadLong()); + EXPECT_EQ(block, log_buffer_input.ReadLong()); + EXPECT_EQ(offset, log_buffer_input.ReadLong()); + EXPECT_EQ(new_block, log_buffer_input.ReadLong()); + EXPECT_EQ(new_offset, log_buffer_input.ReadLong()); + EXPECT_EQ(update_record.GetNumValues(), log_buffer_input.ReadLong()); + EXPECT_EQ(target_oid1, log_buffer_input.ReadInt()); + EXPECT_EQ(value1, log_buffer_input.ReadDouble()); + EXPECT_EQ(target_oid2, log_buffer_input.ReadInt()); + EXPECT_EQ(value2, log_buffer_input.ReadDouble()); + EXPECT_EQ(target_oid3, log_buffer_input.ReadInt()); + EXPECT_EQ(value3, log_buffer_input.ReadDouble()); + // log_buffer->WriteRecord(update_record); // std::cout << log_buffer->GetSize() << std::endl; // std::cout << log_buffer->GetData() << std::endl; From 66160c604fffcf3a2e3d00f23655e812eb52deb5 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Mon, 16 Apr 2018 21:26:39 -0400 Subject: [PATCH 38/81] fix included header files --- src/concurrency/timestamp_ordering_transaction_manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 9462ba6e557..cffaa7ae4bf 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -11,7 +11,7 @@ //===----------------------------------------------------------------------===// #include -#include +#include "include/threadpool/mono_queue_pool.h" #include "concurrency/timestamp_ordering_transaction_manager.h" #include "catalog/manager.h" From 8858d19572ec9ff5718c60ce2ce2c227f984e1cd Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Mon, 16 Apr 2018 21:51:25 -0400 Subject: [PATCH 39/81] Complete delete record tests --- test/logging/log_buffer_test.cpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/test/logging/log_buffer_test.cpp b/test/logging/log_buffer_test.cpp index 4c85b9421bc..02c355373cc 100644 --- a/test/logging/log_buffer_test.cpp +++ b/test/logging/log_buffer_test.cpp @@ -107,6 +107,10 @@ TEST_F(LogBufferTests, LogBufferTest) { (sizeof(int32_t) + sizeof(double)) * 3; EXPECT_EQ(buffer_size2, log_buffer->GetSize() - buffer_size1); + log_buffer->WriteRecord(delete_record); + size_t buffer_size3 = sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t) * 3 + sizeof(int64_t) * 4; + EXPECT_EQ(buffer_size3, log_buffer->GetSize() - buffer_size1 - buffer_size2); + ReferenceSerializeInput log_buffer_input(log_buffer->GetData(), log_buffer->GetSize()); // Insert record EXPECT_EQ(buffer_size1 - sizeof(int32_t), log_buffer_input.ReadInt()); @@ -142,13 +146,16 @@ TEST_F(LogBufferTests, LogBufferTest) { EXPECT_EQ(target_oid3, log_buffer_input.ReadInt()); EXPECT_EQ(value3, log_buffer_input.ReadDouble()); -// log_buffer->WriteRecord(update_record); -// std::cout << log_buffer->GetSize() << std::endl; -// std::cout << log_buffer->GetData() << std::endl; - -// log_buffer->WriteRecord(delete_record); -// std::cout << log_buffer->GetSize() << std::endl; -// std::cout << log_buffer->GetData() << std::endl; + // Delete record + EXPECT_EQ(buffer_size3 - sizeof(int32_t), log_buffer_input.ReadInt()); + EXPECT_EQ(static_cast(LogRecordType::TUPLE_DELETE), log_buffer_input.ReadEnumInSingleByte()); + EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); + EXPECT_EQ(txn_id, log_buffer_input.ReadLong()); + EXPECT_EQ(commit_id, log_buffer_input.ReadLong()); + EXPECT_EQ(database_id, log_buffer_input.ReadLong()); + EXPECT_EQ(table_id, log_buffer_input.ReadLong()); + EXPECT_EQ(old_block, log_buffer_input.ReadLong()); + EXPECT_EQ(old_offset, log_buffer_input.ReadLong()); } } } \ No newline at end of file From 65fca27861c06c7fb6603bfd6348d454a887eb45 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Fri, 20 Apr 2018 12:52:09 -0400 Subject: [PATCH 40/81] Add logging tests --- test/logging/logging_test.cpp | 92 +++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 test/logging/logging_test.cpp diff --git a/test/logging/logging_test.cpp b/test/logging/logging_test.cpp new file mode 100644 index 00000000000..c966d67bd9b --- /dev/null +++ b/test/logging/logging_test.cpp @@ -0,0 +1,92 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// logging_test.cpp +// +// Identification: test/logging/logging_test.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include /* libpqxx is used to instantiate C++ client */ +#include "logging/log_buffer.h" +#include "common/harness.h" +#include "gtest/gtest.h" +#include "network/peloton_server.h" +#include "network/postgres_protocol_handler.h" +#include "util/string_util.h" +#include "network/connection_handle_factory.h" + +namespace peloton { +namespace test { + +//===--------------------------------------------------------------------===// +// Logging Tests +//===--------------------------------------------------------------------===// + +class LoggingTests : public PelotonTest {}; + +void *LoggingTest(int port) { + try { + // forcing the factory to generate jdbc protocol handler + pqxx::connection C(StringUtil::Format( + "host=127.0.0.1 port=%d user=default_database sslmode=disable", port)); + LOG_INFO("[LoggingTest] Connected to %s", C.dbname()); + pqxx::work txn1(C); + + peloton::network::ConnectionHandle *conn = + peloton::network::ConnectionHandleFactory::GetInstance().ConnectionHandleAt( + peloton::network::PelotonServer::recent_connfd).get(); + + //Check type of protocol handler + network::PostgresProtocolHandler* handler = + dynamic_cast(conn->GetProtocolHandler().get()); + + EXPECT_NE(handler, nullptr); + + // create table and insert some data + txn1.exec("DROP TABLE IF EXISTS employee;"); + txn1.exec("CREATE TABLE employee(id INT, name VARCHAR(100));"); + txn1.commit(); + + pqxx::work txn2(C); + txn2.exec("INSERT INTO employee VALUES (1, 'Aaron Tian');"); + txn2.exec("INSERT INTO employee VALUES (2, 'Gandeevan Raghuraman');"); + txn2.exec("INSERT INTO employee VALUES (3, 'Anirudh Kanjani');"); + + // test prepare statement + C.prepare("searchstmt", "SELECT name FROM employee WHERE id=$1;"); + // invocation as in variable binding + txn2.prepared("searchstmt")(1).exec(); + txn2.commit(); + + } catch (const std::exception &e) { + LOG_INFO("[LoggingTest] Exception occurred: %s", e.what()); + EXPECT_TRUE(false); + } + return NULL; +} + +TEST_F(LoggingTests, LoggingTest) { + peloton::PelotonInit::Initialize(); + LOG_INFO("Server initialized"); + peloton::network::PelotonServer server; + + int port = 15721; + try { + server.SetPort(port); + server.SetupServer(); + } catch (peloton::ConnectionException &exception) { + LOG_INFO("[LaunchServer] exception when launching server"); + } + std::thread serverThread([&]() { server.ServerLoop(); }); + LoggingTest(port); + server.Close(); + serverThread.join(); + peloton::PelotonInit::Shutdown(); + LOG_DEBUG("Peloton has shut down"); +} +} +} \ No newline at end of file From 6d3f61c08d7198df83daac6b820f55b34d3f4b07 Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Sun, 22 Apr 2018 16:32:41 -0400 Subject: [PATCH 41/81] minor update bug fix --- src/concurrency/timestamp_ordering_transaction_manager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index cffaa7ae4bf..98d42fd98e1 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -631,7 +631,7 @@ void TimestampOrderingTransactionManager::PerformUpdate( if (values_buf != nullptr) { logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_UPDATE, location, INVALID_ITEMPOINTER, + LogRecordType::TUPLE_UPDATE, location, location, current_txn->GetEpochId(), current_txn->GetTransactionId(), current_txn->GetCommitId()); @@ -1256,4 +1256,4 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( } } // namespace storage -} // namespace peloton \ No newline at end of file +} // namespace peloton From f2f59941a3c6a92b2cb1db13e351b59dfdd14fd6 Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Sun, 22 Apr 2018 17:36:15 -0400 Subject: [PATCH 42/81] minor update bug fix --- src/concurrency/timestamp_ordering_transaction_manager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index c3361f9ce6a..a508001b79d 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -665,7 +665,7 @@ void TimestampOrderingTransactionManager::PerformUpdate( if (values_buf != nullptr) { logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_UPDATE, location, INVALID_ITEMPOINTER, + LogRecordType::TUPLE_UPDATE, location, location, current_txn->GetEpochId(), current_txn->GetTransactionId(), current_txn->GetCommitId()); @@ -1295,4 +1295,4 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( } } // namespace storage -} // namespace peloton \ No newline at end of file +} // namespace peloton From 54953837ff6d5ffa75c6ba7a4c27049697d73787 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Thu, 26 Apr 2018 11:38:22 -0400 Subject: [PATCH 43/81] Change the log file name to avoid conflicts --- src/include/logging/wal_log_manager.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/logging/wal_log_manager.h b/src/include/logging/wal_log_manager.h index d026c2d7ad1..0224e2108f7 100644 --- a/src/include/logging/wal_log_manager.h +++ b/src/include/logging/wal_log_manager.h @@ -25,7 +25,7 @@ class LogManager{ directory_ = logging_dir; if(!logger_ofstream_.is_open()){ - logger_ofstream_.open(logging_dir+"/log", std::ofstream::out | std::ofstream::app); + logger_ofstream_.open(logging_dir+"/log_file", std::ofstream::out | std::ofstream::app); if(!logger_ofstream_.fail()) { enable_logging_ = true; From e9cd27bcf6df54641bd4194ecdcd4c0b54da3c5a Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Fri, 27 Apr 2018 06:09:31 -0400 Subject: [PATCH 44/81] Add the flexibility to choose the directory and file for logs --- src/common/init.cpp | 4 ++-- src/include/common/init.h | 3 ++- src/include/logging/wal_log_manager.h | 6 +++--- test/logging/logging_test.cpp | 9 ++------- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/common/init.cpp b/src/common/init.cpp index cf35696d23e..c444937047b 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -31,7 +31,7 @@ namespace peloton { ThreadPool thread_pool; -void PelotonInit::Initialize() { +void PelotonInit::Initialize(std::string log_dir, std::string log_file) { CONNECTION_THREAD_COUNT = settings::SettingsManager::GetInt( settings::SettingId::connection_thread_count); LOGGING_THREAD_COUNT = 1; @@ -93,7 +93,7 @@ void PelotonInit::Initialize() { StatementCacheManager::Init(); // TODO(gandeevan): start logger thread - if(logging::LogManager::GetInstance().init("/tmp")) { + if(logging::LogManager::GetInstance().init(log_dir, log_file)) { LOG_DEBUG("logging enabled"); } else { LOG_DEBUG("logging disabled"); diff --git a/src/include/common/init.h b/src/include/common/init.h index 8e386d46a2e..74a8f45c654 100644 --- a/src/include/common/init.h +++ b/src/include/common/init.h @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +#include #pragma once namespace peloton { @@ -24,7 +25,7 @@ extern ThreadPool thread_pool; class PelotonInit { public: - static void Initialize(); + static void Initialize(std::string log_dir = "/tmp", std::string log_file = "log_file"); static void Shutdown(); diff --git a/src/include/logging/wal_log_manager.h b/src/include/logging/wal_log_manager.h index 0224e2108f7..cc1e5e1876d 100644 --- a/src/include/logging/wal_log_manager.h +++ b/src/include/logging/wal_log_manager.h @@ -21,11 +21,11 @@ class LogManager{ LogManager() {} ~LogManager() {} - inline bool init(std::string logging_dir) { - directory_ = logging_dir; + inline bool init(std::string log_dir, std::string log_file) { + directory_ = log_dir; if(!logger_ofstream_.is_open()){ - logger_ofstream_.open(logging_dir+"/log_file", std::ofstream::out | std::ofstream::app); + logger_ofstream_.open(log_dir+"/"+log_file, std::ofstream::out | std::ofstream::app); if(!logger_ofstream_.fail()) { enable_logging_ = true; diff --git a/test/logging/logging_test.cpp b/test/logging/logging_test.cpp index c966d67bd9b..b462f014ad9 100644 --- a/test/logging/logging_test.cpp +++ b/test/logging/logging_test.cpp @@ -56,12 +56,6 @@ void *LoggingTest(int port) { txn2.exec("INSERT INTO employee VALUES (2, 'Gandeevan Raghuraman');"); txn2.exec("INSERT INTO employee VALUES (3, 'Anirudh Kanjani');"); - // test prepare statement - C.prepare("searchstmt", "SELECT name FROM employee WHERE id=$1;"); - // invocation as in variable binding - txn2.prepared("searchstmt")(1).exec(); - txn2.commit(); - } catch (const std::exception &e) { LOG_INFO("[LoggingTest] Exception occurred: %s", e.what()); EXPECT_TRUE(false); @@ -70,7 +64,7 @@ void *LoggingTest(int port) { } TEST_F(LoggingTests, LoggingTest) { - peloton::PelotonInit::Initialize(); + peloton::PelotonInit::Initialize("/tmp", "test_log_file"); LOG_INFO("Server initialized"); peloton::network::PelotonServer server; @@ -87,6 +81,7 @@ TEST_F(LoggingTests, LoggingTest) { serverThread.join(); peloton::PelotonInit::Shutdown(); LOG_DEBUG("Peloton has shut down"); + } } } \ No newline at end of file From d15d52274c6256fbb78f5ffafb693accc1db2b41 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Fri, 27 Apr 2018 11:32:39 -0400 Subject: [PATCH 45/81] Fix the race condition --- src/include/threadpool/logger_queue_pool.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/include/threadpool/logger_queue_pool.h b/src/include/threadpool/logger_queue_pool.h index 73b8a496c31..1d59053420b 100644 --- a/src/include/threadpool/logger_queue_pool.h +++ b/src/include/threadpool/logger_queue_pool.h @@ -42,7 +42,7 @@ class LoggerQueuePool { : logger_queue_(kDefaultLoggerQueueSize), num_workers_(num_workers), is_running_(false) { - + LOG_INFO("aaron: LoggerQueuePool"); logger_queue_.GenerateTokens(log_tokens_, kDefaultNumTokens); } @@ -57,13 +57,15 @@ class LoggerQueuePool { void Startup() { + LOG_INFO("aaron: LoggerQueuePoolStartup"); + is_running_ = true; for (size_t i = 0; i < num_workers_; i++) { loggers_.emplace_back(LoggerFunc, &is_running_, &logger_queue_); } - is_running_ = true; } void Shutdown() { + LOG_INFO("aaron: LoggerQueuePoolShutdown"); is_running_ = false; for (auto &logger : loggers_) { From a3fe33f55f55e4aa91e050e9bcb9a597bc3a6b49 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Fri, 27 Apr 2018 11:34:12 -0400 Subject: [PATCH 46/81] Enable the choice of turn on or off the logging when initializing --- src/common/init.cpp | 4 ++-- src/include/common/init.h | 2 +- src/include/logging/wal_log_manager.h | 7 ++++++- src/logging/log_buffer.cpp | 3 +++ src/logging/wal_logger.cpp | 5 ++++- src/threadpool/logger_queue_pool.cpp | 1 + 6 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/common/init.cpp b/src/common/init.cpp index c444937047b..3c6884c8cdf 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -31,7 +31,7 @@ namespace peloton { ThreadPool thread_pool; -void PelotonInit::Initialize(std::string log_dir, std::string log_file) { +void PelotonInit::Initialize(std::string log_dir, std::string log_file, bool enable_logging) { CONNECTION_THREAD_COUNT = settings::SettingsManager::GetInt( settings::SettingId::connection_thread_count); LOGGING_THREAD_COUNT = 1; @@ -93,7 +93,7 @@ void PelotonInit::Initialize(std::string log_dir, std::string log_file) { StatementCacheManager::Init(); // TODO(gandeevan): start logger thread - if(logging::LogManager::GetInstance().init(log_dir, log_file)) { + if(logging::LogManager::GetInstance().init(log_dir, log_file, enable_logging)) { LOG_DEBUG("logging enabled"); } else { LOG_DEBUG("logging disabled"); diff --git a/src/include/common/init.h b/src/include/common/init.h index 74a8f45c654..49fd5c60498 100644 --- a/src/include/common/init.h +++ b/src/include/common/init.h @@ -25,7 +25,7 @@ extern ThreadPool thread_pool; class PelotonInit { public: - static void Initialize(std::string log_dir = "/tmp", std::string log_file = "log_file"); + static void Initialize(std::string log_dir = "/tmp", std::string log_file = "log_file", bool enable_logging = true); static void Shutdown(); diff --git a/src/include/logging/wal_log_manager.h b/src/include/logging/wal_log_manager.h index cc1e5e1876d..a223b26bdbb 100644 --- a/src/include/logging/wal_log_manager.h +++ b/src/include/logging/wal_log_manager.h @@ -21,7 +21,12 @@ class LogManager{ LogManager() {} ~LogManager() {} - inline bool init(std::string log_dir, std::string log_file) { + inline bool init(std::string log_dir, std::string log_file, bool enable_logging) { + LOG_INFO("aaron: LogManagerInit"); + if (enable_logging == false) { + enable_logging_ = false; + return false; + } directory_ = log_dir; if(!logger_ofstream_.is_open()){ diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index 9283c8286c0..77a7fa9e099 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -18,6 +18,8 @@ void LogBuffer::WriteRecord(LogRecord &record) { size_t start = log_buffer_.Position(); log_buffer_.WriteInt(0); + LOG_INFO("aaron: WriteRecord %ld", start); + LogRecordType type = record.GetType(); log_buffer_.WriteEnumInSingleByte( static_cast(type)); @@ -110,6 +112,7 @@ void LogBuffer::WriteRecord(LogRecord &record) { // XXX: We rely on the fact that the serializer treat a int32_t as 4 bytes int32_t length = log_buffer_.Position() - start - sizeof(int32_t); log_buffer_.WriteIntAt(start, length); + LOG_INFO("aaron: RecordLength %d", length); } } diff --git a/src/logging/wal_logger.cpp b/src/logging/wal_logger.cpp index d45d7620bb8..115ac0ace1e 100644 --- a/src/logging/wal_logger.cpp +++ b/src/logging/wal_logger.cpp @@ -50,8 +50,10 @@ bool WalLogger::IsFlushNeeded(bool pending_buffers){ void WalLogger::FlushToDisk(){ - if(disk_buffer_->GetSize()==0) + if(disk_buffer_->GetSize()==0){ + LOG_INFO("aaron: disk_buffer_->GetSize()==0"); return; + } std::ofstream *stream = LogManager::GetInstance().GetFileStream(); stream->write(disk_buffer_->GetData(), disk_buffer_->GetSize()); @@ -60,6 +62,7 @@ void WalLogger::FlushToDisk(){ PELOTON_ASSERT(false); } + LOG_INFO("aaron: stream->flush()"); stream->flush(); if(stream->fail()){ diff --git a/src/threadpool/logger_queue_pool.cpp b/src/threadpool/logger_queue_pool.cpp index a643d624b66..8f4e2b7722c 100644 --- a/src/threadpool/logger_queue_pool.cpp +++ b/src/threadpool/logger_queue_pool.cpp @@ -11,6 +11,7 @@ namespace threadpool{ void LoggerFunc(std::atomic_bool *is_running, LoggerQueue *logger_queue) { + LOG_INFO("aaron: LoggerFunc"); constexpr auto kMinPauseTime = std::chrono::microseconds(1); constexpr auto kMaxPauseTime = std::chrono::microseconds(1000); auto pause_time = kMinPauseTime; From 6a0ec6ac814bc83969da9f907c43d6cdb6315ed2 Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Fri, 4 May 2018 00:51:57 -0400 Subject: [PATCH 47/81] Moving buffer sizes to pg_settings --- src/include/logging/log_record.h | 12 ------------ src/include/logging/wal_log_manager.h | 16 +++++++++------- src/include/settings/settings.h | 12 ++++++++++++ 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/include/logging/log_record.h b/src/include/logging/log_record.h index f0ea21cc8d9..6b48a0e2127 100644 --- a/src/include/logging/log_record.h +++ b/src/include/logging/log_record.h @@ -128,18 +128,6 @@ class LogRecordFactory { return LogRecord(log_type, old_pos, pos, current_eid, txn_id, current_cid); } -// -// static LogRecord CreateTxnRecord(const LogRecordType log_type, const cid_t commit_id) { -// PELOTON_ASSERT(log_type == LogRecordType::TRANSACTION_BEGIN || -// log_type == LogRecordType::TRANSACTION_COMMIT); -// return LogRecord(log_type, INVALID_ITEMPOINTER, INVALID_EID, commit_id); -// } -// -// static LogRecord CreateEpochRecord(const LogRecordType log_type, const eid_t epoch_id) { -// PELOTON_ASSERT(log_type == LogRecordType::EPOCH_BEGIN || -// log_type == LogRecordType::EPOCH_END); -// return LogRecord(log_type, INVALID_ITEMPOINTER, epoch_id, INVALID_CID); -// } }; } // namespace logging diff --git a/src/include/logging/wal_log_manager.h b/src/include/logging/wal_log_manager.h index 1b88024ab06..f45a200cb07 100644 --- a/src/include/logging/wal_log_manager.h +++ b/src/include/logging/wal_log_manager.h @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #pragma once #include +#include namespace peloton{ @@ -47,8 +48,13 @@ class LogManager{ return log_manager; } - inline static size_t GetTransactionBufferSize() { return transaction_buffer_threshold_; } - inline static size_t GetLoggerBufferSize() { return logger_buffer_threshold_; } + inline static size_t GetTransactionBufferSize() { + return settings::SettingsManager::GetInt(settings::SettingId::log_buffer_size); + } + + inline static size_t GetLoggerBufferSize() { + return settings::SettingsManager::GetInt(settings::SettingId::transaction_buffer_size); + } inline bool IsLoggingEnabled() { return enable_logging_; } inline std::string GetDirectory() { return directory_; } @@ -61,11 +67,7 @@ class LogManager{ private: - - const static size_t logger_buffer_threshold_ = 1024 * 512; // 512 KB - const static size_t transaction_buffer_threshold_ = 1024 * 16; // 16 KB - - // NOTE: ofstream is not thread safe, might need to change it if more than one logger thread is used + // NOTE: ofstream is not thread safe, might need to change it if more than one logger thread is used bool enable_logging_; std::string directory_; std::ofstream logger_ofstream_; diff --git a/src/include/settings/settings.h b/src/include/settings/settings.h index e90cb78b7da..27e1fab33c9 100644 --- a/src/include/settings/settings.h +++ b/src/include/settings/settings.h @@ -116,6 +116,18 @@ SETTING_int(gc_num_threads, 1, 128, true, true) +SETTING_int(log_buffer_size, + "The default log buffer size (default: 512KB)", + 512 * 1024, + 1024, 1024 * 1024, + true, true) + +SETTING_int(transaction_buffer_size, + "The default log buffer size for each transaction (default: 16KB)", + 16 * 1024, + 512, 256 * 1024, + true, true) + //===----------------------------------------------------------------------===// // WRITE AHEAD LOG //===----------------------------------------------------------------------===// From 4a7ed2d2411ddb211c44b8c0afa597ae00b9d35a Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Fri, 4 May 2018 02:13:03 -0400 Subject: [PATCH 48/81] buffer sizes moved to settings --- src/include/logging/wal_log_manager.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/include/logging/wal_log_manager.h b/src/include/logging/wal_log_manager.h index f45a200cb07..5741b28af3c 100644 --- a/src/include/logging/wal_log_manager.h +++ b/src/include/logging/wal_log_manager.h @@ -49,11 +49,11 @@ class LogManager{ } inline static size_t GetTransactionBufferSize() { - return settings::SettingsManager::GetInt(settings::SettingId::log_buffer_size); + return settings::SettingsManager::GetInt(settings::SettingId::transaction_buffer_size); } inline static size_t GetLoggerBufferSize() { - return settings::SettingsManager::GetInt(settings::SettingId::transaction_buffer_size); + return settings::SettingsManager::GetInt(settings::SettingId::log_buffer_size); } inline bool IsLoggingEnabled() { return enable_logging_; } @@ -76,4 +76,4 @@ class LogManager{ }; } -} \ No newline at end of file +} From e6a37e4fe053fd56062f5ac43d58acad9640f5f9 Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Fri, 4 May 2018 11:43:42 -0400 Subject: [PATCH 49/81] moving log directory and file paths to settings --- src/common/init.cpp | 1 - src/include/common/init.h | 5 ++++- src/include/logging/wal_log_manager.h | 17 +++++++++++++++-- src/include/settings/settings.h | 10 ++++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/common/init.cpp b/src/common/init.cpp index 9fcd14489ec..eb6a2792ca1 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -93,7 +93,6 @@ void PelotonInit::Initialize(std::string log_dir, std::string log_file, bool ena // Initialize the Statement Cache Manager StatementCacheManager::Init(); - // TODO(gandeevan): start logger thread if(logging::LogManager::GetInstance().init(log_dir, log_file, enable_logging)) { LOG_DEBUG("logging enabled"); } else { diff --git a/src/include/common/init.h b/src/include/common/init.h index 49fd5c60498..dc7ed223629 100644 --- a/src/include/common/init.h +++ b/src/include/common/init.h @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include +#include #pragma once namespace peloton { @@ -25,7 +26,9 @@ extern ThreadPool thread_pool; class PelotonInit { public: - static void Initialize(std::string log_dir = "/tmp", std::string log_file = "log_file", bool enable_logging = true); + static void Initialize(std::string log_dir = settings::SettingsManager::GetString(settings::SettingId::log_directory_name), + std::string log_file = settings::SettingsManager::GetString(settings::SettingId::log_file_name), + bool enable_logging = true); static void Shutdown(); diff --git a/src/include/logging/wal_log_manager.h b/src/include/logging/wal_log_manager.h index f45a200cb07..554e8ee2b3d 100644 --- a/src/include/logging/wal_log_manager.h +++ b/src/include/logging/wal_log_manager.h @@ -12,6 +12,7 @@ #pragma once #include #include +#include "util/file_util.h" namespace peloton{ @@ -30,9 +31,20 @@ class LogManager{ return false; } directory_ = log_dir; + log_file = log_file; + + if (!peloton::FileUtil::Exists(log_dir)) { + boost::filesystem::path dir(log_dir.c_str()); + try { + boost::filesystem::create_directory(dir); + } + catch (boost::filesystem::filesystem_error &e) { + LOG_DEBUG("Failed to create log directory %s. Reason %s", log_dir.c_str(), e.what()); + } + } if(!logger_ofstream_.is_open()){ - logger_ofstream_.open(log_dir+"/"+log_file, std::ofstream::out | std::ofstream::app); + logger_ofstream_.open(log_dir + "/" + log_file, std::ofstream::out | std::ofstream::app); if(!logger_ofstream_.fail()) { enable_logging_ = true; @@ -58,7 +70,7 @@ class LogManager{ inline bool IsLoggingEnabled() { return enable_logging_; } inline std::string GetDirectory() { return directory_; } - inline std::string GetLogFilePath() { return directory_+"/tmp"; } + inline std::string GetLogFilePath() { return directory_ + log_file_; } inline std::ofstream *GetFileStream() { return &logger_ofstream_; } // TODO(gandeevan): support StartLogging and StopLogging @@ -70,6 +82,7 @@ class LogManager{ // NOTE: ofstream is not thread safe, might need to change it if more than one logger thread is used bool enable_logging_; std::string directory_; + std::string log_file_; std::ofstream logger_ofstream_; diff --git a/src/include/settings/settings.h b/src/include/settings/settings.h index 27e1fab33c9..a8bc5272beb 100644 --- a/src/include/settings/settings.h +++ b/src/include/settings/settings.h @@ -131,6 +131,16 @@ SETTING_int(transaction_buffer_size, //===----------------------------------------------------------------------===// // WRITE AHEAD LOG //===----------------------------------------------------------------------===// +SETTING_string(log_directory_name, + "The relative path(name) to the directory where the WAL will be stored (default : ./logging)", + "./logging", + true, true) + +SETTING_string(log_file_name, + "The name of the write ahead log file in the directory specified above (default : wal.log)", + "wal.log", + true, true) + //===----------------------------------------------------------------------===// // ERROR REPORTING AND LOGGING From 229fb62f89b5d1e52e5d46e79b22c7ac7631e022 Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Fri, 4 May 2018 12:16:07 -0400 Subject: [PATCH 50/81] Asserting that only one logger thread runs --- src/include/threadpool/logger_queue_pool.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/include/threadpool/logger_queue_pool.h b/src/include/threadpool/logger_queue_pool.h index 1c7ea6ba338..a81ef92e2dc 100644 --- a/src/include/threadpool/logger_queue_pool.h +++ b/src/include/threadpool/logger_queue_pool.h @@ -43,6 +43,7 @@ class LoggerQueuePool { num_workers_(num_workers), is_running_(false) { LOG_INFO("aaron: LoggerQueuePool"); + PELOTON_ASSERT(num_workers == 1); logger_queue_.GenerateTokens(log_tokens_, kDefaultNumTokens); } From f8d4d195f7c204496a78c9e625a3c31924c87d95 Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Sun, 29 Apr 2018 17:36:33 -0400 Subject: [PATCH 51/81] recovery: completed phase 1 --- src/common/init.cpp | 14 +- src/include/concurrency/transaction_context.h | 1 - src/include/logging/log_util.h | 48 +++++ src/include/logging/wal_log_manager.h | 30 +-- src/include/logging/wal_recovery.h | 39 ++++ src/logging/log_buffer.cpp | 4 +- src/logging/log_util.cpp | 123 +++++++++++ src/logging/wal_logger.cpp | 2 - src/logging/wal_recovery.cpp | 195 ++++++++++++++++++ 9 files changed, 433 insertions(+), 23 deletions(-) create mode 100644 src/include/logging/log_util.h create mode 100644 src/include/logging/wal_recovery.h create mode 100644 src/logging/log_util.cpp create mode 100644 src/logging/wal_recovery.cpp diff --git a/src/common/init.cpp b/src/common/init.cpp index eb6a2792ca1..16a7dcfb22e 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -25,8 +25,7 @@ #include "index/index.h" #include "settings/settings_manager.h" #include "threadpool/mono_queue_pool.h" -#include "threadpool/logger_queue_pool.h" -#include "logging/wal_log_manager.h" + namespace peloton { @@ -93,12 +92,15 @@ void PelotonInit::Initialize(std::string log_dir, std::string log_file, bool ena // Initialize the Statement Cache Manager StatementCacheManager::Init(); - if(logging::LogManager::GetInstance().init(log_dir, log_file, enable_logging)) { - LOG_DEBUG("logging enabled"); - } else { - LOG_DEBUG("logging disabled"); + logging::LogManager::GetInstance().DoRecovery(); + + if(enable_logging){ + if(!logging::LogManager::GetInstance().init(log_dir, log_file)){ + LOG_ERROR("LogManager Initialization failed"); + } } + threadpool::LoggerQueuePool::GetInstance().Startup(); } diff --git a/src/include/concurrency/transaction_context.h b/src/include/concurrency/transaction_context.h index a116c80e20f..1796d2de22d 100644 --- a/src/include/concurrency/transaction_context.h +++ b/src/include/concurrency/transaction_context.h @@ -24,7 +24,6 @@ #include "common/printable.h" #include "common/internal_types.h" #include "logging/log_buffer.h" -#include "logging/wal_log_manager.h" #include "threadpool/logger_queue_pool.h" #define INTITIAL_RW_SET_SIZE 64 diff --git a/src/include/logging/log_util.h b/src/include/logging/log_util.h new file mode 100644 index 00000000000..7f34a10078a --- /dev/null +++ b/src/include/logging/log_util.h @@ -0,0 +1,48 @@ +/*------------------------------------------------------------------------- + * + * logger.h + * file description + * + * Copyright(c) 2015, CMU + * + * /src/backend/logging/logging_util.h + * + *------------------------------------------------------------------------- + */ + +#pragma once + +#include "common/internal_types.h" +#include "common/logger.h" +#include "storage/data_table.h" +#include "type/serializer.h" + +namespace peloton { +namespace logging { + +//===--------------------------------------------------------------------===// +// LoggingUtil +//===--------------------------------------------------------------------===// + +class LoggingUtil { +public: + // FILE SYSTEM RELATED OPERATIONS + + static bool CheckDirectoryExistence(const char *dir_name); + + static bool CreateDirectory(const char *dir_name, int mode); + + static bool RemoveDirectory(const char *dir_name, bool only_remove_file); + + + static bool OpenFile(const char *name, std::ios_base::openmode mode, + std::fstream &fs); + + static void CloseFile(std::fstream &fs); + + static int32_t ReadNBytesFromFile(std::fstream &fs, char *bytes_read, + size_t n); +}; + +} // namespace logging +} // namespace peloton diff --git a/src/include/logging/wal_log_manager.h b/src/include/logging/wal_log_manager.h index 8f0a635d0ce..5f15df7f05d 100644 --- a/src/include/logging/wal_log_manager.h +++ b/src/include/logging/wal_log_manager.h @@ -11,8 +11,10 @@ //===----------------------------------------------------------------------===// #pragma once #include -#include + +#include "settings/settings_manager.h" #include "util/file_util.h" +#include "logging/wal_recovery.h" namespace peloton{ @@ -21,15 +23,11 @@ namespace logging{ class LogManager{ public: - LogManager() {} + LogManager() : enable_logging_(false) {} ~LogManager() {} - inline bool init(std::string log_dir, std::string log_file, bool enable_logging) { - LOG_INFO("aaron: LogManagerInit"); - if (enable_logging == false) { - enable_logging_ = false; - return false; - } + inline bool init(std::string log_dir, std::string log_file) { + directory_ = log_dir; log_file = log_file; @@ -39,7 +37,8 @@ class LogManager{ boost::filesystem::create_directory(dir); } catch (boost::filesystem::filesystem_error &e) { - LOG_DEBUG("Failed to create log directory %s. Reason %s", log_dir.c_str(), e.what()); + LOG_ERROR("Failed to create log directory %s. Reason %s", log_dir.c_str(), e.what()); + return false; } } @@ -51,7 +50,7 @@ class LogManager{ return true; } } - enable_logging_ = false; + return false; } @@ -71,11 +70,16 @@ class LogManager{ inline bool IsLoggingEnabled() { return enable_logging_; } inline std::string GetDirectory() { return directory_; } inline std::string GetLogFilePath() { return directory_ + log_file_; } + + inline void DoRecovery(){ + WalRecovery* wr = new WalRecovery(); + wr->StartRecovery(); + delete wr; + } + + inline std::ofstream *GetFileStream() { return &logger_ofstream_; } -// TODO(gandeevan): support StartLogging and StopLogging -// static void StartLogging() {} -// static void StopLogging() {} private: diff --git a/src/include/logging/wal_recovery.h b/src/include/logging/wal_recovery.h new file mode 100644 index 00000000000..969e8affa42 --- /dev/null +++ b/src/include/logging/wal_recovery.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include "type/serializeio.h" +#include "logging/log_record.h" + +namespace peloton{ +namespace logging { + +class WalRecovery{ + +public: + WalRecovery() {} + ~WalRecovery(){} + + + void StartRecovery(); + +private: + + void ReplayLogFile(); + void ParseLog(std::map &all_txns); + LogRecord ReadRecord(CopySerializeInput record_decode); + + +// void DoRecovery(); +// void ReplayLogRecord(); + + //TODO(graghura): don't hardcode the path + std::string logpath_ = "/tmp/log"; + std::fstream fstream_; + + + + +}; + +} +} \ No newline at end of file diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index 77a7fa9e099..e07139598cb 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -12,6 +12,8 @@ namespace peloton{ namespace logging{ + + void LogBuffer::WriteRecord(LogRecord &record) { // Reserve for the frame length @@ -24,8 +26,8 @@ void LogBuffer::WriteRecord(LogRecord &record) { log_buffer_.WriteEnumInSingleByte( static_cast(type)); - log_buffer_.WriteLong(record.GetEpochId()); log_buffer_.WriteLong(record.GetTransactionId()); + log_buffer_.WriteLong(record.GetEpochId()); log_buffer_.WriteLong(record.GetCommitId()); diff --git a/src/logging/log_util.cpp b/src/logging/log_util.cpp new file mode 100644 index 00000000000..b6f5b720ba3 --- /dev/null +++ b/src/logging/log_util.cpp @@ -0,0 +1,123 @@ +/*------------------------------------------------------------------------- + * + * logger.h + * file description + * + * Copyright(c) 2015, CMU + * + * /peloton/src/backend/logging/logging_util.cpp + * + *------------------------------------------------------------------------- + */ + +#include +#include +#include +#include + +#include "catalog/manager.h" +#include "logging/log_util.h" + +namespace peloton { + namespace logging { + +//===--------------------------------------------------------------------===// +// LoggingUtil +//===--------------------------------------------------------------------===// + +bool LoggingUtil::CreateDirectory(const char *dir_name, int mode) { + int return_val = mkdir(dir_name, mode); + if (return_val == 0) { + LOG_DEBUG("Created directory %s successfully", dir_name); + } else if (errno == EEXIST) { + LOG_DEBUG("Directory %s already exists", dir_name); + } else { + LOG_DEBUG("Creating directory failed: %s", strerror(errno)); + return false; + } + return true; +} + +bool LoggingUtil::CheckDirectoryExistence(const char *dir_name) { + struct stat info; + int return_val = stat(dir_name, &info); + return return_val == 0 && S_ISDIR(info.st_mode); +} + +/** + * @return false if fail to remove directory + */ +bool LoggingUtil::RemoveDirectory(const char *dir_name, bool only_remove_file) { + struct dirent *file; + DIR *dir; + + dir = opendir(dir_name); + if (dir == nullptr) { + return true; + } + + // XXX readdir is not thread safe??? + while ((file = readdir(dir)) != nullptr) { + if (strcmp(file->d_name, ".") == 0 || strcmp(file->d_name, "..") == 0) { + continue; + } + char complete_path[256]; + strcpy(complete_path, dir_name); + strcat(complete_path, "/"); + strcat(complete_path, file->d_name); + auto ret_val = remove(complete_path); + if (ret_val != 0) { + LOG_ERROR("Failed to delete file: %s, error: %s", complete_path, + strerror(errno)); + } + } + closedir(dir); + if (!only_remove_file) { + auto ret_val = remove(dir_name); + if (ret_val != 0) { + LOG_ERROR("Failed to delete dir: %s, error: %s", file->d_name, + strerror(errno)); + } + } + return true; +} + +/** + * @return false if fail to open file + */ +bool LoggingUtil::OpenFile(const char *name, std::ios_base::openmode mode, + std::fstream &fs) { + if(fs.is_open()){ + LOG_ERROR("Failed to open stream: already open"); + return false; + } + + fs.open(name, mode); + + if (fs.fail()){ + LOG_ERROR("Failed to open stream: %s", strerror(errno)); + return false; + } + + return true; +} + + +void LoggingUtil::CloseFile(std::fstream &fs) { + fs.close(); +} + +/** + * @return the number of bytes read from the file + */ +int32_t LoggingUtil::ReadNBytesFromFile(std::fstream &fs, char *bytes_read, + size_t n) { + + PL_ASSERT(!fs.fail()); + fs.read(bytes_read, n); + return (int32_t)fs.gcount(); +} + + +} // namespace logging +} // namespace peloton diff --git a/src/logging/wal_logger.cpp b/src/logging/wal_logger.cpp index 115ac0ace1e..21188672b90 100644 --- a/src/logging/wal_logger.cpp +++ b/src/logging/wal_logger.cpp @@ -15,8 +15,6 @@ #include "logging/log_buffer.h" #include "catalog/manager.h" #include "common/container_tuple.h" -#include "storage/tile_group.h" -#include "../include/logging/wal_logger.h" namespace peloton{ diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp new file mode 100644 index 00000000000..df9ae60e856 --- /dev/null +++ b/src/logging/wal_recovery.cpp @@ -0,0 +1,195 @@ +#include +#include + +#include "logging/wal_recovery.h" +#include "logging/log_util.h" +#include "common/logger.h" + + +namespace peloton{ +namespace logging{ + +void WalRecovery::StartRecovery(){ + + if(!LoggingUtil::OpenFile(logpath_.c_str(), + std::ifstream::in, fstream_)) { + + LOG_ERROR("Recovery failed"); + return; + } + + ReplayLogFile(); +} + + +//void WalRecovery::InstallInsertRecord(){ +// +//} +// +//void WalRecovery::InstallDeleteRecord(){ +// +//} +// +//void WalRecovery::InstallUpdateRecord(){ +// +//} + +//LogRecord WalRecovery::ReadRecord(CopySerializeInput record_decode){ +// +// LogRecordType record_type = +// (LogRecordType)(record_decode.ReadEnumInSingleByte()); +// +// eid_t record_eid = record_decode.ReadLong(); +// txn_id_t record_txn_id = record_decode.ReadLong(); +// cid_t record_cid = record_decode.ReadLong(); +// +// +// switch(record_type){ +// case LogRecordType::TRANSACTION_BEGIN: +// case LogRecordType::TRANSACTION_ABORT: +// case LogRecordType::TRANSACTION_COMMIT: { +// return logging::LogRecordFactory::CreateTupleRecord( +// record_type, record_eid, record_txn_id, +// record_cid); +// } +// +// case LogRecordType::TUPLE_INSERT: +// case LogRecordType::TUPLE_DELETE: { +// oid_t database_id = (oid_t)record_decode.ReadLong(); +// oid_t table_id = (oid_t)record_decode.ReadLong(); +// +// oid_t tg_block = (oid_t)record_decode.ReadLong(); +// oid_t tg_offset = (oid_t)record_decode.ReadLong(); +// +// return logging::LogRecordFactory::CreateTupleRecord( +// record_type, ItemPointer(tg_block, tg_offset), +// record_eid, record_txn_id, +// record_cid); +// } +// +// +// case LogRecordType::TUPLE_UPDATE: +// oid_t database_id = (oid_t)record_decode.ReadLong(); +// oid_t table_id = (oid_t)record_decode.ReadLong(); +// +// oid_t old_tg_block = (oid_t)record_decode.ReadLong(); +// oid_t old_tg_offset = (oid_t)record_decode.ReadLong(); +// +// oid_t tg_block = (oid_t)record_decode.ReadLong(); +// oid_t tg_offset = (oid_t)record_decode.ReadLong(); +// +// return logging::LogRecordFactory::CreateTupleRecord( +// record_type, ItemPointer(old_tg_block, old_tg_offset), +// ItemPointer(tg_block, tg_offset), +// record_eid, record_txn_id, +// record_cid); +// +// +// default: +// LOG_ERROR("Unknown LogRecordType"); +// PL_ASSERT(false); +// +// } +// +//} + +//void WalRecovery::ReplayLogRecord(LogRecord &log_record){ +// +//} +// + +void WalRecovery::ParseLog(std::map &all_txns){ + + int buf_size = 4096, buf_rem = 0, buf_curr = 0; + char *buf = new char[buf_size]; + + bool replay_complete = false; + + while(!replay_complete) { + + int len, lensize = sizeof(len), nbytes = buf_size-buf_rem; + int bytes_read = LoggingUtil::ReadNBytesFromFile(fstream_, buf+buf_curr, nbytes); + + LOG_INFO("Read %d bytes from the log", bytes_read); + + buf_rem += bytes_read; + + if(bytes_read != nbytes){ + if(fstream_.eof()){ + replay_complete = true; + + if(buf_rem==0) + continue; + } + } + + while(buf_rem >= (lensize)){ + CopySerializeInput length_decode((const void *)(buf+buf_curr), 4); + len = length_decode.ReadInt(); + + LOG_INFO("length = %d", len); + + if(buf_rem >= (len + lensize)){ + buf_curr += lensize; + + CopySerializeInput record_decode((const void *)(buf+buf_curr), len); + LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); + txn_id_t txn_id = record_decode.ReadLong(); + + + switch(record_type){ + case LogRecordType::TRANSACTION_BEGIN:{ + if(all_txns.find(txn_id) != all_txns.end()){ + LOG_ERROR("Duplicate transaction"); + PL_ASSERT(false); + } + all_txns.insert(std::make_pair(txn_id, record_type)); + break; + } + + case LogRecordType::TRANSACTION_COMMIT: + case LogRecordType::TRANSACTION_ABORT: { + all_txns[txn_id] = record_type; + break; + } + + default: { + + } + } + + buf_curr += len; + buf_rem -= (len + lensize); + } else { + break; + } + } + + PL_MEMCPY(buf, buf+buf_curr, buf_rem); + PL_MEMSET(buf+buf_rem, 0, buf_size - buf_rem); + buf_curr = buf_rem; + } + + delete[] buf; +} + +void WalRecovery::ReplayLogFile(){ + + std::map all_txns; + ParseLog(all_txns); + + + for(auto it = all_txns.begin(); it!=all_txns.end(); it++){ + if(it->second==LogRecordType::TRANSACTION_COMMIT){ + LOG_INFO("Replaying peloton txn : %" PRId64, it->first); + } + } + + + +} + + + +} +} \ No newline at end of file From baed22fbdc57028b70b26485255fa0dfb7bee92e Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Fri, 4 May 2018 19:23:56 -0400 Subject: [PATCH 52/81] PL_* to PELOTON_* --- src/logging/log_util.cpp | 2 +- src/logging/wal_recovery.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/logging/log_util.cpp b/src/logging/log_util.cpp index b6f5b720ba3..cc4234dd452 100644 --- a/src/logging/log_util.cpp +++ b/src/logging/log_util.cpp @@ -113,7 +113,7 @@ void LoggingUtil::CloseFile(std::fstream &fs) { int32_t LoggingUtil::ReadNBytesFromFile(std::fstream &fs, char *bytes_read, size_t n) { - PL_ASSERT(!fs.fail()); + PELOTON_ASSERT(!fs.fail()); fs.read(bytes_read, n); return (int32_t)fs.gcount(); } diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index df9ae60e856..ef50e6b14ac 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -141,7 +141,7 @@ void WalRecovery::ParseLog(std::map &all_txns){ case LogRecordType::TRANSACTION_BEGIN:{ if(all_txns.find(txn_id) != all_txns.end()){ LOG_ERROR("Duplicate transaction"); - PL_ASSERT(false); + PELOTON_ASSERT(false); } all_txns.insert(std::make_pair(txn_id, record_type)); break; @@ -165,8 +165,8 @@ void WalRecovery::ParseLog(std::map &all_txns){ } } - PL_MEMCPY(buf, buf+buf_curr, buf_rem); - PL_MEMSET(buf+buf_rem, 0, buf_size - buf_rem); + PELOTON_MEMCPY(buf, buf+buf_curr, buf_rem); + PELOTON_MEMSET(buf+buf_rem, 0, buf_size - buf_rem); buf_curr = buf_rem; } From bb875d20e15a145b2c801e3b90153e4b0ce09b6f Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Mon, 30 Apr 2018 19:00:50 -0400 Subject: [PATCH 53/81] recovery: added support for calculating replay order in phase 1 --- src/include/logging/wal_recovery.h | 44 +++++++++++++++-- src/logging/wal_recovery.cpp | 78 +++++++++++++++++++++++------- 2 files changed, 101 insertions(+), 21 deletions(-) diff --git a/src/include/logging/wal_recovery.h b/src/include/logging/wal_recovery.h index 969e8affa42..b4dfe54fc4e 100644 --- a/src/include/logging/wal_recovery.h +++ b/src/include/logging/wal_recovery.h @@ -7,8 +7,46 @@ namespace peloton{ namespace logging { + + class WalRecovery{ +private: + + // (graghura): this is a dirty hack to establish the + // replay order. This is tightly coupled with the way + // transaction id is generated :( + // The transaction with the smaller epoch_id is replayed + // first. If the epoch's are the same, then the txn with the + // lower next_txn_id (decentralized_epoch_manager.cpp) + // is replayed first. + + // TODO(graghura) : change this after the mvcc bugs are fixed. + + struct RecoveryComparator { + bool operator()(txn_id_t txn_id_1, txn_id_t txn_id_2) const { + eid_t epoch_txn_1 = txn_id_1 >> 32; + eid_t epoch_txn_2 = txn_id_2 >> 32; + + int next_id_1 = txn_id_1 & 0xFFFFFFFF; + int next_id_2 = txn_id_2 & 0xFFFFFFFF; + + if(epoch_txn_1==epoch_txn_2){ + if(next_id_1<=next_id_2) + return true; + else + return false; + } else if(epoch_txn_1; + + public: WalRecovery() {} ~WalRecovery(){} @@ -19,10 +57,10 @@ class WalRecovery{ private: void ReplayLogFile(); - void ParseLog(std::map &all_txns); - LogRecord ReadRecord(CopySerializeInput record_decode); - + void FirstPass(std::map> &, + ReplayMap &, uint32_t &nbytes); +// LogRecord ReadRecord(CopySerializeInput record_decode); // void DoRecovery(); // void ReplayLogRecord(); diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index ef50e6b14ac..a60f43ea31f 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -98,8 +98,11 @@ void WalRecovery::StartRecovery(){ //} // -void WalRecovery::ParseLog(std::map &all_txns){ +// Pass 1: +void WalRecovery::FirstPass(std::map> &all_txns, + ReplayMap &commited_txns, uint32_t &mem_size){ + int total_length = 0; int buf_size = 4096, buf_rem = 0, buf_curr = 0; char *buf = new char[buf_size]; @@ -107,7 +110,7 @@ void WalRecovery::ParseLog(std::map &all_txns){ while(!replay_complete) { - int len, lensize = sizeof(len), nbytes = buf_size-buf_rem; + int len, nbytes = buf_size-buf_rem; int bytes_read = LoggingUtil::ReadNBytesFromFile(fstream_, buf+buf_curr, nbytes); LOG_INFO("Read %d bytes from the log", bytes_read); @@ -123,19 +126,21 @@ void WalRecovery::ParseLog(std::map &all_txns){ } } - while(buf_rem >= (lensize)){ + while(buf_rem >= (int)sizeof(len)){ CopySerializeInput length_decode((const void *)(buf+buf_curr), 4); len = length_decode.ReadInt(); - LOG_INFO("length = %d", len); - - if(buf_rem >= (len + lensize)){ - buf_curr += lensize; + if(buf_rem >= (int)(len + sizeof(len))){ + buf_curr += sizeof(len); CopySerializeInput record_decode((const void *)(buf+buf_curr), len); LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); txn_id_t txn_id = record_decode.ReadLong(); + total_length += len; + + + //uint64_t id = txn_id & 0xFFFFFFFF; switch(record_type){ case LogRecordType::TRANSACTION_BEGIN:{ @@ -143,23 +148,45 @@ void WalRecovery::ParseLog(std::map &all_txns){ LOG_ERROR("Duplicate transaction"); PELOTON_ASSERT(false); } - all_txns.insert(std::make_pair(txn_id, record_type)); + + auto value = std::make_pair(record_type, len); + all_txns.insert(std::make_pair(txn_id, value)); + break; } - case LogRecordType::TRANSACTION_COMMIT: + case LogRecordType::TRANSACTION_COMMIT:{ + all_txns[txn_id].first = record_type; + all_txns[txn_id].second += len; + + // keeps track of the memory that + // needs to be allocated. + mem_size += all_txns[txn_id].second; + + commited_txns.insert(std::make_pair(txn_id, all_txns[txn_id].second)); + } + case LogRecordType::TRANSACTION_ABORT: { - all_txns[txn_id] = record_type; + all_txns[txn_id].first = record_type; + all_txns[txn_id].second += len; break; } - default: { + case LogRecordType::TUPLE_DELETE: + case LogRecordType::TUPLE_UPDATE: + case LogRecordType::TUPLE_INSERT: { + all_txns[txn_id].second += len; + break; + } + default: { + LOG_ERROR("Unknown LogRecordType."); + PL_ASSERT(false); } } buf_curr += len; - buf_rem -= (len + lensize); + buf_rem -= (len + sizeof(len)); } else { break; } @@ -170,21 +197,36 @@ void WalRecovery::ParseLog(std::map &all_txns){ buf_curr = buf_rem; } + LOG_INFO("total_length = %d", total_length); + delete[] buf; } + + void WalRecovery::ReplayLogFile(){ - std::map all_txns; - ParseLog(all_txns); + uint32_t mem_size = 0; + ReplayMap commited_txns; + std::map> all_txns; + // Pass 1 + FirstPass(all_txns, commited_txns, mem_size); - for(auto it = all_txns.begin(); it!=all_txns.end(); it++){ - if(it->second==LogRecordType::TRANSACTION_COMMIT){ - LOG_INFO("Replaying peloton txn : %" PRId64, it->first); - } + //char *log_buffer = new char[mem_size]; + + // Pass 2 +// SecondPass(commited_txns, log_buffer); + + for(auto it = commited_txns.begin(); it!=commited_txns.end(); it++){ + txn_id_t txn_id = it->first; + int epoch_id = txn_id >> 32; + int counter = txn_id & 0xFFFFFFFF; + LOG_INFO("Replaying peloton txn : %llu, epoch %d, counter = %d", it->first, epoch_id, counter); } + LOG_INFO("buf_size = %u", mem_size); + } From 1332dc9a5647c529a79dd9cc04c50c2fcc32e53c Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Mon, 30 Apr 2018 22:17:31 -0400 Subject: [PATCH 54/81] recovery: removed an extraneous map --- src/include/logging/wal_recovery.h | 15 ++++- src/logging/wal_recovery.cpp | 95 ++++++++++++++++++------------ 2 files changed, 68 insertions(+), 42 deletions(-) diff --git a/src/include/logging/wal_recovery.h b/src/include/logging/wal_recovery.h index b4dfe54fc4e..24c497a35e4 100644 --- a/src/include/logging/wal_recovery.h +++ b/src/include/logging/wal_recovery.h @@ -31,6 +31,9 @@ class WalRecovery{ int next_id_1 = txn_id_1 & 0xFFFFFFFF; int next_id_2 = txn_id_2 & 0xFFFFFFFF; + if(txn_id_1==txn_id_2) + return false; + if(epoch_txn_1==epoch_txn_2){ if(next_id_1<=next_id_2) return true; @@ -44,7 +47,8 @@ class WalRecovery{ } }; - using ReplayMap = std::map; + // key: txn_id, value: {COMMITED/ABORTED, len of all log_records of txn_id} + using ReplayTxnMap = std::map, RecoveryComparator>; public: @@ -57,8 +61,13 @@ class WalRecovery{ private: void ReplayLogFile(); - void FirstPass(std::map> &, - ReplayMap &, uint32_t &nbytes); + void FirstPass(ReplayTxnMap &, size_t &log_buffer_size); + void SecondPass( std::map> &all_txns, + char *buffer, size_t buf_len); + +// void SecondPass(ReplayTxnMap &all_txns, char *buffer, size_t buf_size); + + // LogRecord ReadRecord(CopySerializeInput record_decode); // void DoRecovery(); diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index a60f43ea31f..0272b65ee87 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -99,18 +99,16 @@ void WalRecovery::StartRecovery(){ // // Pass 1: -void WalRecovery::FirstPass(std::map> &all_txns, - ReplayMap &commited_txns, uint32_t &mem_size){ +void WalRecovery::FirstPass(ReplayTxnMap &all_txns, size_t &log_buffer_size){ - int total_length = 0; - int buf_size = 4096, buf_rem = 0, buf_curr = 0; + int buf_size = 4096, buf_rem = 0, buf_curr = 0,total_len=0; char *buf = new char[buf_size]; bool replay_complete = false; while(!replay_complete) { - int len, nbytes = buf_size-buf_rem; + int len, nbytes = buf_size-buf_rem; int bytes_read = LoggingUtil::ReadNBytesFromFile(fstream_, buf+buf_curr, nbytes); LOG_INFO("Read %d bytes from the log", bytes_read); @@ -131,17 +129,14 @@ void WalRecovery::FirstPass(std::map> &a len = length_decode.ReadInt(); if(buf_rem >= (int)(len + sizeof(len))){ + + total_len += len; buf_curr += sizeof(len); CopySerializeInput record_decode((const void *)(buf+buf_curr), len); LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); txn_id_t txn_id = record_decode.ReadLong(); - total_length += len; - - - //uint64_t id = txn_id & 0xFFFFFFFF; - switch(record_type){ case LogRecordType::TRANSACTION_BEGIN:{ if(all_txns.find(txn_id) != all_txns.end()){ @@ -156,26 +151,25 @@ void WalRecovery::FirstPass(std::map> &a } case LogRecordType::TRANSACTION_COMMIT:{ - all_txns[txn_id].first = record_type; - all_txns[txn_id].second += len; - - // keeps track of the memory that - // needs to be allocated. - mem_size += all_txns[txn_id].second; - - commited_txns.insert(std::make_pair(txn_id, all_txns[txn_id].second)); + // keeps track of the memory that needs to be allocated. + log_buffer_size += (len+all_txns[txn_id].second); } - case LogRecordType::TRANSACTION_ABORT: { - all_txns[txn_id].first = record_type; - all_txns[txn_id].second += len; - break; - } + // intentional fall through + case LogRecordType::TRANSACTION_ABORT: case LogRecordType::TUPLE_DELETE: case LogRecordType::TUPLE_UPDATE: case LogRecordType::TUPLE_INSERT: { + + if(all_txns.find(txn_id) == all_txns.end()){ + LOG_ERROR("Transaction not found"); + PL_ASSERT(false); + } + + all_txns[txn_id].first = record_type; all_txns[txn_id].second += len; + break; } @@ -197,38 +191,61 @@ void WalRecovery::FirstPass(std::map> &a buf_curr = buf_rem; } - LOG_INFO("total_length = %d", total_length); - delete[] buf; } +//void WalRecovery::SecondPass(std::map> &all_txns, +// char *buffer, size_t buf_len){ +// +// +// +// +//} void WalRecovery::ReplayLogFile(){ - uint32_t mem_size = 0; - ReplayMap commited_txns; - std::map> all_txns; + + size_t curr_txn_offset = 0, log_buffer_size = 0; + + ReplayTxnMap all_txns; + + // key: txn_id, value: {log_buf_start_offset, log_buf_curr_offset} + std::map> commited_txns; + // Pass 1 - FirstPass(all_txns, commited_txns, mem_size); + FirstPass(all_txns, log_buffer_size); - //char *log_buffer = new char[mem_size]; + LOG_INFO("all_txns_len = %zu", all_txns.size()); - // Pass 2 -// SecondPass(commited_txns, log_buffer); + for(auto it = all_txns.begin(); it!=all_txns.end(); it++){ - for(auto it = commited_txns.begin(); it!=commited_txns.end(); it++){ - txn_id_t txn_id = it->first; - int epoch_id = txn_id >> 32; - int counter = txn_id & 0xFFFFFFFF; - LOG_INFO("Replaying peloton txn : %llu, epoch %d, counter = %d", it->first, epoch_id, counter); - } + if(it->second.first != LogRecordType::TRANSACTION_COMMIT) + continue; - LOG_INFO("buf_size = %u", mem_size); + auto offset_pair = std::make_pair(curr_txn_offset, curr_txn_offset); + commited_txns.insert(std::make_pair(it->first, offset_pair)); + curr_txn_offset += it->second.second; + } + // Pass 2 +// char *log_buffer = new char[log_buffer_size]; +// SecondPass(commited_txns, log_buffer, log_buffer_size); +// +// +// // DEBUG INFO +// for(auto it = commited_txns.begin(); it!=commited_txns.end(); it++){ +// txn_id_t txn_id = it->first; +// int epoch_id = txn_id >> 32; +// int counter = txn_id & 0xFFFFFFFF; +// LOG_INFO("Replaying peloton txn : %llu, epoch %d, counter = %d", it->first, epoch_id, counter); +// } +// +// LOG_INFO("buf_size = %zu", log_buffer_size); + } From 49c9512b3a8d0b17eff847d24c8f824ad2634421 Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Mon, 30 Apr 2018 23:23:35 -0400 Subject: [PATCH 55/81] recovery: refactored code for two passes --- src/include/logging/wal_recovery.h | 30 +++--- src/logging/wal_recovery.cpp | 150 ++++++++++++++++------------- 2 files changed, 100 insertions(+), 80 deletions(-) diff --git a/src/include/logging/wal_recovery.h b/src/include/logging/wal_recovery.h index 24c497a35e4..23a5fd057f5 100644 --- a/src/include/logging/wal_recovery.h +++ b/src/include/logging/wal_recovery.h @@ -1,6 +1,8 @@ #pragma once +#include #include + #include "type/serializeio.h" #include "logging/log_record.h" @@ -13,6 +15,11 @@ class WalRecovery{ private: + enum class ReplayStage{ + PASS_1, + PASS_2 + }; + // (graghura): this is a dirty hack to establish the // replay order. This is tightly coupled with the way // transaction id is generated :( @@ -52,7 +59,9 @@ class WalRecovery{ public: - WalRecovery() {} + WalRecovery() : + log_buffer_size_(0), log_buffer_(nullptr) {} + ~WalRecovery(){} @@ -60,24 +69,21 @@ class WalRecovery{ private: - void ReplayLogFile(); - void FirstPass(ReplayTxnMap &, size_t &log_buffer_size); - void SecondPass( std::map> &all_txns, - char *buffer, size_t buf_len); - -// void SecondPass(ReplayTxnMap &all_txns, char *buffer, size_t buf_size); - - + void ReplayLogFile(); + void ParseFromDisk(ReplayStage stage); + void Pass1(char *buf, int len); + void Pass2(char *buf, int len); -// LogRecord ReadRecord(CopySerializeInput record_decode); -// void DoRecovery(); -// void ReplayLogRecord(); //TODO(graghura): don't hardcode the path std::string logpath_ = "/tmp/log"; std::fstream fstream_; + int log_buffer_size_; + char *log_buffer_; + ReplayTxnMap all_txns_; + std::map> commited_txns_; // {txn_id, {log_buf_start_offset, log_buf_curr_offset}} }; diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index 0272b65ee87..217d48e37ad 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -1,5 +1,4 @@ -#include -#include + #include "logging/wal_recovery.h" #include "logging/log_util.h" @@ -98,14 +97,15 @@ void WalRecovery::StartRecovery(){ //} // -// Pass 1: -void WalRecovery::FirstPass(ReplayTxnMap &all_txns, size_t &log_buffer_size){ +void WalRecovery::ParseFromDisk(ReplayStage stage){ int buf_size = 4096, buf_rem = 0, buf_curr = 0,total_len=0; char *buf = new char[buf_size]; - bool replay_complete = false; + fstream_.clear(); + fstream_.seekg(0, std::ios::beg); + while(!replay_complete) { int len, nbytes = buf_size-buf_rem; @@ -133,50 +133,10 @@ void WalRecovery::FirstPass(ReplayTxnMap &all_txns, size_t &log_buffer_size){ total_len += len; buf_curr += sizeof(len); - CopySerializeInput record_decode((const void *)(buf+buf_curr), len); - LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); - txn_id_t txn_id = record_decode.ReadLong(); - - switch(record_type){ - case LogRecordType::TRANSACTION_BEGIN:{ - if(all_txns.find(txn_id) != all_txns.end()){ - LOG_ERROR("Duplicate transaction"); - PELOTON_ASSERT(false); - } - - auto value = std::make_pair(record_type, len); - all_txns.insert(std::make_pair(txn_id, value)); - - break; - } - - case LogRecordType::TRANSACTION_COMMIT:{ - // keeps track of the memory that needs to be allocated. - log_buffer_size += (len+all_txns[txn_id].second); - } - - // intentional fall through - - case LogRecordType::TRANSACTION_ABORT: - case LogRecordType::TUPLE_DELETE: - case LogRecordType::TUPLE_UPDATE: - case LogRecordType::TUPLE_INSERT: { - - if(all_txns.find(txn_id) == all_txns.end()){ - LOG_ERROR("Transaction not found"); - PL_ASSERT(false); - } - - all_txns[txn_id].first = record_type; - all_txns[txn_id].second += len; - - break; - } - - default: { - LOG_ERROR("Unknown LogRecordType."); - PL_ASSERT(false); - } + if(stage==ReplayStage::PASS_1){ + Pass1(buf+buf_curr, len); + } else if (stage==ReplayStage::PASS_2) { + Pass2(buf+buf_curr, len); } buf_curr += len; @@ -194,48 +154,102 @@ void WalRecovery::FirstPass(ReplayTxnMap &all_txns, size_t &log_buffer_size){ delete[] buf; } +// Pass 1: +void WalRecovery::Pass1(char *buf, int len) { + + CopySerializeInput record_decode((const void *)buf, len); + LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); + txn_id_t txn_id = record_decode.ReadLong(); -//void WalRecovery::SecondPass(std::map> &all_txns, -// char *buffer, size_t buf_len){ -// + switch(record_type){ + case LogRecordType::TRANSACTION_BEGIN:{ + if(all_txns_.find(txn_id) != all_txns_.end()){ + LOG_ERROR("Duplicate transaction"); + PELOTON_ASSERT(false); + } + + auto value = std::make_pair(record_type, len); + all_txns_.insert(std::make_pair(txn_id, value)); + + break; + } + + case LogRecordType::TRANSACTION_COMMIT:{ + // keeps track of the memory that needs to be allocated. + log_buffer_size_ += (len+all_txns_[txn_id].second); + } + + // intentional fall through + case LogRecordType::TRANSACTION_ABORT: + case LogRecordType::TUPLE_DELETE: + case LogRecordType::TUPLE_UPDATE: + case LogRecordType::TUPLE_INSERT: { + + if(all_txns_.find(txn_id) == all_txns_.end()){ + LOG_ERROR("Transaction not found"); + PELOTON_ASSERT(false); + } + + all_txns_[txn_id].first = record_type; + all_txns_[txn_id].second += len; + + break; + } + + default: { + LOG_ERROR("Unknown LogRecordType."); + PELOTON_ASSERT(false); + } + } + +} + +// Pass 2: +void WalRecovery::Pass2(char *buf, int len){ + (void) buf; + (void) len; // +// CopySerializeInput record_decode((const void *)buf, len); +// LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); +// txn_id_t txn_id = record_decode.ReadLong(); // // -//} - -void WalRecovery::ReplayLogFile(){ - +// if(commited_txns_.find(txn_id)==commited_txns_.end()) { +// return; +// } - size_t curr_txn_offset = 0, log_buffer_size = 0; +} - ReplayTxnMap all_txns; +void WalRecovery::ReplayLogFile(){ - // key: txn_id, value: {log_buf_start_offset, log_buf_curr_offset} - std::map> commited_txns; + size_t curr_txn_offset = 0; // Pass 1 - FirstPass(all_txns, log_buffer_size); + ParseFromDisk(ReplayStage::PASS_1); - LOG_INFO("all_txns_len = %zu", all_txns.size()); + LOG_INFO("all_txns_len = %zu", all_txns_.size()); - for(auto it = all_txns.begin(); it!=all_txns.end(); it++){ + for(auto it = all_txns_.begin(); it!=all_txns_.end(); it++){ if(it->second.first != LogRecordType::TRANSACTION_COMMIT) continue; auto offset_pair = std::make_pair(curr_txn_offset, curr_txn_offset); - commited_txns.insert(std::make_pair(it->first, offset_pair)); + commited_txns_.insert(std::make_pair(it->first, offset_pair)); curr_txn_offset += it->second.second; } + for(auto it = commited_txns_.begin(); it!=commited_txns_.end(); it++){ + LOG_INFO("txn_id = %llu offset = %d", it->first, it->second.first); + } // Pass 2 -// char *log_buffer = new char[log_buffer_size]; -// SecondPass(commited_txns, log_buffer, log_buffer_size); -// -// + log_buffer_ = new char[log_buffer_size_]; + ParseFromDisk(ReplayStage::PASS_2); + + // // DEBUG INFO // for(auto it = commited_txns.begin(); it!=commited_txns.end(); it++){ // txn_id_t txn_id = it->first; From af1240b58c95199e74afa540ad5ae433c679f06d Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Mon, 30 Apr 2018 23:37:34 -0400 Subject: [PATCH 56/81] recovery: completed second pass --- src/logging/wal_recovery.cpp | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index 217d48e37ad..26e82d2705c 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -208,15 +208,20 @@ void WalRecovery::Pass1(char *buf, int len) { void WalRecovery::Pass2(char *buf, int len){ (void) buf; (void) len; -// -// CopySerializeInput record_decode((const void *)buf, len); -// LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); -// txn_id_t txn_id = record_decode.ReadLong(); -// -// -// if(commited_txns_.find(txn_id)==commited_txns_.end()) { -// return; -// } + + CopySerializeInput record_decode((const void *)buf, len); + LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); + txn_id_t txn_id = record_decode.ReadLong(); + + (void) record_type; + + if(commited_txns_.find(txn_id)==commited_txns_.end()) { + return; + } + + int curr_offset = commited_txns_[txn_id].second; + PL_MEMCPY(log_buffer_+curr_offset, buf, len); + commited_txns_[txn_id].second += len; } From 789fe4e1dc39d3501e73f9e8a5229a36779d784a Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Tue, 1 May 2018 14:29:04 -0400 Subject: [PATCH 57/81] recovery: code refactor --- src/include/logging/wal_recovery.h | 2 +- src/logging/wal_recovery.cpp | 161 ++++++++++++++--------------- 2 files changed, 81 insertions(+), 82 deletions(-) diff --git a/src/include/logging/wal_recovery.h b/src/include/logging/wal_recovery.h index 23a5fd057f5..e24547c9a50 100644 --- a/src/include/logging/wal_recovery.h +++ b/src/include/logging/wal_recovery.h @@ -71,7 +71,7 @@ class WalRecovery{ void ReplayLogFile(); void ParseFromDisk(ReplayStage stage); - void Pass1(char *buf, int len); +// void Pass1(char *buf, int len); void Pass2(char *buf, int len); diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index 26e82d2705c..cc644815f91 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -108,7 +108,7 @@ void WalRecovery::ParseFromDisk(ReplayStage stage){ while(!replay_complete) { - int len, nbytes = buf_size-buf_rem; + int record_len, nbytes = buf_size-buf_rem; int bytes_read = LoggingUtil::ReadNBytesFromFile(fstream_, buf+buf_curr, nbytes); LOG_INFO("Read %d bytes from the log", bytes_read); @@ -124,106 +124,103 @@ void WalRecovery::ParseFromDisk(ReplayStage stage){ } } - while(buf_rem >= (int)sizeof(len)){ + while(buf_rem >= (int)sizeof(record_len)){ CopySerializeInput length_decode((const void *)(buf+buf_curr), 4); - len = length_decode.ReadInt(); + record_len = length_decode.ReadInt(); - if(buf_rem >= (int)(len + sizeof(len))){ + if(buf_rem >= (int)(record_len + sizeof(record_len))){ - total_len += len; - buf_curr += sizeof(len); + total_len += record_len; - if(stage==ReplayStage::PASS_1){ - Pass1(buf+buf_curr, len); - } else if (stage==ReplayStage::PASS_2) { - Pass2(buf+buf_curr, len); - } + int record_offset = buf_curr + sizeof(record_len); - buf_curr += len; - buf_rem -= (len + sizeof(len)); - } else { - break; - } - } + /***********/ + /* PASS 1 */ + /***********/ - PELOTON_MEMCPY(buf, buf+buf_curr, buf_rem); - PELOTON_MEMSET(buf+buf_rem, 0, buf_size - buf_rem); - buf_curr = buf_rem; - } - - delete[] buf; -} - -// Pass 1: -void WalRecovery::Pass1(char *buf, int len) { + if(stage==ReplayStage::PASS_1){ + CopySerializeInput record_decode((const void *)(buf+record_offset), record_len); + LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); + txn_id_t txn_id = record_decode.ReadLong(); + + int req_mem_size = record_len + sizeof(record_len); + + switch(record_type){ + case LogRecordType::TRANSACTION_BEGIN:{ + if(all_txns_.find(txn_id) != all_txns_.end()){ + LOG_ERROR("Duplicate transaction"); + PELOTON_ASSERT(false); + } + + auto value = std::make_pair(record_type, req_mem_size); + all_txns_.insert(std::make_pair(txn_id, value)); + + break; + } + + case LogRecordType::TRANSACTION_COMMIT:{ + // keeps track of the memory that needs to be allocated. + log_buffer_size_ += (req_mem_size+all_txns_[txn_id].second); + } + + // intentional fall through + case LogRecordType::TRANSACTION_ABORT: + case LogRecordType::TUPLE_DELETE: + case LogRecordType::TUPLE_UPDATE: + case LogRecordType::TUPLE_INSERT: { + + if(all_txns_.find(txn_id) == all_txns_.end()){ + LOG_ERROR("Transaction not found"); + PELOTON_ASSERT(false); + } + + all_txns_[txn_id].first = record_type; + all_txns_[txn_id].second += req_mem_size; + + break; + } + + default: { + LOG_ERROR("Unknown LogRecordType."); + PELOTON_ASSERT(false); + } + } + } - CopySerializeInput record_decode((const void *)buf, len); - LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); - txn_id_t txn_id = record_decode.ReadLong(); + else if (stage==ReplayStage::PASS_2) { - switch(record_type){ - case LogRecordType::TRANSACTION_BEGIN:{ - if(all_txns_.find(txn_id) != all_txns_.end()){ - LOG_ERROR("Duplicate transaction"); - PELOTON_ASSERT(false); - } + CopySerializeInput record_decode((const void *)(buf+record_offset), record_len); + LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); + txn_id_t txn_id = record_decode.ReadLong(); - auto value = std::make_pair(record_type, len); - all_txns_.insert(std::make_pair(txn_id, value)); + (void) record_type; - break; - } + if(commited_txns_.find(txn_id)!=commited_txns_.end()) { + int copy_len = record_len + sizeof(record_len); + int curr_offset = commited_txns_[txn_id].second; - case LogRecordType::TRANSACTION_COMMIT:{ - // keeps track of the memory that needs to be allocated. - log_buffer_size_ += (len+all_txns_[txn_id].second); - } + LOG_INFO("txn %llu writing from %d to %d", txn_id, curr_offset, curr_offset+copy_len-1); - // intentional fall through - case LogRecordType::TRANSACTION_ABORT: - case LogRecordType::TUPLE_DELETE: - case LogRecordType::TUPLE_UPDATE: - case LogRecordType::TUPLE_INSERT: { + PELOTON_MEMCPY(log_buffer_+curr_offset, buf+buf_curr, copy_len); + commited_txns_[txn_id].second += copy_len; + } + } - if(all_txns_.find(txn_id) == all_txns_.end()){ - LOG_ERROR("Transaction not found"); - PELOTON_ASSERT(false); + buf_curr += (record_len + sizeof(record_len)); + buf_rem -= (record_len + sizeof(record_len)); + } else { + break; } - - all_txns_[txn_id].first = record_type; - all_txns_[txn_id].second += len; - - break; } - default: { - LOG_ERROR("Unknown LogRecordType."); - PELOTON_ASSERT(false); - } + PELOTON_MEMCPY(buf, buf+buf_curr, buf_rem); + PELOTON_MEMSET(buf+buf_rem, 0, buf_size - buf_rem); + buf_curr = buf_rem; } + delete[] buf; } -// Pass 2: -void WalRecovery::Pass2(char *buf, int len){ - (void) buf; - (void) len; - - CopySerializeInput record_decode((const void *)buf, len); - LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); - txn_id_t txn_id = record_decode.ReadLong(); - - (void) record_type; - - if(commited_txns_.find(txn_id)==commited_txns_.end()) { - return; - } - - int curr_offset = commited_txns_[txn_id].second; - PL_MEMCPY(log_buffer_+curr_offset, buf, len); - commited_txns_[txn_id].second += len; - -} void WalRecovery::ReplayLogFile(){ @@ -240,6 +237,8 @@ void WalRecovery::ReplayLogFile(){ if(it->second.first != LogRecordType::TRANSACTION_COMMIT) continue; + LOG_INFO("txn_id = %llu length = %d", it->first, it->second.second); + auto offset_pair = std::make_pair(curr_txn_offset, curr_txn_offset); commited_txns_.insert(std::make_pair(it->first, offset_pair)); From 94a4c921e031b7f05ec92da8c1a971adf902969c Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Tue, 1 May 2018 15:17:38 -0400 Subject: [PATCH 58/81] recovery: reset epoch to max_epoch in log --- src/include/logging/wal_recovery.h | 13 ++- src/logging/wal_recovery.cpp | 140 ++++++++++++----------------- 2 files changed, 69 insertions(+), 84 deletions(-) diff --git a/src/include/logging/wal_recovery.h b/src/include/logging/wal_recovery.h index e24547c9a50..80ca94b77e2 100644 --- a/src/include/logging/wal_recovery.h +++ b/src/include/logging/wal_recovery.h @@ -60,7 +60,9 @@ class WalRecovery{ public: WalRecovery() : - log_buffer_size_(0), log_buffer_(nullptr) {} + max_epoch_id_(INVALID_EID), + log_buffer_size_(0), + log_buffer_(nullptr) {} ~WalRecovery(){} @@ -71,14 +73,19 @@ class WalRecovery{ void ReplayLogFile(); void ParseFromDisk(ReplayStage stage); -// void Pass1(char *buf, int len); - void Pass2(char *buf, int len); + void ReplayAllTxns(); + void ReplaySingleTxn(txn_id_t txn_id); + + + //TODO(graghura): don't hardcode the path std::string logpath_ = "/tmp/log"; std::fstream fstream_; + eid_t max_epoch_id_; + int log_buffer_size_; char *log_buffer_; diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index cc644815f91..8a6b06edcee 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -1,5 +1,6 @@ +#include #include "logging/wal_recovery.h" #include "logging/log_util.h" #include "common/logger.h" @@ -21,82 +22,6 @@ void WalRecovery::StartRecovery(){ } -//void WalRecovery::InstallInsertRecord(){ -// -//} -// -//void WalRecovery::InstallDeleteRecord(){ -// -//} -// -//void WalRecovery::InstallUpdateRecord(){ -// -//} - -//LogRecord WalRecovery::ReadRecord(CopySerializeInput record_decode){ -// -// LogRecordType record_type = -// (LogRecordType)(record_decode.ReadEnumInSingleByte()); -// -// eid_t record_eid = record_decode.ReadLong(); -// txn_id_t record_txn_id = record_decode.ReadLong(); -// cid_t record_cid = record_decode.ReadLong(); -// -// -// switch(record_type){ -// case LogRecordType::TRANSACTION_BEGIN: -// case LogRecordType::TRANSACTION_ABORT: -// case LogRecordType::TRANSACTION_COMMIT: { -// return logging::LogRecordFactory::CreateTupleRecord( -// record_type, record_eid, record_txn_id, -// record_cid); -// } -// -// case LogRecordType::TUPLE_INSERT: -// case LogRecordType::TUPLE_DELETE: { -// oid_t database_id = (oid_t)record_decode.ReadLong(); -// oid_t table_id = (oid_t)record_decode.ReadLong(); -// -// oid_t tg_block = (oid_t)record_decode.ReadLong(); -// oid_t tg_offset = (oid_t)record_decode.ReadLong(); -// -// return logging::LogRecordFactory::CreateTupleRecord( -// record_type, ItemPointer(tg_block, tg_offset), -// record_eid, record_txn_id, -// record_cid); -// } -// -// -// case LogRecordType::TUPLE_UPDATE: -// oid_t database_id = (oid_t)record_decode.ReadLong(); -// oid_t table_id = (oid_t)record_decode.ReadLong(); -// -// oid_t old_tg_block = (oid_t)record_decode.ReadLong(); -// oid_t old_tg_offset = (oid_t)record_decode.ReadLong(); -// -// oid_t tg_block = (oid_t)record_decode.ReadLong(); -// oid_t tg_offset = (oid_t)record_decode.ReadLong(); -// -// return logging::LogRecordFactory::CreateTupleRecord( -// record_type, ItemPointer(old_tg_block, old_tg_offset), -// ItemPointer(tg_block, tg_offset), -// record_eid, record_txn_id, -// record_cid); -// -// -// default: -// LOG_ERROR("Unknown LogRecordType"); -// PL_ASSERT(false); -// -// } -// -//} - -//void WalRecovery::ReplayLogRecord(LogRecord &log_record){ -// -//} -// - void WalRecovery::ParseFromDisk(ReplayStage stage){ int buf_size = 4096, buf_rem = 0, buf_curr = 0,total_len=0; @@ -221,6 +146,57 @@ void WalRecovery::ParseFromDisk(ReplayStage stage){ delete[] buf; } +void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ + int start_offset = commited_txns_[txn_id].first; + int curr_offset = start_offset; + int total_len = all_txns_[txn_id].second; + + while(total_len > 0){ + +// LOG_INFO("length curr_offset = %d", curr_offset); + + CopySerializeInput length_decode((const void *)(log_buffer_+curr_offset), sizeof(int)); + int record_len = length_decode.ReadInt(); + + curr_offset += sizeof(int); + +// LOG_INFO("length curr_offset = %d, record_len = %d", curr_offset, record_len); + + CopySerializeInput record_decode((const void *)(log_buffer_+curr_offset), record_len); + LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); + txn_id_t txn_id = record_decode.ReadLong(); + eid_t epoch_id = record_decode.ReadLong(); + + (void) txn_id; +// LOG_INFO("replaying txn %llu", txn_id); + + if(max_epoch_id_==INVALID_EID) + max_epoch_id_ = epoch_id; + else if(epoch_id>max_epoch_id_) + max_epoch_id_ = epoch_id; + + (void) record_type; + + curr_offset += record_len; + total_len -= record_len + sizeof(int); + } + +} + +void WalRecovery::ReplayAllTxns(){ + + + for(auto it = commited_txns_.begin(); it!=commited_txns_.end(); it++){ + ReplaySingleTxn(it->first); + } + + // reset epoch + if (max_epoch_id_ != INVALID_EID) { + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(max_epoch_id_+1); + } +} + void WalRecovery::ReplayLogFile(){ @@ -230,14 +206,14 @@ void WalRecovery::ReplayLogFile(){ // Pass 1 ParseFromDisk(ReplayStage::PASS_1); - LOG_INFO("all_txns_len = %zu", all_txns_.size()); +// LOG_INFO("all_txns_len = %zu", all_txns_.size()); for(auto it = all_txns_.begin(); it!=all_txns_.end(); it++){ if(it->second.first != LogRecordType::TRANSACTION_COMMIT) continue; - LOG_INFO("txn_id = %llu length = %d", it->first, it->second.second); +// LOG_INFO("txn_id = %llu length = %d", it->first, it->second.second); auto offset_pair = std::make_pair(curr_txn_offset, curr_txn_offset); commited_txns_.insert(std::make_pair(it->first, offset_pair)); @@ -245,14 +221,16 @@ void WalRecovery::ReplayLogFile(){ curr_txn_offset += it->second.second; } - for(auto it = commited_txns_.begin(); it!=commited_txns_.end(); it++){ - LOG_INFO("txn_id = %llu offset = %d", it->first, it->second.first); - } +// for(auto it = commited_txns_.begin(); it!=commited_txns_.end(); it++){ +// LOG_INFO("txn_id = %llu offset = %d", it->first, it->second.first); +// } // Pass 2 log_buffer_ = new char[log_buffer_size_]; ParseFromDisk(ReplayStage::PASS_2); + ReplayAllTxns(); + // // DEBUG INFO // for(auto it = commited_txns.begin(); it!=commited_txns.end(); it++){ From f174beaa6539f5b50467d6c12d930c2506e4fe21 Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Tue, 1 May 2018 19:38:21 -0400 Subject: [PATCH 59/81] recovery: recovers a database --- src/include/logging/wal_recovery.h | 6 +- src/logging/wal_recovery.cpp | 119 ++++++++++++++++++++++++++++- 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/src/include/logging/wal_recovery.h b/src/include/logging/wal_recovery.h index 80ca94b77e2..451e6c40b86 100644 --- a/src/include/logging/wal_recovery.h +++ b/src/include/logging/wal_recovery.h @@ -5,6 +5,8 @@ #include "type/serializeio.h" #include "logging/log_record.h" +#include "storage/storage_manager.h" +#include "storage/tuple.h" namespace peloton{ namespace logging { @@ -76,7 +78,9 @@ class WalRecovery{ void ReplayAllTxns(); void ReplaySingleTxn(txn_id_t txn_id); - + bool InstallCatalogTuple(LogRecordType type, storage::Tuple *tuple, + storage::DataTable *table, cid_t cur_cid, + ItemPointer location); diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index 8a6b06edcee..32b09f65830 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -1,6 +1,10 @@ -#include +#include "concurrency/transaction_manager_factory.h" +#include "catalog/catalog_defaults.h" +#include "type/ephemeral_pool.h" +#include "catalog/manager.h" +#include "concurrency/epoch_manager_factory.h" #include "logging/wal_recovery.h" #include "logging/log_util.h" #include "common/logger.h" @@ -146,10 +150,54 @@ void WalRecovery::ParseFromDisk(ReplayStage stage){ delete[] buf; } +bool WalRecovery::InstallCatalogTuple(LogRecordType record_type, storage::Tuple *tuple, + storage::DataTable *table, cid_t cur_cid, + ItemPointer location){ + + ItemPointer *i = nullptr; + + auto& txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(IsolationLevelType::SERIALIZABLE); + + oid_t tile_group_id = location.block; + auto tg = table->GetTileGroupById(location.block); + auto tile_group_header = + catalog::Manager::GetInstance().GetTileGroup(tile_group_id)->GetHeader(); + +// oid_t tuple_slot = location.offset; + + if(record_type==LogRecordType::TUPLE_INSERT){ + + PL_ASSERT(tg!= nullptr); + + tg->CopyTuple(tuple, location.offset); + bool result = table->InsertTuple(tuple, location, txn, &i); + + (void) result; + + PL_ASSERT(result==true); + + tile_group_header->SetBeginCommitId(location.offset, cur_cid); + tile_group_header->SetEndCommitId(location.offset, MAX_CID); + tile_group_header->SetTransactionId(location.offset, INITIAL_TXN_ID); + tile_group_header->SetNextItemPointer(location.offset, + INVALID_ITEMPOINTER); + + tile_group_header->SetIndirection(location.offset,i); + } + + txn_manager.CommitTransaction(txn); + return true; +} + + + void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ int start_offset = commited_txns_[txn_id].first; int curr_offset = start_offset; int total_len = all_txns_[txn_id].second; + std::unique_ptr pool(new type::EphemeralPool()); + while(total_len > 0){ @@ -163,19 +211,84 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ // LOG_INFO("length curr_offset = %d, record_len = %d", curr_offset, record_len); CopySerializeInput record_decode((const void *)(log_buffer_+curr_offset), record_len); + LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); txn_id_t txn_id = record_decode.ReadLong(); eid_t epoch_id = record_decode.ReadLong(); + cid_t commit_id = record_decode.ReadLong(); (void) txn_id; -// LOG_INFO("replaying txn %llu", txn_id); if(max_epoch_id_==INVALID_EID) max_epoch_id_ = epoch_id; else if(epoch_id>max_epoch_id_) max_epoch_id_ = epoch_id; - (void) record_type; + switch(record_type){ + + case LogRecordType::TRANSACTION_BEGIN: { + break; + } + + case LogRecordType::TRANSACTION_COMMIT: { + //(graghura): return probably? + break; + } + + case LogRecordType::TRANSACTION_ABORT: { + LOG_ERROR("Should not be replaying an aborted transaction"); + PL_ASSERT(false); + } + + case LogRecordType::TUPLE_INSERT: { + oid_t database_id = (oid_t) record_decode.ReadLong(); + oid_t table_id = (oid_t) record_decode.ReadLong(); + + oid_t tg_block = (oid_t) record_decode.ReadLong(); + oid_t tg_offset = (oid_t) record_decode.ReadLong(); + + ItemPointer location(tg_block, tg_offset); + auto table = storage::StorageManager::GetInstance()->GetTableWithOid( + database_id, table_id); + auto schema = table->GetSchema(); + auto tg = table->GetTileGroupById(tg_block); + + // Create the tile group if it does not exist + if (tg.get() == nullptr) { + table->AddTileGroupWithOidForRecovery(tg_block); + tg = table->GetTileGroupById(tg_block); + catalog::Manager::GetInstance().GetNextTileGroupId(); + } + + std::unique_ptr tuple(new storage::Tuple(schema, true)); + for (oid_t oid = 0; oid < schema->GetColumns().size(); oid++) { + type::Value val = type::Value::DeserializeFrom( + record_decode, schema->GetColumn(oid).GetType()); + tuple->SetValue(oid, val, pool.get()); + } + + if (database_id == CATALOG_DATABASE_OID) { // catalog database oid + + InstallCatalogTuple(record_type, tuple.get(), table, commit_id, + location); + } + + break; + } + + case LogRecordType::TUPLE_UPDATE: { + break; + } + + case LogRecordType::TUPLE_DELETE: { + break; + } + + default:{ + break; + } + } + curr_offset += record_len; total_len -= record_len + sizeof(int); From dd04cd000f7574cebec603121199f6abf2223381 Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Wed, 2 May 2018 19:22:34 -0400 Subject: [PATCH 60/81] removed stale values from tcop_txn_state_ --- src/traffic_cop/traffic_cop.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/traffic_cop/traffic_cop.cpp b/src/traffic_cop/traffic_cop.cpp index bbd0345539a..8cd8033913e 100644 --- a/src/traffic_cop/traffic_cop.cpp +++ b/src/traffic_cop/traffic_cop.cpp @@ -215,7 +215,12 @@ executor::ExecutionResult TrafficCop::ExecuteHelper( ResultType TrafficCop::ExecuteStatementPlanGetResult() { - if (p_status_.m_result == ResultType::FAILURE) return ResultType::FAILURE; + if (p_status_.m_result == ResultType::FAILURE) { + if(!tcop_txn_state_.empty()) { + tcop_txn_state_.pop(); + } + return ResultType::FAILURE; + } if(tcop_txn_state_.empty()) return ResultType::NOOP; From 12acc7722d249f4532e21dfae0222a0102413f31 Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Wed, 2 May 2018 21:45:05 -0400 Subject: [PATCH 61/81] update pd_database NextOid during recovery --- src/logging/wal_recovery.cpp | 87 ++++++++++++++---------------------- 1 file changed, 34 insertions(+), 53 deletions(-) diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index 32b09f65830..11556c0bc37 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -1,5 +1,6 @@ +#include "catalog/database_catalog.h" #include "concurrency/transaction_manager_factory.h" #include "catalog/catalog_defaults.h" #include "type/ephemeral_pool.h" @@ -150,6 +151,8 @@ void WalRecovery::ParseFromDisk(ReplayStage stage){ delete[] buf; } + + bool WalRecovery::InstallCatalogTuple(LogRecordType record_type, storage::Tuple *tuple, storage::DataTable *table, cid_t cur_cid, ItemPointer location){ @@ -160,29 +163,28 @@ bool WalRecovery::InstallCatalogTuple(LogRecordType record_type, storage::Tuple auto txn = txn_manager.BeginTransaction(IsolationLevelType::SERIALIZABLE); oid_t tile_group_id = location.block; - auto tg = table->GetTileGroupById(location.block); + auto tile_group = table->GetTileGroupById(location.block); auto tile_group_header = catalog::Manager::GetInstance().GetTileGroup(tile_group_id)->GetHeader(); -// oid_t tuple_slot = location.offset; - if(record_type==LogRecordType::TUPLE_INSERT){ - PL_ASSERT(tg!= nullptr); + PL_ASSERT(tile_group != nullptr); + + auto status = tile_group_header->GetEmptyTupleSlot(location.offset); - tg->CopyTuple(tuple, location.offset); + (void) status; + + tile_group->CopyTuple(tuple, location.offset); bool result = table->InsertTuple(tuple, location, txn, &i); (void) result; - PL_ASSERT(result==true); - tile_group_header->SetBeginCommitId(location.offset, cur_cid); tile_group_header->SetEndCommitId(location.offset, MAX_CID); tile_group_header->SetTransactionId(location.offset, INITIAL_TXN_ID); tile_group_header->SetNextItemPointer(location.offset, INVALID_ITEMPOINTER); - tile_group_header->SetIndirection(location.offset,i); } @@ -190,8 +192,6 @@ bool WalRecovery::InstallCatalogTuple(LogRecordType record_type, storage::Tuple return true; } - - void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ int start_offset = commited_txns_[txn_id].first; int curr_offset = start_offset; @@ -201,15 +201,11 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ while(total_len > 0){ -// LOG_INFO("length curr_offset = %d", curr_offset); - CopySerializeInput length_decode((const void *)(log_buffer_+curr_offset), sizeof(int)); int record_len = length_decode.ReadInt(); curr_offset += sizeof(int); -// LOG_INFO("length curr_offset = %d, record_len = %d", curr_offset, record_len); - CopySerializeInput record_decode((const void *)(log_buffer_+curr_offset), record_len); LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); @@ -225,21 +221,6 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ max_epoch_id_ = epoch_id; switch(record_type){ - - case LogRecordType::TRANSACTION_BEGIN: { - break; - } - - case LogRecordType::TRANSACTION_COMMIT: { - //(graghura): return probably? - break; - } - - case LogRecordType::TRANSACTION_ABORT: { - LOG_ERROR("Should not be replaying an aborted transaction"); - PL_ASSERT(false); - } - case LogRecordType::TUPLE_INSERT: { oid_t database_id = (oid_t) record_decode.ReadLong(); oid_t table_id = (oid_t) record_decode.ReadLong(); @@ -267,12 +248,17 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ tuple->SetValue(oid, val, pool.get()); } + if (database_id == CATALOG_DATABASE_OID) { // catalog database oid - InstallCatalogTuple(record_type, tuple.get(), table, commit_id, - location); - } + InstallCatalogTuple(record_type, tuple.get(), table, commit_id, location); + if(table_id==DATABASE_CATALOG_OID) { + LOG_INFO("replaying database insert"); + auto pg_database = catalog::DatabaseCatalog::GetInstance(); + pg_database->GetNextOid(); + } + } break; } @@ -284,12 +270,26 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ break; } - default:{ + + case LogRecordType::TRANSACTION_BEGIN: { + // NOOP + break; + } + + case LogRecordType::TRANSACTION_COMMIT: { + //(graghura): return probably? break; } - } + case LogRecordType::TRANSACTION_ABORT: { + LOG_ERROR("Should not be replaying an aborted transaction"); + PL_ASSERT(false); + } + default:{ + break; + } + } curr_offset += record_len; total_len -= record_len + sizeof(int); } @@ -298,7 +298,6 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ void WalRecovery::ReplayAllTxns(){ - for(auto it = commited_txns_.begin(); it!=commited_txns_.end(); it++){ ReplaySingleTxn(it->first); } @@ -319,42 +318,24 @@ void WalRecovery::ReplayLogFile(){ // Pass 1 ParseFromDisk(ReplayStage::PASS_1); -// LOG_INFO("all_txns_len = %zu", all_txns_.size()); for(auto it = all_txns_.begin(); it!=all_txns_.end(); it++){ if(it->second.first != LogRecordType::TRANSACTION_COMMIT) continue; -// LOG_INFO("txn_id = %llu length = %d", it->first, it->second.second); - auto offset_pair = std::make_pair(curr_txn_offset, curr_txn_offset); commited_txns_.insert(std::make_pair(it->first, offset_pair)); curr_txn_offset += it->second.second; } -// for(auto it = commited_txns_.begin(); it!=commited_txns_.end(); it++){ -// LOG_INFO("txn_id = %llu offset = %d", it->first, it->second.first); -// } - // Pass 2 log_buffer_ = new char[log_buffer_size_]; ParseFromDisk(ReplayStage::PASS_2); ReplayAllTxns(); - -// // DEBUG INFO -// for(auto it = commited_txns.begin(); it!=commited_txns.end(); it++){ -// txn_id_t txn_id = it->first; -// int epoch_id = txn_id >> 32; -// int counter = txn_id & 0xFFFFFFFF; -// LOG_INFO("Replaying peloton txn : %llu, epoch %d, counter = %d", it->first, epoch_id, counter); -// } -// -// LOG_INFO("buf_size = %zu", log_buffer_size); - } From 8ac72f769c15679327b7620d472ba109464b1a40 Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Thu, 3 May 2018 20:57:55 -0400 Subject: [PATCH 62/81] 1. Fix database recovery bug - registered db with catalog 2. Completed table schema restore --- src/include/logging/wal_recovery.h | 4 + src/logging/wal_recovery.cpp | 199 +++++++++++++++++++++-------- 2 files changed, 149 insertions(+), 54 deletions(-) diff --git a/src/include/logging/wal_recovery.h b/src/include/logging/wal_recovery.h index 451e6c40b86..729458bedfc 100644 --- a/src/include/logging/wal_recovery.h +++ b/src/include/logging/wal_recovery.h @@ -83,6 +83,10 @@ class WalRecovery{ ItemPointer location); + void CreateTableOnRecovery(std::unique_ptr &tuple, + std::vector &columns); + + //TODO(graghura): don't hardcode the path std::string logpath_ = "/tmp/log"; diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index 11556c0bc37..3f3c0402bba 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -1,5 +1,7 @@ +#include +#include "catalog/table_catalog.h" #include "catalog/database_catalog.h" #include "concurrency/transaction_manager_factory.h" #include "catalog/catalog_defaults.h" @@ -192,15 +194,34 @@ bool WalRecovery::InstallCatalogTuple(LogRecordType record_type, storage::Tuple return true; } +void WalRecovery::CreateTableOnRecovery(std::unique_ptr &tuple, + std::vector &columns){ + + // Getting database + auto database = + storage::StorageManager::GetInstance()->GetDatabaseWithOid( + tuple->GetValue(2).GetAs()); + + // register table with catalog + database->AddTable(new storage::DataTable( + new catalog::Schema(columns), tuple->GetValue(1).ToString(), + database->GetOid(), tuple->GetValue(0).GetAs(), + DEFAULT_TUPLES_PER_TILEGROUP, true, false, false)); + + columns.clear(); +} + void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ int start_offset = commited_txns_[txn_id].first; int curr_offset = start_offset; int total_len = all_txns_[txn_id].second; std::unique_ptr pool(new type::EphemeralPool()); + bool pending_table_create; + std::unique_ptr tuple_table_create; + std::vector columns; while(total_len > 0){ - CopySerializeInput length_decode((const void *)(log_buffer_+curr_offset), sizeof(int)); int record_len = length_decode.ReadInt(); @@ -220,80 +241,150 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ else if(epoch_id>max_epoch_id_) max_epoch_id_ = epoch_id; - switch(record_type){ - case LogRecordType::TUPLE_INSERT: { - oid_t database_id = (oid_t) record_decode.ReadLong(); - oid_t table_id = (oid_t) record_decode.ReadLong(); - - oid_t tg_block = (oid_t) record_decode.ReadLong(); - oid_t tg_offset = (oid_t) record_decode.ReadLong(); - - ItemPointer location(tg_block, tg_offset); - auto table = storage::StorageManager::GetInstance()->GetTableWithOid( - database_id, table_id); - auto schema = table->GetSchema(); - auto tg = table->GetTileGroupById(tg_block); - - // Create the tile group if it does not exist - if (tg.get() == nullptr) { - table->AddTileGroupWithOidForRecovery(tg_block); - tg = table->GetTileGroupById(tg_block); - catalog::Manager::GetInstance().GetNextTileGroupId(); - } + // complete table_creation if pending + if(record_type != LogRecordType::TUPLE_INSERT){ + if(pending_table_create) { + CreateTableOnRecovery(tuple_table_create, columns); + pending_table_create = false; + } + } + + if(record_type==LogRecordType::TUPLE_INSERT) { - std::unique_ptr tuple(new storage::Tuple(schema, true)); - for (oid_t oid = 0; oid < schema->GetColumns().size(); oid++) { - type::Value val = type::Value::DeserializeFrom( - record_decode, schema->GetColumn(oid).GetType()); - tuple->SetValue(oid, val, pool.get()); + oid_t database_id = (oid_t) record_decode.ReadLong(); + oid_t table_id = (oid_t) record_decode.ReadLong(); + + if(database_id != CATALOG_DATABASE_OID || table_id != COLUMN_CATALOG_OID){ + if(pending_table_create){ + CreateTableOnRecovery(tuple_table_create, columns); + pending_table_create = false; } + } + oid_t tg_block = (oid_t) record_decode.ReadLong(); + oid_t tg_offset = (oid_t) record_decode.ReadLong(); - if (database_id == CATALOG_DATABASE_OID) { // catalog database oid + ItemPointer location(tg_block, tg_offset); + auto table = storage::StorageManager::GetInstance()->GetTableWithOid( + database_id, table_id); + auto schema = table->GetSchema(); + auto tg = table->GetTileGroupById(tg_block); + + // Create the tile group if it does not exist + if (tg.get() == nullptr) { + table->AddTileGroupWithOidForRecovery(tg_block); + tg = table->GetTileGroupById(tg_block); + catalog::Manager::GetInstance().GetNextTileGroupId(); + } + + std::unique_ptr tuple(new storage::Tuple(schema, true)); + for (oid_t oid = 0; oid < schema->GetColumns().size(); oid++) { + type::Value val = type::Value::DeserializeFrom( + record_decode, schema->GetColumn(oid).GetType()); + tuple->SetValue(oid, val, pool.get()); + } + if (database_id == CATALOG_DATABASE_OID) { // catalog database oid + + + if (table_id == DATABASE_CATALOG_OID) { + + LOG_INFO("REPLAYING INSERT TO PG_DATABASE"); + + // insert tuple into pg_database InstallCatalogTuple(record_type, tuple.get(), table, commit_id, location); - if(table_id==DATABASE_CATALOG_OID) { - LOG_INFO("replaying database insert"); - auto pg_database = catalog::DatabaseCatalog::GetInstance(); - pg_database->GetNextOid(); + auto storage_manager = storage::StorageManager::GetInstance(); + auto pg_database = catalog::DatabaseCatalog::GetInstance(); + + auto database_oid = tuple->GetValue(0).GetAs(); + auto database_name = tuple->GetValue(1).ToString(); + + storage::Database *database = new storage::Database(database_oid); + database->setDBName(database_name); + + // register database with storage manager + storage_manager->AddDatabaseToStorageManager(database); + + // update database oid to prevent oid reuse after recovery + pg_database->GetNextOid(); + + } else if (table_id == TABLE_CATALOG_OID) { + + LOG_INFO("REPLAYING INSERT TO PG_TABLE"); + + // insert table info into pg_table + InstallCatalogTuple(record_type, tuple.get(), table, commit_id, location); + + // update table oid to prevent oid reuse after recovery + catalog::TableCatalog::GetInstance()->GetNextOid(); + tuple_table_create = std::move(tuple); + pending_table_create = true; + + } else if (table_id == COLUMN_CATALOG_OID) { + + LOG_INFO("REPLAYING INSERT TO PG_COLUMN"); + + // insert tuple into pg_column - one for every attribute in the schema + InstallCatalogTuple(record_type, tuple.get(), table, commit_id, location); + + // update column oid to prevent oid reuse after recovery + catalog::ColumnCatalog::GetInstance()->GetNextOid(); + + bool is_inline = true; + + std::string typeId = tuple->GetValue(4).ToString(); + type::TypeId column_type = StringToTypeId(typeId); + + if (column_type == type::TypeId::VARCHAR || + column_type == type::TypeId::VARBINARY) { + is_inline = false; } - } - break; - } - case LogRecordType::TUPLE_UPDATE: { - break; - } + catalog::Column tmp_col(column_type, + type::Type::GetTypeSize(column_type), + tuple->GetValue(1).ToString(), is_inline, + tuple->GetValue(1).GetAs()); - case LogRecordType::TUPLE_DELETE: { - break; - } + uint index = stoi(tuple->GetValue(2).ToString()); + if (index >= columns.size()) { + columns.resize(index + 1); + } + columns[index] = tmp_col; - case LogRecordType::TRANSACTION_BEGIN: { - // NOOP - break; - } + } else if (table_id == INDEX_CATALOG_OID) { + LOG_INFO("REPLAYING INSERT TO PG_INDEX"); - case LogRecordType::TRANSACTION_COMMIT: { - //(graghura): return probably? - break; + } } + } - case LogRecordType::TRANSACTION_ABORT: { - LOG_ERROR("Should not be replaying an aborted transaction"); - PL_ASSERT(false); - } + else if(record_type==LogRecordType::TUPLE_UPDATE) { + LOG_INFO("Replaying TUPLE_UPDATE"); + } - default:{ - break; - } + else if(record_type==LogRecordType::TRANSACTION_BEGIN) { + LOG_INFO("Replaying TXN_BEGIN"); } + + else if(record_type==LogRecordType::TRANSACTION_COMMIT) { + LOG_INFO("Replaying TXN_COMMIT"); + } + + else if(record_type==LogRecordType::TRANSACTION_ABORT) { + LOG_ERROR("Shouldn't be replaying TXN_ABORT"); + PL_ASSERT(false); + } + curr_offset += record_len; total_len -= record_len + sizeof(int); } + if(pending_table_create) { + CreateTableOnRecovery(tuple_table_create, columns); + } + } void WalRecovery::ReplayAllTxns(){ From 629218c78ff326559eb44a3dffe539c3d49a1bbb Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Thu, 3 May 2018 21:33:50 -0400 Subject: [PATCH 63/81] completed index recovery --- src/include/catalog/table_catalog.h | 7 ++- src/logging/wal_recovery.cpp | 80 ++++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/src/include/catalog/table_catalog.h b/src/include/catalog/table_catalog.h index 3ef4668d5ca..0e6f5e78d5b 100644 --- a/src/include/catalog/table_catalog.h +++ b/src/include/catalog/table_catalog.h @@ -136,9 +136,12 @@ class TableCatalog : public AbstractCatalog { //===--------------------------------------------------------------------===// // Read Related API //===--------------------------------------------------------------------===// - private: + + // used by logger during recovery std::shared_ptr GetTableObject( - oid_t table_oid, concurrency::TransactionContext *txn); + oid_t table_oid, concurrency::TransactionContext *txn); + private: + std::shared_ptr GetTableObject( const std::string &table_name, oid_t database_oid, concurrency::TransactionContext *txn); diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index 3f3c0402bba..5b9717ccc47 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -1,6 +1,8 @@ +#include "index/index_factory.h" -#include +#include "catalog/index_catalog.h" +#include "catalog/column_catalog.h" #include "catalog/table_catalog.h" #include "catalog/database_catalog.h" #include "concurrency/transaction_manager_factory.h" @@ -217,6 +219,9 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ int total_len = all_txns_[txn_id].second; std::unique_ptr pool(new type::EphemeralPool()); + std::vector> indexes; + + bool pending_table_create; std::unique_ptr tuple_table_create; std::vector columns; @@ -355,7 +360,8 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ } else if (table_id == INDEX_CATALOG_OID) { LOG_INFO("REPLAYING INSERT TO PG_INDEX"); - + indexes.push_back(std::move(tuple)); + catalog::IndexCatalog::GetInstance()->GetNextOid(); } } } @@ -385,6 +391,76 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ CreateTableOnRecovery(tuple_table_create, columns); } + + std::set tables_with_indexes; + + // Index construction that was deferred from the read records phase + for (auto const &tup : indexes) { + LOG_INFO("Install Index"); + std::vector key_attrs; + oid_t key_attr; + auto txn = concurrency::TransactionManagerFactory::GetInstance().BeginTransaction( + IsolationLevelType::SERIALIZABLE); + + auto table_catalog = catalog::TableCatalog::GetInstance(); + auto table_object = table_catalog->GetTableObject(tup->GetValue(2).GetAs(), txn); + auto database_oid = table_object->GetDatabaseOid(); + auto table = storage::StorageManager::GetInstance()->GetTableWithOid( + database_oid, table_object->GetTableOid()); + concurrency::TransactionManagerFactory::GetInstance().CommitTransaction( + txn); + auto tuple_schema = table->GetSchema(); + std::stringstream iss(tup->GetValue(6).ToString()); + while (iss >> key_attr) key_attrs.push_back(key_attr); + auto key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs); + key_schema->SetIndexedColumns(key_attrs); + + index::IndexMetadata *index_metadata = new index::IndexMetadata( + tup->GetValue(1).ToString(), tup->GetValue(0).GetAs(), + table->GetOid(), database_oid, + static_cast(tup->GetValue(3).GetAs()), + static_cast(tup->GetValue(4).GetAs()), + tuple_schema, key_schema, key_attrs, tup->GetValue(4).GetAs()); + std::shared_ptr index( + index::IndexFactory::GetIndex(index_metadata)); + table->AddIndex(index); + tables_with_indexes.insert(table); + // Attributes must be changed once we have arraytype + } + + // add tuples to index + + for (storage::DataTable *table : tables_with_indexes) { + LOG_INFO("Install table_index"); + auto schema = table->GetSchema(); + size_t tg_count = table->GetTileGroupCount(); + for (oid_t tg = 0; tg < tg_count; tg++) { + auto tile_group = table->GetTileGroup(tg); + + for (int tuple_slot_id = START_OID; tuple_slot_id < DEFAULT_TUPLES_PER_TILEGROUP; + tuple_slot_id++) { + txn_id_t tuple_txn_id = tile_group->GetHeader()->GetTransactionId(tuple_slot_id); + if (tuple_txn_id != INVALID_TXN_ID) { + PL_ASSERT(tuple_txn_id == INITIAL_TXN_ID); + std::unique_ptr t(new storage::Tuple(schema, true)); + for (size_t col = 0; col < schema->GetColumnCount(); col++) { + t->SetValue(col, tile_group->GetValue(tuple_slot_id, col), pool.get()); + } + ItemPointer p(tile_group->GetTileGroupId(), tuple_slot_id); + auto txn = concurrency::TransactionManagerFactory::GetInstance() + .BeginTransaction(IsolationLevelType::SERIALIZABLE); + ItemPointer *ip = nullptr; + table->InsertInIndexes(t.get(), p, txn, &ip); + tile_group->GetHeader()->SetIndirection(tuple_slot_id, ip); + concurrency::TransactionManagerFactory::GetInstance().CommitTransaction( + txn); + } + } + + + } + } + } void WalRecovery::ReplayAllTxns(){ From 46ceb0ed37161887463e36850d90a7ad9e3b7b9d Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Fri, 4 May 2018 10:07:18 -0400 Subject: [PATCH 64/81] recovery: recovers a tuple inserts --- src/logging/wal_recovery.cpp | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index 5b9717ccc47..54bed077053 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -363,6 +363,20 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ indexes.push_back(std::move(tuple)); catalog::IndexCatalog::GetInstance()->GetNextOid(); } + } else { + + LOG_INFO("REPLAYING INSERT TO NON CATALOG TABLE"); + + // insert tuple + auto tuple_id = tg->InsertTupleFromRecovery(commit_id, tg_offset, tuple.get()); + table->IncreaseTupleCount(1); + if (tuple_id == tg->GetAllocatedTupleCount() - 1) { + // add a new tile group if the last slot has been taken + if (table->GetTileGroupById(tg->GetTileGroupId() + 1).get() == + nullptr) + table->AddTileGroupWithOidForRecovery(tg->GetTileGroupId() + 1); + catalog::Manager::GetInstance().GetNextTileGroupId(); + } } } @@ -429,7 +443,6 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ } // add tuples to index - for (storage::DataTable *table : tables_with_indexes) { LOG_INFO("Install table_index"); auto schema = table->GetSchema(); @@ -456,8 +469,6 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ txn); } } - - } } From e00732c10dbbdce5a88d340918d7a0a70a3fdd03 Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Fri, 4 May 2018 10:48:41 -0400 Subject: [PATCH 65/81] recovery: recovers tuple-deletes from non-catalog tables. (drop table recovery is broken) --- src/logging/wal_recovery.cpp | 61 +++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index 54bed077053..197694562cf 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -312,7 +312,7 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ storage_manager->AddDatabaseToStorageManager(database); // update database oid to prevent oid reuse after recovery - pg_database->GetNextOid(); + pg_database->GetNextOid(); // TODO(graghura): This is a hack, rewrite !!! } else if (table_id == TABLE_CATALOG_OID) { @@ -322,7 +322,7 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ InstallCatalogTuple(record_type, tuple.get(), table, commit_id, location); // update table oid to prevent oid reuse after recovery - catalog::TableCatalog::GetInstance()->GetNextOid(); + catalog::TableCatalog::GetInstance()->GetNextOid(); // TODO(graghura): This is a hack, rewrite !!! tuple_table_create = std::move(tuple); pending_table_create = true; @@ -334,7 +334,7 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ InstallCatalogTuple(record_type, tuple.get(), table, commit_id, location); // update column oid to prevent oid reuse after recovery - catalog::ColumnCatalog::GetInstance()->GetNextOid(); + catalog::ColumnCatalog::GetInstance()->GetNextOid(); // TODO(graghura): This is a hack, rewrite !!! bool is_inline = true; @@ -361,7 +361,7 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ } else if (table_id == INDEX_CATALOG_OID) { LOG_INFO("REPLAYING INSERT TO PG_INDEX"); indexes.push_back(std::move(tuple)); - catalog::IndexCatalog::GetInstance()->GetNextOid(); + catalog::IndexCatalog::GetInstance()->GetNextOid(); // TODO(graghura): This is a hack, rewrite !!! } } else { @@ -380,6 +380,59 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ } } + else if(record_type==LogRecordType::TUPLE_DELETE) { + LOG_INFO("Tuple Delete"); + + oid_t database_id = (oid_t)record_decode.ReadLong(); + oid_t table_id = (oid_t)record_decode.ReadLong(); + + oid_t tg_block = (oid_t)record_decode.ReadLong(); + oid_t tg_offset = (oid_t)record_decode.ReadLong(); + + auto table = storage::StorageManager::GetInstance()->GetTableWithOid( + database_id, table_id); + auto tg = table->GetTileGroupById(tg_block); + + // This code might be useful on drop + if (database_id == CATALOG_DATABASE_OID) { // catalog database oid + switch (table_id) { + + case DATABASE_CATALOG_OID:{ //pg_database + // TODO(graghura): delete database + } + + case TABLE_CATALOG_OID: { // pg_table + auto db_oid = tg->GetValue(tg_offset, 2).GetAs(); + auto table_oid = tg->GetValue(tg_offset, 0).GetAs(); + auto database = + storage::StorageManager::GetInstance()->GetDatabaseWithOid( + db_oid); // Getting database oid from pg_table + database->DropTableWithOid(table_oid); + break; + } + + case INDEX_CATALOG_OID: { // pg_index + // TODO(graghura): delete index + } + + case COLUMN_CATALOG_OID: { // pg_column + // TODO(graghura): delete column (I think schema updates are not supported now) + } + + } + } + + auto tuple_id = tg->DeleteTupleFromRecovery(commit_id, tg_offset); + table->IncreaseTupleCount(1); + if (tuple_id == tg->GetAllocatedTupleCount() - 1) { + if (table->GetTileGroupById(tg->GetTileGroupId() + 1).get() == + nullptr) + table->AddTileGroupWithOidForRecovery(tg->GetTileGroupId() + 1); + catalog::Manager::GetInstance().GetNextTileGroupId(); + } + break; + } + else if(record_type==LogRecordType::TUPLE_UPDATE) { LOG_INFO("Replaying TUPLE_UPDATE"); } From 544c6f9a244ff35b7040bef013290c9fde17c652 Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Fri, 4 May 2018 20:13:12 -0400 Subject: [PATCH 66/81] PL_* to PELOTON_* --- src/logging/wal_recovery.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index 197694562cf..7e415e984f3 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -173,7 +173,7 @@ bool WalRecovery::InstallCatalogTuple(LogRecordType record_type, storage::Tuple if(record_type==LogRecordType::TUPLE_INSERT){ - PL_ASSERT(tile_group != nullptr); + PELOTON_ASSERT(tile_group != nullptr); auto status = tile_group_header->GetEmptyTupleSlot(location.offset); @@ -447,7 +447,7 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ else if(record_type==LogRecordType::TRANSACTION_ABORT) { LOG_ERROR("Shouldn't be replaying TXN_ABORT"); - PL_ASSERT(false); + PELOTON_ASSERT(false); } curr_offset += record_len; @@ -507,7 +507,7 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ tuple_slot_id++) { txn_id_t tuple_txn_id = tile_group->GetHeader()->GetTransactionId(tuple_slot_id); if (tuple_txn_id != INVALID_TXN_ID) { - PL_ASSERT(tuple_txn_id == INITIAL_TXN_ID); + PELOTON_ASSERT(tuple_txn_id == INITIAL_TXN_ID); std::unique_ptr t(new storage::Tuple(schema, true)); for (size_t col = 0; col < schema->GetColumnCount(); col++) { t->SetValue(col, tile_group->GetValue(tuple_slot_id, col), pool.get()); From 07f2e2295b18c009f046aabdbc5e5f721f6e32e3 Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Fri, 4 May 2018 21:45:14 -0400 Subject: [PATCH 67/81] Few bug fixes and code cleanup --- src/codegen/buffering_consumer.cpp | 36 --------------------------- src/executor/insert_executor.cpp | 2 -- src/include/logging/wal_log_manager.h | 2 +- src/logging/log_buffer.cpp | 7 +----- src/logging/wal_recovery.cpp | 6 ++--- 5 files changed, 5 insertions(+), 48 deletions(-) diff --git a/src/codegen/buffering_consumer.cpp b/src/codegen/buffering_consumer.cpp index cdf7db23c5c..bfd7dd1d5c6 100644 --- a/src/codegen/buffering_consumer.cpp +++ b/src/codegen/buffering_consumer.cpp @@ -136,42 +136,6 @@ void BufferingConsumer::ConsumeResult(ConsumerContext &ctx, PELOTON_ASSERT(output_ais_[i]->type == val.GetType()); AddToTupleBuffer(val, codegen, tuple_buffer_, i); - PELOTON_ASSERT(output_ais_[i]->type == val.GetType()); - const auto &sql_type = val.GetType().GetSqlType(); - - // Check if it's NULL - Value null_val; - lang::If val_is_null{codegen, val.IsNull(codegen)}; - { - // If the value is NULL (i.e., has the NULL bit set), produce the NULL - // value for the given type. - null_val = sql_type.GetNullValue(codegen); - } - val_is_null.EndIf(); - val = val_is_null.BuildPHI(null_val, val); - - // Output the value using the type's output function - auto *output_func = sql_type.GetOutputFunction(codegen, val.GetType()); - - // Setup the function arguments - std::vector args = {tuple_buffer_, codegen.Const32(i), - val.GetValue()}; - // If the value is a string, push back the length - if (val.GetLength() != nullptr) { - args.push_back(val.GetLength()); - } - - // If the value is a boolean, push back the NULL bit. We don't do that for - // the other data types because we have special values for NULL. Booleans - // in codegen are 1-bit types, as opposed to 1-byte types in the rest of the - // system. Since, we cannot have a special value for NULL in a 1-bit boolean - // system, we pass along the NULL bit during output. - if (sql_type.TypeId() == peloton::type::TypeId::BOOLEAN) { - args.push_back(val.IsNull(codegen)); - } - - // Call the function - codegen.CallFunc(output_func, args); } // Append the tuple to the output buffer (by calling BufferTuple(...)) diff --git a/src/executor/insert_executor.cpp b/src/executor/insert_executor.cpp index 4a683e5b3a9..d895de38e34 100644 --- a/src/executor/insert_executor.cpp +++ b/src/executor/insert_executor.cpp @@ -193,7 +193,6 @@ bool InsertExecutor::DExecute() { tuple = node.GetTuple(insert_itr); if (tuple == nullptr) { - LOG_INFO("tuple is null"); storage_tuple.reset(new storage::Tuple(schema, true)); // read from values @@ -255,7 +254,6 @@ bool InsertExecutor::DExecute() { values.push_back(new_tuple->GetValue(col_id)); } -// LOG_INFO("Perform insert is being called"); transaction_manager.PerformInsert(current_txn, location, index_entry_ptr, reinterpret_cast(values.data()), values.size()); LOG_TRACE("Number of tuples in table after insert: %lu", diff --git a/src/include/logging/wal_log_manager.h b/src/include/logging/wal_log_manager.h index 5f15df7f05d..ace4a8176b4 100644 --- a/src/include/logging/wal_log_manager.h +++ b/src/include/logging/wal_log_manager.h @@ -69,7 +69,7 @@ class LogManager{ inline bool IsLoggingEnabled() { return enable_logging_; } inline std::string GetDirectory() { return directory_; } - inline std::string GetLogFilePath() { return directory_ + log_file_; } + inline std::string GetLogFilePath() { return directory_ + "/" + log_file_; } inline void DoRecovery(){ WalRecovery* wr = new WalRecovery(); diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index e07139598cb..dec8f88afa0 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -33,7 +33,6 @@ void LogBuffer::WriteRecord(LogRecord &record) { switch (type) { case LogRecordType::TUPLE_INSERT: { -// LOG_INFO("inserting tuple"); auto &manager = catalog::Manager::GetInstance(); auto tuple_pos = record.GetItemPointer(); auto tg = manager.GetTileGroup(tuple_pos.block).get(); @@ -48,14 +47,12 @@ void LogBuffer::WriteRecord(LogRecord &record) { peloton::type::Value *values_array = reinterpret_cast(record.GetValuesArray()); for (uint32_t i = 0; i < record.GetNumValues(); i++) { - //TODO(akanjani): Check if just copying the offset info will perform better values_array[i].SerializeTo(log_buffer_); } break; } case LogRecordType::TUPLE_DELETE: { -// LOG_INFO("Deleting tuple"); auto &manager = catalog::Manager::GetInstance(); auto tuple_pos = record.GetItemPointer(); auto tg = manager.GetTileGroup(tuple_pos.block).get(); @@ -70,7 +67,6 @@ void LogBuffer::WriteRecord(LogRecord &record) { break; } case LogRecordType::TUPLE_UPDATE: { -// LOG_INFO("Updating tuple"); auto &manager = catalog::Manager::GetInstance(); auto tuple_pos = record.GetItemPointer(); auto old_tuple_pos = record.GetOldItemPointer(); @@ -91,7 +87,6 @@ void LogBuffer::WriteRecord(LogRecord &record) { peloton::type::Value *values_array = reinterpret_cast(record.GetValuesArray()); TargetList *offsets = record.GetOffsets(); for (uint32_t i = 0; i < record.GetNumValues(); i++) { - //TODO(akanjani): Check if just copying the offset info will perform better log_buffer_.WriteInt(((*offsets)[i]).first); values_array[i].SerializeTo(log_buffer_); } @@ -118,4 +113,4 @@ void LogBuffer::WriteRecord(LogRecord &record) { } } -} \ No newline at end of file +} diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index 7e415e984f3..b52a3ab91cd 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -133,7 +133,7 @@ void WalRecovery::ParseFromDisk(ReplayStage stage){ int copy_len = record_len + sizeof(record_len); int curr_offset = commited_txns_[txn_id].second; - LOG_INFO("txn %llu writing from %d to %d", txn_id, curr_offset, curr_offset+copy_len-1); + LOG_INFO("txn %lu writing from %d to %d", txn_id, curr_offset, curr_offset+copy_len-1); PELOTON_MEMCPY(log_buffer_+curr_offset, buf+buf_curr, copy_len); commited_txns_[txn_id].second += copy_len; @@ -222,7 +222,7 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ std::vector> indexes; - bool pending_table_create; + bool pending_table_create = false; std::unique_ptr tuple_table_create; std::vector columns; @@ -572,4 +572,4 @@ void WalRecovery::ReplayLogFile(){ } -} \ No newline at end of file +} From 5e9b17b61f99a7531b0a7f71860fe693f238265e Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Fri, 4 May 2018 22:27:16 -0400 Subject: [PATCH 68/81] code refactoring --- src/common/init.cpp | 10 ++++++---- src/include/common/init.h | 4 +--- src/include/logging/wal_log_manager.h | 22 +++++++++++----------- src/include/logging/wal_recovery.h | 7 +++---- src/include/threadpool/logger_queue_pool.h | 3 --- src/logging/log_buffer.cpp | 2 -- src/logging/wal_logger.cpp | 2 -- src/logging/wal_recovery.cpp | 3 +-- 8 files changed, 22 insertions(+), 31 deletions(-) diff --git a/src/common/init.cpp b/src/common/init.cpp index 16a7dcfb22e..19b1fdbdada 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -31,7 +31,7 @@ namespace peloton { ThreadPool thread_pool; -void PelotonInit::Initialize(std::string log_dir, std::string log_file, bool enable_logging) { +void PelotonInit::Initialize(bool enable_logging) { CONNECTION_THREAD_COUNT = settings::SettingsManager::GetInt( settings::SettingId::connection_thread_count); LOGGING_THREAD_COUNT = 1; @@ -92,14 +92,16 @@ void PelotonInit::Initialize(std::string log_dir, std::string log_file, bool ena // Initialize the Statement Cache Manager StatementCacheManager::Init(); - logging::LogManager::GetInstance().DoRecovery(); - if(enable_logging){ - if(!logging::LogManager::GetInstance().init(log_dir, log_file)){ + if(!logging::LogManager::GetInstance().init()){ LOG_ERROR("LogManager Initialization failed"); } } + logging::LogManager::GetInstance().DoRecovery(); + + + threadpool::LoggerQueuePool::GetInstance().Startup(); diff --git a/src/include/common/init.h b/src/include/common/init.h index dc7ed223629..9e573891751 100644 --- a/src/include/common/init.h +++ b/src/include/common/init.h @@ -26,9 +26,7 @@ extern ThreadPool thread_pool; class PelotonInit { public: - static void Initialize(std::string log_dir = settings::SettingsManager::GetString(settings::SettingId::log_directory_name), - std::string log_file = settings::SettingsManager::GetString(settings::SettingId::log_file_name), - bool enable_logging = true); + static void Initialize(bool enable_logging = true); static void Shutdown(); diff --git a/src/include/logging/wal_log_manager.h b/src/include/logging/wal_log_manager.h index 5f15df7f05d..f0160babf21 100644 --- a/src/include/logging/wal_log_manager.h +++ b/src/include/logging/wal_log_manager.h @@ -26,24 +26,24 @@ class LogManager{ LogManager() : enable_logging_(false) {} ~LogManager() {} - inline bool init(std::string log_dir, std::string log_file) { + inline bool init() { - directory_ = log_dir; - log_file = log_file; + log_dir_ = settings::SettingsManager::GetString(settings::SettingId::log_directory_name); + log_file_ = settings::SettingsManager::GetString(settings::SettingId::log_file_name); - if (!peloton::FileUtil::Exists(log_dir)) { - boost::filesystem::path dir(log_dir.c_str()); + if (!peloton::FileUtil::Exists(log_dir_)) { + boost::filesystem::path dir(log_dir_.c_str()); try { boost::filesystem::create_directory(dir); } catch (boost::filesystem::filesystem_error &e) { - LOG_ERROR("Failed to create log directory %s. Reason %s", log_dir.c_str(), e.what()); + LOG_INFO("Failed to create log directory %s. Reason %s", log_dir_.c_str(), e.what()); return false; } } if(!logger_ofstream_.is_open()){ - logger_ofstream_.open(log_dir + "/" + log_file, std::ofstream::out | std::ofstream::app); + logger_ofstream_.open(log_dir_ + "/" + log_file_, std::ofstream::out | std::ofstream::app); if(!logger_ofstream_.fail()) { enable_logging_ = true; @@ -68,11 +68,11 @@ class LogManager{ } inline bool IsLoggingEnabled() { return enable_logging_; } - inline std::string GetDirectory() { return directory_; } - inline std::string GetLogFilePath() { return directory_ + log_file_; } + inline std::string GetDirectory() { return log_dir_; } + inline std::string GetLogFilePath() { return log_dir_ + "/" + log_file_; } inline void DoRecovery(){ - WalRecovery* wr = new WalRecovery(); + WalRecovery* wr = new WalRecovery(GetLogFilePath()); wr->StartRecovery(); delete wr; } @@ -85,7 +85,7 @@ class LogManager{ private: // NOTE: ofstream is not thread safe, might need to change it if more than one logger thread is used bool enable_logging_; - std::string directory_; + std::string log_dir_; std::string log_file_; std::ofstream logger_ofstream_; diff --git a/src/include/logging/wal_recovery.h b/src/include/logging/wal_recovery.h index 729458bedfc..dde14713564 100644 --- a/src/include/logging/wal_recovery.h +++ b/src/include/logging/wal_recovery.h @@ -61,7 +61,8 @@ class WalRecovery{ public: - WalRecovery() : + WalRecovery(std::string logpath) : + log_path_(logpath), max_epoch_id_(INVALID_EID), log_buffer_size_(0), log_buffer_(nullptr) {} @@ -87,9 +88,7 @@ class WalRecovery{ std::vector &columns); - - //TODO(graghura): don't hardcode the path - std::string logpath_ = "/tmp/log"; + std::string log_path_; std::fstream fstream_; eid_t max_epoch_id_; diff --git a/src/include/threadpool/logger_queue_pool.h b/src/include/threadpool/logger_queue_pool.h index a81ef92e2dc..14f5e84a6f8 100644 --- a/src/include/threadpool/logger_queue_pool.h +++ b/src/include/threadpool/logger_queue_pool.h @@ -42,7 +42,6 @@ class LoggerQueuePool { : logger_queue_(kDefaultLoggerQueueSize), num_workers_(num_workers), is_running_(false) { - LOG_INFO("aaron: LoggerQueuePool"); PELOTON_ASSERT(num_workers == 1); logger_queue_.GenerateTokens(log_tokens_, kDefaultNumTokens); } @@ -58,7 +57,6 @@ class LoggerQueuePool { void Startup() { - LOG_INFO("aaron: LoggerQueuePoolStartup"); is_running_ = true; for (size_t i = 0; i < num_workers_; i++) { @@ -67,7 +65,6 @@ class LoggerQueuePool { } void Shutdown() { - LOG_INFO("aaron: LoggerQueuePoolShutdown"); is_running_ = false; for (auto &logger : loggers_) { diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index e07139598cb..73d6a0bf055 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -20,7 +20,6 @@ void LogBuffer::WriteRecord(LogRecord &record) { size_t start = log_buffer_.Position(); log_buffer_.WriteInt(0); - LOG_INFO("aaron: WriteRecord %ld", start); LogRecordType type = record.GetType(); log_buffer_.WriteEnumInSingleByte( @@ -114,7 +113,6 @@ void LogBuffer::WriteRecord(LogRecord &record) { // XXX: We rely on the fact that the serializer treat a int32_t as 4 bytes int32_t length = log_buffer_.Position() - start - sizeof(int32_t); log_buffer_.WriteIntAt(start, length); - LOG_INFO("aaron: RecordLength %d", length); } } diff --git a/src/logging/wal_logger.cpp b/src/logging/wal_logger.cpp index 21188672b90..9a64a9cea0f 100644 --- a/src/logging/wal_logger.cpp +++ b/src/logging/wal_logger.cpp @@ -49,7 +49,6 @@ bool WalLogger::IsFlushNeeded(bool pending_buffers){ void WalLogger::FlushToDisk(){ if(disk_buffer_->GetSize()==0){ - LOG_INFO("aaron: disk_buffer_->GetSize()==0"); return; } @@ -60,7 +59,6 @@ void WalLogger::FlushToDisk(){ PELOTON_ASSERT(false); } - LOG_INFO("aaron: stream->flush()"); stream->flush(); if(stream->fail()){ diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index 7e415e984f3..43235cbabff 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -20,7 +20,7 @@ namespace logging{ void WalRecovery::StartRecovery(){ - if(!LoggingUtil::OpenFile(logpath_.c_str(), + if(!LoggingUtil::OpenFile(log_path_.c_str(), std::ifstream::in, fstream_)) { LOG_ERROR("Recovery failed"); @@ -543,7 +543,6 @@ void WalRecovery::ReplayAllTxns(){ void WalRecovery::ReplayLogFile(){ - size_t curr_txn_offset = 0; // Pass 1 From 6ec3e76c72dc0935c7592267d0e39861e27db78c Mon Sep 17 00:00:00 2001 From: Gandeevan Raghuraman Date: Fri, 4 May 2018 22:43:40 -0400 Subject: [PATCH 69/81] moved enable_logging flag to settings --- src/common/init.cpp | 3 ++- src/include/common/init.h | 2 +- src/include/settings/settings.h | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/common/init.cpp b/src/common/init.cpp index 19b1fdbdada..074e8ef7de6 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -31,7 +31,7 @@ namespace peloton { ThreadPool thread_pool; -void PelotonInit::Initialize(bool enable_logging) { +void PelotonInit::Initialize() { CONNECTION_THREAD_COUNT = settings::SettingsManager::GetInt( settings::SettingId::connection_thread_count); LOGGING_THREAD_COUNT = 1; @@ -92,6 +92,7 @@ void PelotonInit::Initialize(bool enable_logging) { // Initialize the Statement Cache Manager StatementCacheManager::Init(); + bool enable_logging = settings::SettingsManager::GetBool(settings::SettingId::enable_logging); if(enable_logging){ if(!logging::LogManager::GetInstance().init()){ LOG_ERROR("LogManager Initialization failed"); diff --git a/src/include/common/init.h b/src/include/common/init.h index 9e573891751..5aede8c715a 100644 --- a/src/include/common/init.h +++ b/src/include/common/init.h @@ -26,7 +26,7 @@ extern ThreadPool thread_pool; class PelotonInit { public: - static void Initialize(bool enable_logging = true); + static void Initialize(); static void Shutdown(); diff --git a/src/include/settings/settings.h b/src/include/settings/settings.h index a8bc5272beb..1c7eeb0ec91 100644 --- a/src/include/settings/settings.h +++ b/src/include/settings/settings.h @@ -141,6 +141,12 @@ SETTING_string(log_file_name, "wal.log", true, true) +// Enable logging. The default value is true for testing purposes +// TODO(graghura): logging should be turned off by default +SETTING_bool(enable_logging, + "Enable logging for DB persistence (default: true)", + true, + false, false) //===----------------------------------------------------------------------===// // ERROR REPORTING AND LOGGING From d53904012c0b0ce7423e4b58b7a124c2eff7839f Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Sat, 5 May 2018 03:28:19 -0400 Subject: [PATCH 70/81] Adapt to new APIs --- test/logging/logging_test.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/logging/logging_test.cpp b/test/logging/logging_test.cpp index b462f014ad9..76a6a0053a6 100644 --- a/test/logging/logging_test.cpp +++ b/test/logging/logging_test.cpp @@ -55,6 +55,7 @@ void *LoggingTest(int port) { txn2.exec("INSERT INTO employee VALUES (1, 'Aaron Tian');"); txn2.exec("INSERT INTO employee VALUES (2, 'Gandeevan Raghuraman');"); txn2.exec("INSERT INTO employee VALUES (3, 'Anirudh Kanjani');"); + txn2.commit(); } catch (const std::exception &e) { LOG_INFO("[LoggingTest] Exception occurred: %s", e.what()); @@ -64,7 +65,7 @@ void *LoggingTest(int port) { } TEST_F(LoggingTests, LoggingTest) { - peloton::PelotonInit::Initialize("/tmp", "test_log_file"); + peloton::PelotonInit::Initialize(); LOG_INFO("Server initialized"); peloton::network::PelotonServer server; From 52b46dd55b97711b16fad408ad99d71689a8f353 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Sat, 5 May 2018 03:48:55 -0400 Subject: [PATCH 71/81] Specify whether to turn on recovery mode at initilization --- src/common/init.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/init.cpp b/src/common/init.cpp index 074e8ef7de6..0bf2376bcd2 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -99,10 +99,10 @@ void PelotonInit::Initialize() { } } - logging::LogManager::GetInstance().DoRecovery(); - - - + bool enable_recovery = settings::SettingsManager::GetBool(settings::SettingId::enable_recovery); + if(enable_recovery){ + logging::LogManager::GetInstance().DoRecovery(); + } threadpool::LoggerQueuePool::GetInstance().Startup(); From 91f792076cfefc9170fd365a4c6d146ef3ea5560 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Sat, 5 May 2018 03:49:38 -0400 Subject: [PATCH 72/81] Add the recovery switch into settings --- src/include/settings/settings.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/include/settings/settings.h b/src/include/settings/settings.h index 1c7eeb0ec91..6b7f7d36e4b 100644 --- a/src/include/settings/settings.h +++ b/src/include/settings/settings.h @@ -148,6 +148,11 @@ SETTING_bool(enable_logging, true, false, false) +SETTING_bool(enable_recovery, + "Enable recovery for DB persistence (default: false)", + false, + false, false) + //===----------------------------------------------------------------------===// // ERROR REPORTING AND LOGGING //===----------------------------------------------------------------------===// From 2b752da2b248ecc86160ca1f4d4459cafa80ea9a Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Sat, 5 May 2018 03:50:01 -0400 Subject: [PATCH 73/81] Adjust the log level --- src/concurrency/timestamp_ordering_transaction_manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 61e9ad3cbd4..5caa32c4589 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -477,7 +477,7 @@ void TimestampOrderingTransactionManager::PerformInsert( if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { - LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); + LOG_TRACE("Submitting log buffer %p", current_txn->GetLogBuffer()); /* insert to the queue */ threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( current_txn->GetLogToken(), current_txn->GetLogBuffer()); From 1ba0f2338c10eb06457e2810c3e5fe5dc9b009ea Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Sat, 5 May 2018 04:18:08 -0400 Subject: [PATCH 74/81] Separate the insert log buffer test --- test/logging/log_buffer_test.cpp | 80 ++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 3 deletions(-) diff --git a/test/logging/log_buffer_test.cpp b/test/logging/log_buffer_test.cpp index 02c355373cc..c97cc5d0037 100644 --- a/test/logging/log_buffer_test.cpp +++ b/test/logging/log_buffer_test.cpp @@ -25,6 +25,80 @@ namespace test { class LogBufferTests : public PelotonTest {}; +TEST_F(LogBufferTests, InsertLogBufferTest) { + oid_t block = 1, offset = 16; + ItemPointer location(block, offset); + eid_t epoch_id = 3; + txn_id_t txn_id = 99; + cid_t commit_id = 98; + oid_t database_id = 10; + oid_t table_id = 20; + oid_t tile_group_id = 30; + double value1 = 9.1, value2 = 9.2, value3 = 9.3; + + logging::LogBuffer *log_buffer; + log_buffer = new logging::LogBuffer(logging::LogManager::GetInstance().GetTransactionBufferSize()); + +// TILES + std::vector tile_column_names; + std::vector> column_names; + + tile_column_names.push_back("INTEGER COL"); + column_names.push_back(tile_column_names); + + std::vector schemas; + std::vector columns; + +// SCHEMA + catalog::Column column1(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "A", true); + columns.push_back(column1); + + std::unique_ptr schema1(new catalog::Schema(columns)); + schemas.push_back(*schema1); + + std::map> column_map; + column_map[0] = std::make_pair(0, 0); + + std::shared_ptr tile_group(storage::TileGroupFactory::GetTileGroup( + database_id, table_id, tile_group_id, nullptr, schemas, column_map, 3)); + catalog::Manager::GetInstance().AddTileGroup(block, tile_group); + + logging::LogRecord insert_record = logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_INSERT, location, epoch_id, txn_id, commit_id); + + EXPECT_EQ(log_buffer->GetThreshold(), logging::LogManager::GetInstance().GetTransactionBufferSize()); + + type::Value val1 = type::ValueFactory::GetDecimalValue(value1); + type::Value val2 = type::ValueFactory::GetDecimalValue(value2); + type::Value val3 = type::ValueFactory::GetDecimalValue(value3); + std::vector values; + values.push_back(val1); + values.push_back(val2); + values.push_back(val3); + insert_record.SetValuesArray(reinterpret_cast(values.data()), static_cast(values.size())); + log_buffer->WriteRecord(insert_record); + size_t buffer_size1 = sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t) * 3 + sizeof(int64_t) * 4 + + sizeof(double) * 3; + EXPECT_EQ(buffer_size1, log_buffer->GetSize()); + + ReferenceSerializeInput log_buffer_input(log_buffer->GetData(), log_buffer->GetSize()); + + // Insert record + EXPECT_EQ(buffer_size1 - sizeof(int32_t), log_buffer_input.ReadInt()); + EXPECT_EQ(static_cast(LogRecordType::TUPLE_INSERT), log_buffer_input.ReadEnumInSingleByte()); + EXPECT_EQ(txn_id, log_buffer_input.ReadLong()); + EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); + EXPECT_EQ(commit_id, log_buffer_input.ReadLong()); + EXPECT_EQ(database_id, log_buffer_input.ReadLong()); + EXPECT_EQ(table_id, log_buffer_input.ReadLong()); + EXPECT_EQ(block, log_buffer_input.ReadLong()); + EXPECT_EQ(offset, log_buffer_input.ReadLong()); + EXPECT_EQ(value1, log_buffer_input.ReadDouble()); + EXPECT_EQ(value2, log_buffer_input.ReadDouble()); + EXPECT_EQ(value3, log_buffer_input.ReadDouble()); +} + TEST_F(LogBufferTests, LogBufferTest) { oid_t block = 1, offset = 16, old_block = 1, old_offset = 8, new_block = 1, new_offset = 24; @@ -115,8 +189,8 @@ TEST_F(LogBufferTests, LogBufferTest) { // Insert record EXPECT_EQ(buffer_size1 - sizeof(int32_t), log_buffer_input.ReadInt()); EXPECT_EQ(static_cast(LogRecordType::TUPLE_INSERT), log_buffer_input.ReadEnumInSingleByte()); - EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); EXPECT_EQ(txn_id, log_buffer_input.ReadLong()); + EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); EXPECT_EQ(commit_id, log_buffer_input.ReadLong()); EXPECT_EQ(database_id, log_buffer_input.ReadLong()); EXPECT_EQ(table_id, log_buffer_input.ReadLong()); @@ -129,8 +203,8 @@ TEST_F(LogBufferTests, LogBufferTest) { // Update Record EXPECT_EQ(buffer_size2 - sizeof(int32_t), log_buffer_input.ReadInt()); EXPECT_EQ(static_cast(LogRecordType::TUPLE_UPDATE), log_buffer_input.ReadEnumInSingleByte()); - EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); EXPECT_EQ(txn_id, log_buffer_input.ReadLong()); + EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); EXPECT_EQ(commit_id, log_buffer_input.ReadLong()); EXPECT_EQ(database_id, log_buffer_input.ReadLong()); EXPECT_EQ(table_id, log_buffer_input.ReadLong()); @@ -149,8 +223,8 @@ TEST_F(LogBufferTests, LogBufferTest) { // Delete record EXPECT_EQ(buffer_size3 - sizeof(int32_t), log_buffer_input.ReadInt()); EXPECT_EQ(static_cast(LogRecordType::TUPLE_DELETE), log_buffer_input.ReadEnumInSingleByte()); - EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); EXPECT_EQ(txn_id, log_buffer_input.ReadLong()); + EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); EXPECT_EQ(commit_id, log_buffer_input.ReadLong()); EXPECT_EQ(database_id, log_buffer_input.ReadLong()); EXPECT_EQ(table_id, log_buffer_input.ReadLong()); From 7e38591daf0d8d2222d57166957031d84b09bfe4 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Sat, 5 May 2018 04:32:22 -0400 Subject: [PATCH 75/81] Separate the update log buffer test --- test/logging/log_buffer_test.cpp | 101 ++++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 3 deletions(-) diff --git a/test/logging/log_buffer_test.cpp b/test/logging/log_buffer_test.cpp index c97cc5d0037..cf64ec395a6 100644 --- a/test/logging/log_buffer_test.cpp +++ b/test/logging/log_buffer_test.cpp @@ -78,14 +78,14 @@ TEST_F(LogBufferTests, InsertLogBufferTest) { values.push_back(val3); insert_record.SetValuesArray(reinterpret_cast(values.data()), static_cast(values.size())); log_buffer->WriteRecord(insert_record); - size_t buffer_size1 = sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t) * 3 + sizeof(int64_t) * 4 + + size_t buffer_size = sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t) * 3 + sizeof(int64_t) * 4 + sizeof(double) * 3; - EXPECT_EQ(buffer_size1, log_buffer->GetSize()); + EXPECT_EQ(buffer_size, log_buffer->GetSize()); ReferenceSerializeInput log_buffer_input(log_buffer->GetData(), log_buffer->GetSize()); // Insert record - EXPECT_EQ(buffer_size1 - sizeof(int32_t), log_buffer_input.ReadInt()); + EXPECT_EQ(buffer_size - sizeof(int32_t), log_buffer_input.ReadInt()); EXPECT_EQ(static_cast(LogRecordType::TUPLE_INSERT), log_buffer_input.ReadEnumInSingleByte()); EXPECT_EQ(txn_id, log_buffer_input.ReadLong()); EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); @@ -99,6 +99,101 @@ TEST_F(LogBufferTests, InsertLogBufferTest) { EXPECT_EQ(value3, log_buffer_input.ReadDouble()); } +TEST_F(LogBufferTests, UpdateLogBufferTest) { + oid_t block = 1, offset = 16, new_block = 1, new_offset = 24; + ItemPointer location(block, offset); + ItemPointer new_location(new_block, new_offset); + eid_t epoch_id = 3; + txn_id_t txn_id = 99; + cid_t commit_id = 98; + oid_t database_id = 10; + oid_t table_id = 20; + oid_t tile_group_id = 30; + double value1 = 9.1, value2 = 9.2, value3 = 9.3; + oid_t target_oid1 = 2, target_oid2 = 4, target_oid3 = 5; + + logging::LogBuffer *log_buffer; + log_buffer = new logging::LogBuffer(logging::LogManager::GetInstance().GetTransactionBufferSize()); + +// TILES + std::vector tile_column_names; + std::vector> column_names; + + tile_column_names.push_back("INTEGER COL"); + column_names.push_back(tile_column_names); + + std::vector schemas; + std::vector columns; + +// SCHEMA + catalog::Column column1(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "A", true); + columns.push_back(column1); + + std::unique_ptr schema1(new catalog::Schema(columns)); + schemas.push_back(*schema1); + + std::map> column_map; + column_map[0] = std::make_pair(0, 0); + + std::shared_ptr tile_group(storage::TileGroupFactory::GetTileGroup( + database_id, table_id, tile_group_id, nullptr, schemas, column_map, 3)); + catalog::Manager::GetInstance().AddTileGroup(block, tile_group); + + logging::LogRecord insert_record = logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_INSERT, location, epoch_id, txn_id, commit_id); + logging::LogRecord update_record = logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_UPDATE, location, new_location, epoch_id, txn_id, + commit_id); + + EXPECT_EQ(log_buffer->GetThreshold(), logging::LogManager::GetInstance().GetTransactionBufferSize()); + + type::Value val1 = type::ValueFactory::GetDecimalValue(value1); + type::Value val2 = type::ValueFactory::GetDecimalValue(value2); + type::Value val3 = type::ValueFactory::GetDecimalValue(value3); + std::vector values; + values.push_back(val1); + values.push_back(val2); + values.push_back(val3); + expression::AbstractExpression *expr = nullptr; + planner::DerivedAttribute tmp_att(expr); + Target target1 = std::make_pair(target_oid1, tmp_att); + Target target2 = std::make_pair(target_oid2, tmp_att); + Target target3 = std::make_pair(target_oid3, tmp_att); + TargetList *target_list = new TargetList; + target_list->push_back(target1); + target_list->push_back(target2); + target_list->push_back(target3); + update_record.SetOffsetsArray(target_list); + update_record.SetValuesArray(reinterpret_cast(values.data()), static_cast(values.size())); + log_buffer->WriteRecord(update_record); + size_t buffer_size = sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t) * 3 + sizeof(int64_t) * 7 + + (sizeof(int32_t) + sizeof(double)) * 3; + EXPECT_EQ(buffer_size, log_buffer->GetSize()); + + ReferenceSerializeInput log_buffer_input(log_buffer->GetData(), log_buffer->GetSize()); + + // Update Record + EXPECT_EQ(buffer_size - sizeof(int32_t), log_buffer_input.ReadInt()); + EXPECT_EQ(static_cast(LogRecordType::TUPLE_UPDATE), log_buffer_input.ReadEnumInSingleByte()); + EXPECT_EQ(txn_id, log_buffer_input.ReadLong()); + EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); + EXPECT_EQ(commit_id, log_buffer_input.ReadLong()); + EXPECT_EQ(database_id, log_buffer_input.ReadLong()); + EXPECT_EQ(table_id, log_buffer_input.ReadLong()); + EXPECT_EQ(block, log_buffer_input.ReadLong()); + EXPECT_EQ(offset, log_buffer_input.ReadLong()); + EXPECT_EQ(new_block, log_buffer_input.ReadLong()); + EXPECT_EQ(new_offset, log_buffer_input.ReadLong()); + EXPECT_EQ(update_record.GetNumValues(), log_buffer_input.ReadLong()); + EXPECT_EQ(target_oid1, log_buffer_input.ReadInt()); + EXPECT_EQ(value1, log_buffer_input.ReadDouble()); + EXPECT_EQ(target_oid2, log_buffer_input.ReadInt()); + EXPECT_EQ(value2, log_buffer_input.ReadDouble()); + EXPECT_EQ(target_oid3, log_buffer_input.ReadInt()); + EXPECT_EQ(value3, log_buffer_input.ReadDouble()); +} + TEST_F(LogBufferTests, LogBufferTest) { oid_t block = 1, offset = 16, old_block = 1, old_offset = 8, new_block = 1, new_offset = 24; From 3f63eb5873e988eade06b0a2e0ea06aa6e362a28 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Sat, 5 May 2018 04:37:47 -0400 Subject: [PATCH 76/81] Separate the delete log buffer test --- test/logging/log_buffer_test.cpp | 84 ++------------------------------ 1 file changed, 5 insertions(+), 79 deletions(-) diff --git a/test/logging/log_buffer_test.cpp b/test/logging/log_buffer_test.cpp index cf64ec395a6..0546b88b8f7 100644 --- a/test/logging/log_buffer_test.cpp +++ b/test/logging/log_buffer_test.cpp @@ -140,8 +140,6 @@ TEST_F(LogBufferTests, UpdateLogBufferTest) { database_id, table_id, tile_group_id, nullptr, schemas, column_map, 3)); catalog::Manager::GetInstance().AddTileGroup(block, tile_group); - logging::LogRecord insert_record = logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_INSERT, location, epoch_id, txn_id, commit_id); logging::LogRecord update_record = logging::LogRecordFactory::CreateTupleRecord( LogRecordType::TUPLE_UPDATE, location, new_location, epoch_id, txn_id, commit_id); @@ -194,20 +192,15 @@ TEST_F(LogBufferTests, UpdateLogBufferTest) { EXPECT_EQ(value3, log_buffer_input.ReadDouble()); } -TEST_F(LogBufferTests, LogBufferTest) { - oid_t block = 1, offset = 16, old_block = 1, old_offset = 8, new_block = 1, - new_offset = 24; +TEST_F(LogBufferTests, DeleteLogBufferTest) { + oid_t block = 1, old_block = 1, old_offset = 8; ItemPointer old_location(old_block, old_offset); - ItemPointer location(block, offset); - ItemPointer new_location(new_block, new_offset); eid_t epoch_id = 3; txn_id_t txn_id = 99; cid_t commit_id = 98; oid_t database_id = 10; oid_t table_id = 20; oid_t tile_group_id = 30; - double value1 = 9.1, value2 = 9.2, value3 = 9.3; - oid_t target_oid1 = 2, target_oid2 = 4, target_oid3 = 5; logging::LogBuffer *log_buffer; log_buffer = new logging::LogBuffer(logging::LogManager::GetInstance().GetTransactionBufferSize()); @@ -237,86 +230,19 @@ TEST_F(LogBufferTests, LogBufferTest) { database_id, table_id, tile_group_id, nullptr, schemas, column_map, 3)); catalog::Manager::GetInstance().AddTileGroup(block, tile_group); - logging::LogRecord insert_record = logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_INSERT, location, epoch_id, txn_id, commit_id); - logging::LogRecord update_record = logging::LogRecordFactory::CreateTupleRecord( - LogRecordType::TUPLE_UPDATE, location, new_location, epoch_id, txn_id, - commit_id); logging::LogRecord delete_record = logging::LogRecordFactory::CreateTupleRecord( LogRecordType::TUPLE_DELETE, old_location, epoch_id, txn_id, commit_id); EXPECT_EQ(log_buffer->GetThreshold(), logging::LogManager::GetInstance().GetTransactionBufferSize()); - type::Value val1 = type::ValueFactory::GetDecimalValue(value1); - type::Value val2 = type::ValueFactory::GetDecimalValue(value2); - type::Value val3 = type::ValueFactory::GetDecimalValue(value3); - std::vector values; - values.push_back(val1); - values.push_back(val2); - values.push_back(val3); - insert_record.SetValuesArray(reinterpret_cast(values.data()), static_cast(values.size())); - log_buffer->WriteRecord(insert_record); - size_t buffer_size1 = sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t) * 3 + sizeof(int64_t) * 4 + - sizeof(double) * 3; - EXPECT_EQ(buffer_size1, log_buffer->GetSize()); - - expression::AbstractExpression *expr = nullptr; - planner::DerivedAttribute tmp_att(expr); - Target target1 = std::make_pair(target_oid1, tmp_att); - Target target2 = std::make_pair(target_oid2, tmp_att); - Target target3 = std::make_pair(target_oid3, tmp_att); - TargetList *target_list = new TargetList; - target_list->push_back(target1); - target_list->push_back(target2); - target_list->push_back(target3); - update_record.SetOffsetsArray(target_list); - update_record.SetValuesArray(reinterpret_cast(values.data()), static_cast(values.size())); - log_buffer->WriteRecord(update_record); - size_t buffer_size2 = sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t) * 3 + sizeof(int64_t) * 7 + - (sizeof(int32_t) + sizeof(double)) * 3; - EXPECT_EQ(buffer_size2, log_buffer->GetSize() - buffer_size1); - log_buffer->WriteRecord(delete_record); - size_t buffer_size3 = sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t) * 3 + sizeof(int64_t) * 4; - EXPECT_EQ(buffer_size3, log_buffer->GetSize() - buffer_size1 - buffer_size2); + size_t buffer_size = sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t) * 3 + sizeof(int64_t) * 4; + EXPECT_EQ(buffer_size, log_buffer->GetSize()); ReferenceSerializeInput log_buffer_input(log_buffer->GetData(), log_buffer->GetSize()); - // Insert record - EXPECT_EQ(buffer_size1 - sizeof(int32_t), log_buffer_input.ReadInt()); - EXPECT_EQ(static_cast(LogRecordType::TUPLE_INSERT), log_buffer_input.ReadEnumInSingleByte()); - EXPECT_EQ(txn_id, log_buffer_input.ReadLong()); - EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); - EXPECT_EQ(commit_id, log_buffer_input.ReadLong()); - EXPECT_EQ(database_id, log_buffer_input.ReadLong()); - EXPECT_EQ(table_id, log_buffer_input.ReadLong()); - EXPECT_EQ(block, log_buffer_input.ReadLong()); - EXPECT_EQ(offset, log_buffer_input.ReadLong()); - EXPECT_EQ(value1, log_buffer_input.ReadDouble()); - EXPECT_EQ(value2, log_buffer_input.ReadDouble()); - EXPECT_EQ(value3, log_buffer_input.ReadDouble()); - - // Update Record - EXPECT_EQ(buffer_size2 - sizeof(int32_t), log_buffer_input.ReadInt()); - EXPECT_EQ(static_cast(LogRecordType::TUPLE_UPDATE), log_buffer_input.ReadEnumInSingleByte()); - EXPECT_EQ(txn_id, log_buffer_input.ReadLong()); - EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); - EXPECT_EQ(commit_id, log_buffer_input.ReadLong()); - EXPECT_EQ(database_id, log_buffer_input.ReadLong()); - EXPECT_EQ(table_id, log_buffer_input.ReadLong()); - EXPECT_EQ(block, log_buffer_input.ReadLong()); - EXPECT_EQ(offset, log_buffer_input.ReadLong()); - EXPECT_EQ(new_block, log_buffer_input.ReadLong()); - EXPECT_EQ(new_offset, log_buffer_input.ReadLong()); - EXPECT_EQ(update_record.GetNumValues(), log_buffer_input.ReadLong()); - EXPECT_EQ(target_oid1, log_buffer_input.ReadInt()); - EXPECT_EQ(value1, log_buffer_input.ReadDouble()); - EXPECT_EQ(target_oid2, log_buffer_input.ReadInt()); - EXPECT_EQ(value2, log_buffer_input.ReadDouble()); - EXPECT_EQ(target_oid3, log_buffer_input.ReadInt()); - EXPECT_EQ(value3, log_buffer_input.ReadDouble()); // Delete record - EXPECT_EQ(buffer_size3 - sizeof(int32_t), log_buffer_input.ReadInt()); + EXPECT_EQ(buffer_size - sizeof(int32_t), log_buffer_input.ReadInt()); EXPECT_EQ(static_cast(LogRecordType::TUPLE_DELETE), log_buffer_input.ReadEnumInSingleByte()); EXPECT_EQ(txn_id, log_buffer_input.ReadLong()); EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); From 4627b6a2b5ba7ea571ad3b3fb6a5320eb429f7ac Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Sat, 5 May 2018 04:50:01 -0400 Subject: [PATCH 77/81] Add the codebase for recovery tests --- test/logging/recovery_test.cpp | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 test/logging/recovery_test.cpp diff --git a/test/logging/recovery_test.cpp b/test/logging/recovery_test.cpp new file mode 100644 index 00000000000..d3f939dbd92 --- /dev/null +++ b/test/logging/recovery_test.cpp @@ -0,0 +1,35 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// recovery_test.cpp +// +// Identification: test/logging/recovery_test.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include /* libpqxx is used to instantiate C++ client */ +#include "logging/log_buffer.h" +#include "common/harness.h" +#include "gtest/gtest.h" +#include "network/peloton_server.h" +#include "network/postgres_protocol_handler.h" +#include "util/string_util.h" +#include "network/connection_handle_factory.h" + +namespace peloton { + namespace test { + +//===--------------------------------------------------------------------===// +// Log Recovery Tests +//===--------------------------------------------------------------------===// + +class RecoveryTests : public PelotonTest {}; + +TEST_F(RecoveryTests, InsertRecoveryTest) { + +} +} +} \ No newline at end of file From c6564d9ab6a50b276d50407d2e059510e57ae889 Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Sat, 5 May 2018 22:03:20 -0400 Subject: [PATCH 78/81] Change catalog singletons in recovery to instances --- src/logging/wal_recovery.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index 3cc11be7adf..72d174355e8 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -1,10 +1,12 @@ +#include #include "index/index_factory.h" #include "catalog/index_catalog.h" #include "catalog/column_catalog.h" #include "catalog/table_catalog.h" #include "catalog/database_catalog.h" +#include "catalog/system_catalogs.h" #include "concurrency/transaction_manager_factory.h" #include "catalog/catalog_defaults.h" #include "type/ephemeral_pool.h" @@ -322,7 +324,7 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ InstallCatalogTuple(record_type, tuple.get(), table, commit_id, location); // update table oid to prevent oid reuse after recovery - catalog::TableCatalog::GetInstance()->GetNextOid(); // TODO(graghura): This is a hack, rewrite !!! + catalog::Catalog::GetInstance()->GetSystemCatalogs(CATALOG_DATABASE_OID)->GetTableCatalog()->GetNextOid(); tuple_table_create = std::move(tuple); pending_table_create = true; @@ -334,7 +336,7 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ InstallCatalogTuple(record_type, tuple.get(), table, commit_id, location); // update column oid to prevent oid reuse after recovery - catalog::ColumnCatalog::GetInstance()->GetNextOid(); // TODO(graghura): This is a hack, rewrite !!! + catalog::Catalog::GetInstance()->GetSystemCatalogs(CATALOG_DATABASE_OID)->GetColumnCatalog()->GetNextOid(); bool is_inline = true; @@ -361,7 +363,7 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ } else if (table_id == INDEX_CATALOG_OID) { LOG_INFO("REPLAYING INSERT TO PG_INDEX"); indexes.push_back(std::move(tuple)); - catalog::IndexCatalog::GetInstance()->GetNextOid(); // TODO(graghura): This is a hack, rewrite !!! + catalog::Catalog::GetInstance()->GetSystemCatalogs(CATALOG_DATABASE_OID)->GetIndexCatalog()->GetNextOid(); } } else { @@ -469,7 +471,8 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ auto txn = concurrency::TransactionManagerFactory::GetInstance().BeginTransaction( IsolationLevelType::SERIALIZABLE); - auto table_catalog = catalog::TableCatalog::GetInstance(); + auto table_catalog = catalog::Catalog::GetInstance()-> + GetSystemCatalogs(CATALOG_DATABASE_OID)->GetTableCatalog(); auto table_object = table_catalog->GetTableObject(tup->GetValue(2).GetAs(), txn); auto database_oid = table_object->GetDatabaseOid(); auto table = storage::StorageManager::GetInstance()->GetTableWithOid( From 5a73bcf4948c36bd3addb90b8748ee264c07d0fd Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Sun, 6 May 2018 18:12:45 -0400 Subject: [PATCH 79/81] Change logging & header files --- src/logging/log_util.cpp | 22 +++++++++++----------- src/logging/wal_recovery.cpp | 13 ++++++++++++- src/threadpool/logger_queue_pool.cpp | 16 ++++++++++++---- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/logging/log_util.cpp b/src/logging/log_util.cpp index cc4234dd452..16ffe927d38 100644 --- a/src/logging/log_util.cpp +++ b/src/logging/log_util.cpp @@ -1,14 +1,14 @@ -/*------------------------------------------------------------------------- - * - * logger.h - * file description - * - * Copyright(c) 2015, CMU - * - * /peloton/src/backend/logging/logging_util.cpp - * - *------------------------------------------------------------------------- - */ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// log_til.cpp +// +// Identification: src/logging/log_til.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// #include #include diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index 72d174355e8..a9b00b5b06f 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -1,7 +1,18 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// wal_recovery.cpp +// +// Identification: src/logging/wal_recovery.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// -#include #include "index/index_factory.h" +#include "catalog/catalog.h" #include "catalog/index_catalog.h" #include "catalog/column_catalog.h" #include "catalog/table_catalog.h" diff --git a/src/threadpool/logger_queue_pool.cpp b/src/threadpool/logger_queue_pool.cpp index 1003dd59ca6..a8a4cc67dd6 100644 --- a/src/threadpool/logger_queue_pool.cpp +++ b/src/threadpool/logger_queue_pool.cpp @@ -1,16 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// logger_queue_pool.cpp +// +// Identification: src/threadpool/logger_queue_pool.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + #include "threadpool/logger_queue_pool.h" #include "common/container/lock_free_queue.h" #include "logging/wal_logger.h" #include - namespace peloton{ namespace threadpool{ - - void LoggerFunc(std::atomic_bool *is_running, LoggerQueue *logger_queue) { - LOG_INFO("aaron: LoggerFunc"); constexpr auto kMinPauseTime = std::chrono::microseconds(1); constexpr auto kMaxPauseTime = std::chrono::microseconds(1000); auto pause_time = kMinPauseTime; From 0b6886511240997276b089abc078576ad8bcc61e Mon Sep 17 00:00:00 2001 From: Anirudh Kanjani Date: Tue, 8 May 2018 22:09:51 -0400 Subject: [PATCH 80/81] Incorporating schema/namespace changes with logging/recovery --- ...timestamp_ordering_transaction_manager.cpp | 30 +++++++++++++++---- src/include/logging/log_record.h | 25 +++++++++++----- src/logging/log_buffer.cpp | 4 ++- src/logging/wal_recovery.cpp | 13 ++++---- 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index d2f7c074dfc..fcba33f4cf0 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -12,7 +12,10 @@ #include "concurrency/timestamp_ordering_transaction_manager.h" #include - +#include "catalog/table_catalog.h" +#include "catalog/schema_catalog.h" +#include "catalog/system_catalogs.h" +#include "catalog/catalog.h" #include "catalog/catalog_defaults.h" #include "catalog/manager.h" #include "common/exception.h" @@ -470,10 +473,13 @@ void TimestampOrderingTransactionManager::PerformInsert( if(logging::LogManager::GetInstance().IsLoggingEnabled()) { if (values_buf != nullptr) { + auto tile_group = tile_group_header->GetTileGroup(); + auto table_object = catalog::Catalog::GetInstance()->GetTableObject(tile_group->GetDatabaseId(), tile_group->GetTableId(), current_txn); + auto schema_oid = (catalog::Catalog::GetInstance()->GetSystemCatalogs(tile_group->GetDatabaseId())->GetSchemaCatalog()->GetSchemaObject(table_object->GetSchemaName(), current_txn))->GetSchemaOid(); logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( LogRecordType::TUPLE_INSERT, location, current_txn->GetEpochId(), - current_txn->GetTransactionId(), current_txn->GetCommitId()); + current_txn->GetTransactionId(), current_txn->GetCommitId(), schema_oid); record.SetValuesArray(values_buf, values_size); current_txn->GetLogBuffer()->WriteRecord(record); @@ -579,10 +585,13 @@ void TimestampOrderingTransactionManager::PerformUpdate( if(logging::LogManager::GetInstance().IsLoggingEnabled()) { if (values_buf != nullptr) { + auto tile_group = tile_group_header->GetTileGroup(); + auto table_object = catalog::Catalog::GetInstance()->GetTableObject(tile_group->GetDatabaseId(), tile_group->GetTableId(), current_txn); + auto schema_oid = (catalog::Catalog::GetInstance()->GetSystemCatalogs(tile_group->GetDatabaseId())->GetSchemaCatalog()->GetSchemaObject(table_object->GetSchemaName(), current_txn))->GetSchemaOid(); logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( LogRecordType::TUPLE_UPDATE, location, new_location, current_txn->GetEpochId(), - current_txn->GetTransactionId(), current_txn->GetCommitId()); + current_txn->GetTransactionId(), current_txn->GetCommitId(), schema_oid); record.SetOldItemPointer(location); record.SetValuesArray(values_buf, values_size); @@ -633,11 +642,14 @@ void TimestampOrderingTransactionManager::PerformUpdate( if(logging::LogManager::GetInstance().IsLoggingEnabled()) { if (values_buf != nullptr) { + auto tile_group = tile_group_header->GetTileGroup(); + auto table_object = catalog::Catalog::GetInstance()->GetTableObject(tile_group->GetDatabaseId(), tile_group->GetTableId(), current_txn); + auto schema_oid = (catalog::Catalog::GetInstance()->GetSystemCatalogs(tile_group->GetDatabaseId())->GetSchemaCatalog()->GetSchemaObject(table_object->GetSchemaName(), current_txn))->GetSchemaOid(); logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( LogRecordType::TUPLE_UPDATE, location, location, current_txn->GetEpochId(), current_txn->GetTransactionId(), - current_txn->GetCommitId()); + current_txn->GetCommitId(), schema_oid); record.SetOldItemPointer(location); record.SetValuesArray(values_buf, values_size); @@ -756,10 +768,13 @@ void TimestampOrderingTransactionManager::PerformDelete( if(logging::LogManager::GetInstance().IsLoggingEnabled()) { + auto tile_group = tile_group_header->GetTileGroup(); + auto table_object = catalog::Catalog::GetInstance()->GetTableObject(tile_group->GetDatabaseId(), tile_group->GetTableId(), current_txn); + auto schema_oid = (catalog::Catalog::GetInstance()->GetSystemCatalogs(tile_group->GetDatabaseId())->GetSchemaCatalog()->GetSchemaObject(table_object->GetSchemaName(), current_txn))->GetSchemaOid(); logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( LogRecordType::TUPLE_DELETE, old_location, current_txn->GetEpochId(), - current_txn->GetTransactionId(), current_txn->GetCommitId()); + current_txn->GetTransactionId(), current_txn->GetCommitId(), schema_oid); current_txn->GetLogBuffer()->WriteRecord(record); @@ -815,10 +830,13 @@ void TimestampOrderingTransactionManager::PerformDelete( } if(logging::LogManager::GetInstance().IsLoggingEnabled()) { + auto tile_group = tile_group_header->GetTileGroup(); + auto table_object = catalog::Catalog::GetInstance()->GetTableObject(tile_group->GetDatabaseId(), tile_group->GetTableId(), current_txn); + auto schema_oid = (catalog::Catalog::GetInstance()->GetSystemCatalogs(tile_group->GetDatabaseId())->GetSchemaCatalog()->GetSchemaObject(table_object->GetSchemaName(), current_txn))->GetSchemaOid(); logging::LogRecord record = logging::LogRecordFactory::CreateTupleRecord( LogRecordType::TUPLE_DELETE, location, current_txn->GetEpochId(), - current_txn->GetTransactionId(), current_txn->GetCommitId()); + current_txn->GetTransactionId(), current_txn->GetCommitId(), schema_oid); current_txn->GetLogBuffer()->WriteRecord(record); diff --git a/src/include/logging/log_record.h b/src/include/logging/log_record.h index 6b48a0e2127..9a4b181b66e 100644 --- a/src/include/logging/log_record.h +++ b/src/include/logging/log_record.h @@ -27,13 +27,14 @@ class LogRecord { friend class LogRecordFactory; private: LogRecord(LogRecordType log_type, const ItemPointer &old_pos, const ItemPointer &pos, - const eid_t epoch_id, const txn_id_t txn_id, const cid_t commit_id) + const eid_t epoch_id, const txn_id_t txn_id, const cid_t commit_id, oid_t schema_id) : log_record_type_(log_type), old_tuple_pos_(old_pos), tuple_pos_(pos), eid_(epoch_id), txn_id_(txn_id), - cid_(commit_id) {} + cid_(commit_id), + schema_id_(schema_id) {} public: virtual ~LogRecord() {} @@ -59,6 +60,10 @@ class LogRecord { offsets_ = arr; } + inline void SetSchemaId(oid_t schema_id) { + schema_id_ = schema_id; + } + inline const ItemPointer &GetItemPointer() { return tuple_pos_; } inline const ItemPointer &GetOldItemPointer() { return old_tuple_pos_; } @@ -75,6 +80,8 @@ class LogRecord { inline txn_id_t GetTransactionId() { return txn_id_; } + inline oid_t GetSchemaId() { return schema_id_; } + private: LogRecordType log_record_type_; @@ -93,6 +100,8 @@ class LogRecord { TargetList *offsets_; uint32_t num_values_; + + oid_t schema_id_; }; @@ -101,11 +110,11 @@ class LogRecordFactory { static LogRecord CreateTupleRecord(const LogRecordType log_type, const ItemPointer &pos, eid_t current_eid, - txn_id_t txn_id, cid_t current_cid) { + txn_id_t txn_id, cid_t current_cid, oid_t schema_oid) { PELOTON_ASSERT(log_type == LogRecordType::TUPLE_INSERT || log_type == LogRecordType::TUPLE_DELETE); - return LogRecord(log_type, INVALID_ITEMPOINTER, pos, current_eid, txn_id, current_cid); + return LogRecord(log_type, INVALID_ITEMPOINTER, pos, current_eid, txn_id, current_cid, schema_oid); } static LogRecord CreateTupleRecord(const LogRecordType log_type, eid_t current_eid, @@ -116,19 +125,19 @@ class LogRecordFactory { log_type == LogRecordType::TRANSACTION_BEGIN); return LogRecord(log_type, INVALID_ITEMPOINTER, INVALID_ITEMPOINTER, - current_eid, txn_id, current_cid); + current_eid, txn_id, current_cid, INVALID_OID); } static LogRecord CreateTupleRecord(const LogRecordType log_type, const ItemPointer &old_pos, const ItemPointer &pos, eid_t current_eid, - txn_id_t txn_id, cid_t current_cid) { + txn_id_t txn_id, cid_t current_cid, oid_t schema_oid) { PELOTON_ASSERT(log_type == LogRecordType::TUPLE_UPDATE); - return LogRecord(log_type, old_pos, pos, current_eid, txn_id, current_cid); + return LogRecord(log_type, old_pos, pos, current_eid, txn_id, current_cid, schema_oid); } }; } // namespace logging -} // namespace peloton \ No newline at end of file +} // namespace peloton diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index 1596e944181..d109f0695f6 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -29,7 +29,6 @@ void LogBuffer::WriteRecord(LogRecord &record) { log_buffer_.WriteLong(record.GetEpochId()); log_buffer_.WriteLong(record.GetCommitId()); - switch (type) { case LogRecordType::TUPLE_INSERT: { auto &manager = catalog::Manager::GetInstance(); @@ -39,6 +38,7 @@ void LogBuffer::WriteRecord(LogRecord &record) { // Write down the database id and the table id log_buffer_.WriteLong(tg->GetDatabaseId()); + log_buffer_.WriteLong(record.GetSchemaId()); log_buffer_.WriteLong(tg->GetTableId()); log_buffer_.WriteLong(tuple_pos.block); @@ -58,6 +58,7 @@ void LogBuffer::WriteRecord(LogRecord &record) { // Write down the database id and the table id log_buffer_.WriteLong(tg->GetDatabaseId()); + log_buffer_.WriteLong(record.GetSchemaId()); log_buffer_.WriteLong(tg->GetTableId()); log_buffer_.WriteLong(tuple_pos.block); @@ -73,6 +74,7 @@ void LogBuffer::WriteRecord(LogRecord &record) { // Write down the database id and the table id log_buffer_.WriteLong(tg->GetDatabaseId()); + log_buffer_.WriteLong(record.GetSchemaId()); log_buffer_.WriteLong(tg->GetTableId()); log_buffer_.WriteLong(old_tuple_pos.block); diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index a9b00b5b06f..ba297a8ef4e 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -212,10 +212,11 @@ bool WalRecovery::InstallCatalogTuple(LogRecordType record_type, storage::Tuple void WalRecovery::CreateTableOnRecovery(std::unique_ptr &tuple, std::vector &columns){ + // NOTE: The third column in pg_table stores the database oid. This value should always be corresponding to the location of the database oid in pg_table // Getting database auto database = storage::StorageManager::GetInstance()->GetDatabaseWithOid( - tuple->GetValue(2).GetAs()); + tuple->GetValue(3).GetAs()); // register table with catalog database->AddTable(new storage::DataTable( @@ -270,9 +271,10 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ if(record_type==LogRecordType::TUPLE_INSERT) { oid_t database_id = (oid_t) record_decode.ReadLong(); + oid_t schema_id = (oid_t) record_decode.ReadLong(); oid_t table_id = (oid_t) record_decode.ReadLong(); - if(database_id != CATALOG_DATABASE_OID || table_id != COLUMN_CATALOG_OID){ + if(schema_id != CATALOG_SCHEMA_OID || table_id != COLUMN_CATALOG_OID){ if(pending_table_create){ CreateTableOnRecovery(tuple_table_create, columns); pending_table_create = false; @@ -301,8 +303,7 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ record_decode, schema->GetColumn(oid).GetType()); tuple->SetValue(oid, val, pool.get()); } - - if (database_id == CATALOG_DATABASE_OID) { // catalog database oid + if (schema_id == CATALOG_SCHEMA_OID) { // catalog schema oid if (table_id == DATABASE_CATALOG_OID) { @@ -397,6 +398,7 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ LOG_INFO("Tuple Delete"); oid_t database_id = (oid_t)record_decode.ReadLong(); + oid_t schema_id = (oid_t) record_decode.ReadLong(); oid_t table_id = (oid_t)record_decode.ReadLong(); oid_t tg_block = (oid_t)record_decode.ReadLong(); @@ -407,7 +409,7 @@ void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ auto tg = table->GetTileGroupById(tg_block); // This code might be useful on drop - if (database_id == CATALOG_DATABASE_OID) { // catalog database oid + if (schema_id == CATALOG_SCHEMA_OID) { // catalog schema oid switch (table_id) { case DATABASE_CATALOG_OID:{ //pg_database @@ -579,7 +581,6 @@ void WalRecovery::ReplayLogFile(){ ParseFromDisk(ReplayStage::PASS_2); ReplayAllTxns(); - } From 2793bfafdaf1b0719597a3353f5f9eeb2488e21d Mon Sep 17 00:00:00 2001 From: Aaron Tian Date: Wed, 20 Jun 2018 16:10:23 -0400 Subject: [PATCH 81/81] Add logging and recovery tests --- src/include/index/bwtree.h | 2 +- src/logging/wal_recovery.cpp | 1 + test/logging/logging_test.cpp | 78 ++++++++++++---------------------- test/logging/recovery_test.cpp | 30 ++++++++++++- 4 files changed, 57 insertions(+), 54 deletions(-) diff --git a/src/include/index/bwtree.h b/src/include/index/bwtree.h index 4849682ab3c..f9352aad09a 100755 --- a/src/include/index/bwtree.h +++ b/src/include/index/bwtree.h @@ -7585,7 +7585,7 @@ class BwTree : public BwTreeBase { // would always fail, until we have cleaned all epoch nodes current_epoch_p = nullptr; - LOG_DEBUG("Clearing the epoch in ~EpochManager()..."); + LOG_TRACE("Clearing the epoch in ~EpochManager()..."); // If all threads has exited then all thread counts are // 0, and therefore this should proceed way to the end diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp index ba297a8ef4e..f8fda17ece4 100644 --- a/src/logging/wal_recovery.cpp +++ b/src/logging/wal_recovery.cpp @@ -32,6 +32,7 @@ namespace peloton{ namespace logging{ void WalRecovery::StartRecovery(){ + LOG_INFO("log_path_: %s", log_path_.c_str()); if(!LoggingUtil::OpenFile(log_path_.c_str(), std::ifstream::in, fstream_)) { diff --git a/test/logging/logging_test.cpp b/test/logging/logging_test.cpp index 76a6a0053a6..ad771c946de 100644 --- a/test/logging/logging_test.cpp +++ b/test/logging/logging_test.cpp @@ -18,6 +18,7 @@ #include "network/postgres_protocol_handler.h" #include "util/string_util.h" #include "network/connection_handle_factory.h" +#include "sql/testing_sql_util.h" namespace peloton { namespace test { @@ -28,61 +29,36 @@ namespace test { class LoggingTests : public PelotonTest {}; -void *LoggingTest(int port) { - try { - // forcing the factory to generate jdbc protocol handler - pqxx::connection C(StringUtil::Format( - "host=127.0.0.1 port=%d user=default_database sslmode=disable", port)); - LOG_INFO("[LoggingTest] Connected to %s", C.dbname()); - pqxx::work txn1(C); +TEST_F(LoggingTests, InsertLoggingTest) { + settings::SettingsManager::SetString(settings::SettingId::log_directory_name, + "./logging"); + settings::SettingsManager::SetString(settings::SettingId::log_file_name, + "wal.log"); + settings::SettingsManager::SetBool(settings::SettingId::enable_logging, true); + settings::SettingsManager::SetBool(settings::SettingId::enable_recovery, false); + PelotonInit::Initialize(); - peloton::network::ConnectionHandle *conn = - peloton::network::ConnectionHandleFactory::GetInstance().ConnectionHandleAt( - peloton::network::PelotonServer::recent_connfd).get(); + // Generate table and data for logging. + // TestingSQLUtil::ExecuteSQLQuery("BEGIN;"); + TestingSQLUtil::ExecuteSQLQuery( + "CREATE TABLE test_table (id INTEGER, value1 " + "REAL, value2 VARCHAR(32));"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO test_table VALUES (0, 1.2, 'Aaron');"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO test_table VALUES (1, 12.34, 'loves');"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO test_table VALUES (2, 12345.678912345, 'databases');"); + // TestingSQLUtil::ExecuteSQLQuery("COMMIT;"); - //Check type of protocol handler - network::PostgresProtocolHandler* handler = - dynamic_cast(conn->GetProtocolHandler().get()); + // make sure the data in test_table is correct + std::string sql1 = "SELECT * FROM test_table;"; + std::vector expected1 = {"0|1.2|Aaron", "1|12.34|loves", + "2|12345.7|databases"}; + TestingSQLUtil::ExecuteSQLQueryAndCheckResult(sql1, expected1, false); - EXPECT_NE(handler, nullptr); - - // create table and insert some data - txn1.exec("DROP TABLE IF EXISTS employee;"); - txn1.exec("CREATE TABLE employee(id INT, name VARCHAR(100));"); - txn1.commit(); - - pqxx::work txn2(C); - txn2.exec("INSERT INTO employee VALUES (1, 'Aaron Tian');"); - txn2.exec("INSERT INTO employee VALUES (2, 'Gandeevan Raghuraman');"); - txn2.exec("INSERT INTO employee VALUES (3, 'Anirudh Kanjani');"); - txn2.commit(); - - } catch (const std::exception &e) { - LOG_INFO("[LoggingTest] Exception occurred: %s", e.what()); - EXPECT_TRUE(false); - } - return NULL; + PelotonInit::Shutdown(); } -TEST_F(LoggingTests, LoggingTest) { - peloton::PelotonInit::Initialize(); - LOG_INFO("Server initialized"); - peloton::network::PelotonServer server; - - int port = 15721; - try { - server.SetPort(port); - server.SetupServer(); - } catch (peloton::ConnectionException &exception) { - LOG_INFO("[LaunchServer] exception when launching server"); - } - std::thread serverThread([&]() { server.ServerLoop(); }); - LoggingTest(port); - server.Close(); - serverThread.join(); - peloton::PelotonInit::Shutdown(); - LOG_DEBUG("Peloton has shut down"); - -} } } \ No newline at end of file diff --git a/test/logging/recovery_test.cpp b/test/logging/recovery_test.cpp index d3f939dbd92..c69325d95f6 100644 --- a/test/logging/recovery_test.cpp +++ b/test/logging/recovery_test.cpp @@ -18,9 +18,10 @@ #include "network/postgres_protocol_handler.h" #include "util/string_util.h" #include "network/connection_handle_factory.h" +#include "sql/testing_sql_util.h" namespace peloton { - namespace test { +namespace test { //===--------------------------------------------------------------------===// // Log Recovery Tests @@ -29,7 +30,32 @@ namespace peloton { class RecoveryTests : public PelotonTest {}; TEST_F(RecoveryTests, InsertRecoveryTest) { - + LOG_INFO("start InsertRecoveryTest"); + + // Recover from the log file and test the results + settings::SettingsManager::SetString(settings::SettingId::log_directory_name, + "./logging"); + settings::SettingsManager::SetString(settings::SettingId::log_file_name, + "wal.log"); + settings::SettingsManager::SetBool(settings::SettingId::enable_logging, true); + settings::SettingsManager::SetBool(settings::SettingId::enable_recovery, true); + + LOG_INFO("before Initialize"); + + PelotonInit::Initialize(); + + LOG_INFO("after Initialize"); + + // make sure the data in test_table is correct + std::string sql1 = "SELECT * FROM test_table;"; + std::vector expected1 = {"0|1.2|Aaron", "1|12.34|loves", + "2|12345.7|databases"}; + TestingSQLUtil::ExecuteSQLQueryAndCheckResult(sql1, expected1, false); + + LOG_INFO("after ExecuteSQLQueryAndCheckResult"); + + PelotonInit::Shutdown(); } + } } \ No newline at end of file