diff --git a/Builds/VisualStudio/stellar-core.vcxproj b/Builds/VisualStudio/stellar-core.vcxproj index 0aa028f9d2..c2556530ad 100644 --- a/Builds/VisualStudio/stellar-core.vcxproj +++ b/Builds/VisualStudio/stellar-core.vcxproj @@ -748,7 +748,6 @@ exit /b 0 - @@ -1166,7 +1165,6 @@ exit /b 0 - diff --git a/Builds/VisualStudio/stellar-core.vcxproj.filters b/Builds/VisualStudio/stellar-core.vcxproj.filters index 7782662115..09a3e07f62 100644 --- a/Builds/VisualStudio/stellar-core.vcxproj.filters +++ b/Builds/VisualStudio/stellar-core.vcxproj.filters @@ -1239,9 +1239,6 @@ transactions - - transactions - transactions @@ -2343,9 +2340,6 @@ transactions - - transactions - transactions diff --git a/src/database/Database.cpp b/src/database/Database.cpp index d1a7e5a447..a92efbb40e 100644 --- a/src/database/Database.cpp +++ b/src/database/Database.cpp @@ -29,7 +29,6 @@ #include "overlay/BanManager.h" #include "overlay/OverlayManager.h" #include "overlay/PeerManager.h" -#include "transactions/TransactionSQL.h" #include "medida/counter.h" #include "medida/timer.h" diff --git a/src/history/HistoryManagerImpl.cpp b/src/history/HistoryManagerImpl.cpp index cbb99afb45..97e14bb174 100644 --- a/src/history/HistoryManagerImpl.cpp +++ b/src/history/HistoryManagerImpl.cpp @@ -29,7 +29,6 @@ #include "main/Application.h" #include "main/Config.h" #include "medida/meter.h" -#include "transactions/TransactionSQL.h" #include "util/BufferedAsioCerealOutputArchive.h" #include "util/Fs.h" #include "util/GlobalChecks.h" diff --git a/src/history/StateSnapshot.cpp b/src/history/StateSnapshot.cpp index dde1af66eb..3cdad1afa0 100644 --- a/src/history/StateSnapshot.cpp +++ b/src/history/StateSnapshot.cpp @@ -14,7 +14,6 @@ #include "history/HistoryManager.h" #include "main/Application.h" #include "main/Config.h" -#include "transactions/TransactionSQL.h" #include "util/Fs.h" #include "util/GlobalChecks.h" #include "util/Logging.h" diff --git a/src/history/test/HistoryTests.cpp b/src/history/test/HistoryTests.cpp index 305f29ff00..721a50d762 100644 --- a/src/history/test/HistoryTests.cpp +++ b/src/history/test/HistoryTests.cpp @@ -1700,12 +1700,6 @@ TEST_CASE("persist publish queue", "[history][publish][acceptance]") LedgerManager::GENESIS_LEDGER_SEQ, publishedCount, out); REQUIRE(scp == 0); - CheckpointBuilder cb(*app1); - auto headers = LedgerHeaderUtils::copyToStream( - app1->getDatabase().getRawSession(), - LedgerManager::GENESIS_LEDGER_SEQ, publishedCount, cb); - REQUIRE(headers == 0); - // We should have either an empty publish queue or a // ledger sometime after the 5th checkpoint (ledger 5 * 8 - 1 = 39) auto minLedger = diff --git a/src/ledger/LedgerHeaderUtils.cpp b/src/ledger/LedgerHeaderUtils.cpp index 1a6a383bc3..b98a0f058a 100644 --- a/src/ledger/LedgerHeaderUtils.cpp +++ b/src/ledger/LedgerHeaderUtils.cpp @@ -7,11 +7,7 @@ #include "crypto/SHA.h" #include "database/Database.h" #include "database/DatabaseUtils.h" -#include "history/CheckpointBuilder.h" #include "util/Decoder.h" -#include "util/GlobalChecks.h" -#include "util/Logging.h" -#include "util/XDRStream.h" #include "util/types.h" #include "xdrpp/marshal.h" @@ -22,16 +18,10 @@ namespace stellar { -namespace LedgerHeaderUtils -{ - -uint32_t -getFlags(LedgerHeader const& lh) +namespace { - return lh.ext.v() == 1 ? lh.ext.v1().flags : 0; -} -bool +static bool isValid(LedgerHeader const& lh) { bool res = (lh.ledgerSeq <= INT32_MAX); @@ -42,6 +32,36 @@ isValid(LedgerHeader const& lh) return res; } +static LedgerHeader +decodeFromData(std::string const& data) +{ + ZoneScoped; + LedgerHeader lh; + std::vector decoded; + decoder::decode_b64(data, decoded); + + xdr::xdr_get g(&decoded.front(), &decoded.back() + 1); + xdr::xdr_argpack_archive(g, lh); + g.done(); + + if (!isValid(lh)) + { + throw std::runtime_error("invalid ledger header (load)"); + } + return lh; +} + +} // anonymous namespace + +namespace LedgerHeaderUtils +{ + +uint32_t +getFlags(LedgerHeader const& lh) +{ + return lh.ext.v() == 1 ? lh.ext.v1().flags : 0; +} + void storeInDatabase(Database& db, LedgerHeader const& header, SessionWrapper& sess) { @@ -84,25 +104,6 @@ storeInDatabase(Database& db, LedgerHeader const& header, SessionWrapper& sess) } } -LedgerHeader -decodeFromData(std::string const& data) -{ - ZoneScoped; - LedgerHeader lh; - std::vector decoded; - decoder::decode_b64(data, decoded); - - xdr::xdr_get g(&decoded.front(), &decoded.back() + 1); - xdr::xdr_argpack_archive(g, lh); - g.done(); - - if (!isValid(lh)) - { - throw std::runtime_error("invalid ledger header (load)"); - } - return lh; -} - std::shared_ptr loadByHash(Database& db, Hash const& hash) { @@ -140,55 +141,6 @@ loadByHash(Database& db, Hash const& hash) return lhPtr; } -uint32_t -loadMaxLedgerSeq(Database& db) -{ - ZoneScoped; - uint32_t seq = 0; - soci::indicator maxIndicator; - auto prep = db.getPreparedStatement( - "SELECT MAX(ledgerseq) FROM ledgerheaders", db.getSession()); - auto& st = prep.statement(); - st.exchange(soci::into(seq, maxIndicator)); - st.define_and_bind(); - st.execute(true); - if (maxIndicator == soci::indicator::i_ok) - { - return seq; - } - return 0; -} - -std::shared_ptr -loadBySequence(Database& db, soci::session& sess, uint32_t seq) -{ - ZoneScoped; - std::shared_ptr lhPtr; - - std::string headerEncoded; - { - ZoneNamedN(selectLedgerHeadersZone, "select ledgerheaders", true); - sess << "SELECT data FROM ledgerheaders " - "WHERE ledgerseq = :s", - soci::into(headerEncoded), soci::use(seq); - } - if (sess.got_data()) - { - auto lh = decodeFromData(headerEncoded); - lhPtr = std::make_shared(lh); - - if (lh.ledgerSeq != seq) - { - throw std::runtime_error(fmt::format( - FMT_STRING("Wrong sequence number in ledger header database: " - "loaded ledger {:d} contains {:d}"), - seq, lh.ledgerSeq)); - } - } - - return lhPtr; -} - void deleteOldEntries(soci::session& sess, uint32_t ledgerSeq, uint32_t count) { @@ -197,38 +149,6 @@ deleteOldEntries(soci::session& sess, uint32_t ledgerSeq, uint32_t count) "ledgerheaders", "ledgerseq"); } -size_t -copyToStream(soci::session& sess, uint32_t ledgerSeq, uint32_t ledgerCount, - CheckpointBuilder& checkpointBuilder) -{ - ZoneNamedN(selectLedgerHeadersZone, "select ledgerheaders history", true); - uint32_t begin = ledgerSeq, end = ledgerSeq + ledgerCount; - releaseAssert(begin <= end); - - std::string headerEncoded; - - soci::statement st = - (sess.prepare << "SELECT data FROM ledgerheaders " - "WHERE ledgerseq >= :begin AND ledgerseq < :end ORDER " - "BY ledgerseq ASC", - soci::into(headerEncoded), soci::use(begin), soci::use(end)); - - size_t n = 0; - st.execute(true); - while (st.got_data()) - { - LedgerHeaderHistoryEntry lhe; - lhe.header = decodeFromData(headerEncoded); - lhe.hash = xdrSha256(lhe.header); - CLOG_DEBUG(Ledger, "Streaming ledger-header {}", lhe.header.ledgerSeq); - checkpointBuilder.appendLedgerHeader(lhe.header, - /* skipStartupCheck */ true); - ++n; - st.fetch(); - } - return n; -} - void maybeDropAndCreateNew(Database& db) { diff --git a/src/ledger/LedgerHeaderUtils.h b/src/ledger/LedgerHeaderUtils.h index 1643a18387..afcbaee1ce 100644 --- a/src/ledger/LedgerHeaderUtils.h +++ b/src/ledger/LedgerHeaderUtils.h @@ -9,32 +9,19 @@ namespace stellar { -class CheckpointBuilder; namespace LedgerHeaderUtils { uint32_t getFlags(LedgerHeader const& lh); -bool isValid(LedgerHeader const& lh); - void storeInDatabase(Database& db, LedgerHeader const& header, SessionWrapper& sess); -LedgerHeader decodeFromData(std::string const& data); - std::shared_ptr loadByHash(Database& db, Hash const& hash); -std::shared_ptr loadBySequence(Database& db, soci::session& sess, - uint32_t seq); - -uint32_t loadMaxLedgerSeq(Database& db); - void deleteOldEntries(soci::session& sess, uint32_t ledgerSeq, uint32_t count); -size_t copyToStream(soci::session& sess, uint32_t ledgerSeq, - uint32_t ledgerCount, CheckpointBuilder& checkpointBuilder); - void maybeDropAndCreateNew(Database& db); } } diff --git a/src/ledger/LedgerManagerImpl.cpp b/src/ledger/LedgerManagerImpl.cpp index 33800cd267..61b80b8720 100644 --- a/src/ledger/LedgerManagerImpl.cpp +++ b/src/ledger/LedgerManagerImpl.cpp @@ -40,7 +40,6 @@ #include "transactions/ParallelApplyUtils.h" #include "transactions/TransactionFrameBase.h" #include "transactions/TransactionMeta.h" -#include "transactions/TransactionSQL.h" #include "transactions/TransactionUtils.h" #include "util/DebugMetaUtils.h" #include "util/Fs.h" diff --git a/src/simulation/LoadGenerator.cpp b/src/simulation/LoadGenerator.cpp index ad38888e71..d288e0c3b5 100644 --- a/src/simulation/LoadGenerator.cpp +++ b/src/simulation/LoadGenerator.cpp @@ -12,7 +12,6 @@ #include "test/TxTests.h" #include "transactions/MutableTransactionResult.h" #include "transactions/TransactionBridge.h" -#include "transactions/TransactionSQL.h" #include "transactions/TransactionUtils.h" #include "transactions/test/SorobanTxTestUtils.h" #include "util/Logging.h" diff --git a/src/test/TxTests.cpp b/src/test/TxTests.cpp index 7b697359fc..f29b9f9f26 100644 --- a/src/test/TxTests.cpp +++ b/src/test/TxTests.cpp @@ -23,7 +23,6 @@ #include "transactions/SignatureUtils.h" #include "transactions/TransactionBridge.h" #include "transactions/TransactionFrame.h" -#include "transactions/TransactionSQL.h" #include "transactions/TransactionUtils.h" #include "util/GlobalChecks.h" #include "util/Logging.h" diff --git a/src/transactions/TransactionSQL.cpp b/src/transactions/TransactionSQL.cpp deleted file mode 100644 index b6167ca86e..0000000000 --- a/src/transactions/TransactionSQL.cpp +++ /dev/null @@ -1,346 +0,0 @@ -// Copyright 2019 Stellar Development Foundation and contributors. Licensed -// under the Apache License, Version 2.0. See the COPYING file at the root -// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 - -#include "transactions/TransactionSQL.h" -#include "crypto/Hex.h" -#include "database/Database.h" -#include "database/DatabaseUtils.h" -#include "herder/TxSetFrame.h" -#include "history/CheckpointBuilder.h" -#include "ledger/LedgerHeaderUtils.h" -#include "main/Application.h" -#include "util/Decoder.h" -#include "util/GlobalChecks.h" -#include "util/XDRStream.h" -#include "xdrpp/marshal.h" -#include "xdrpp/message.h" -#include - -namespace stellar -{ -namespace -{ -using namespace xdr; -// XDR un-marshaller that reconstructs the GeneralizedTxSet using the trimmed -// result of GeneralizedTxSetPacker and the actual TransactionEnvelopes. -class GeneralizedTxSetUnpacker -{ - public: - GeneralizedTxSetUnpacker( - std::vector const& buffer, - UnorderedMap const& txs) - : mBuffer(buffer), mTxs(txs) - { - } - - template - typename std::enable_if< - std::is_same::uint_type>::value>::type - operator()(T& t) - { - uint32_t v; - getBytes(&v, 4); - t = xdr_traits::from_uint(v); - } - - template - typename std::enable_if< - std::is_same::uint_type>::value>::type - operator()(T& t) - { - uint64_t v; - getBytes(&v, 8); - t = xdr_traits::from_uint(v); - } - - template - typename std::enable_if::is_bytes>::type - operator()(T& t) - { - if (xdr_traits::variable_nelem) - { - std::uint32_t size; - getBytes(&size, 4); - t.resize(size); - } - getBytes(t.data(), t.size()); - } - - template - typename std::enable_if::value && - (xdr_traits::is_class || - xdr_traits::is_container)>::type - operator()(T& t) - { - xdr_traits::load(*this, t); - } - - template - typename std::enable_if::value>::type - operator()(T& tx) - { - Hash hash; - getBytes(hash.data(), hash.size()); - auto it = mTxs.find(hash); - if (it == mTxs.end()) - { - throw std::invalid_argument( - "Could not find transaction corresponding to hash."); - } - tx = it->second->getEnvelope(); - } - - private: - void - check(std::size_t n) const - { - if (mCurrIndex + n > mBuffer.size()) - { - throw xdr_overflow( - "Input buffer space overflow in GeneralizedTxSetUnpacker."); - } - } - - void - getBytes(void* buf, size_t len) - { - if (len != 0) - { - check(len); - std::memcpy(buf, mBuffer.data() + mCurrIndex, len); - mCurrIndex += len; - } - } - - std::vector const& mBuffer; - UnorderedMap const& mTxs; - size_t mCurrIndex = 0; -}; - -std::vector>> -readEncodedTxSets(Application& app, soci::session& sess, uint32_t ledgerSeq, - uint32_t ledgerCount) -{ - ZoneScoped; - auto& db = app.getDatabase(); - auto timer = db.getSelectTimer("txsethistory"); - std::string base64TxSet; - uint32_t begin = ledgerSeq, end = ledgerSeq + ledgerCount; - uint32_t curLedgerSeq; - releaseAssert(begin <= end); - soci::statement st = - (sess.prepare << "SELECT ledgerseq, txset FROM txsethistory " - "WHERE ledgerseq >= :begin AND ledgerseq < :end ORDER " - "BY ledgerseq ASC", - soci::into(curLedgerSeq), soci::into(base64TxSet), soci::use(begin), - soci::use(end)); - - std::vector>> txSets; - - st.execute(true); - - while (st.got_data()) - { - auto& [txSetLedgerSeq, encodedTxSet] = txSets.emplace_back(); - txSetLedgerSeq = curLedgerSeq; - decoder::decode_b64(base64TxSet, encodedTxSet); - st.fetch(); - } - return txSets; -} - -void -writeNonGeneralizedTxSetToStream( - Database& db, soci::session& sess, uint32 ledgerSeq, - std::vector const& txs, - TransactionHistoryResultEntry& results, - CheckpointBuilder& checkpointBuilder) -{ - ZoneScoped; - // prepare the txset for saving - auto lh = LedgerHeaderUtils::loadBySequence(db, sess, ledgerSeq); - if (!lh) - { - throw std::runtime_error("Could not find ledger"); - } - auto txSet = - TxSetXDRFrame::makeFromHistoryTransactions(lh->previousLedgerHash, txs); - TransactionHistoryEntry hist; - hist.ledgerSeq = ledgerSeq; - txSet->toXDR(hist.txSet); - checkpointBuilder.appendTransactionSet(ledgerSeq, hist, results.txResultSet, - /* skipStartupCheck */ true); -} - -void -checkEncodedGeneralizedTxSetIsEmpty(std::vector const& encodedTxSet) -{ - ZoneScoped; - UnorderedMap txByHash; - GeneralizedTxSetUnpacker unpacker(encodedTxSet, txByHash); - GeneralizedTransactionSet txSet; - // Unpacker throws when transaction envelope is not found, hence this only - // doesn't throw for empty tx sets. - xdr_argpack_archive(unpacker, txSet); -} - -void -writeGeneralizedTxSetToStream(uint32 ledgerSeq, - std::vector const& encodedTxSet, - std::vector const& txs, - TransactionHistoryResultEntry& results, - CheckpointBuilder& checkpointBuilder) -{ - ZoneScoped; - UnorderedMap txByHash; - for (auto const& tx : txs) - { - txByHash[tx->getFullHash()] = tx.get(); - } - - GeneralizedTxSetUnpacker unpacker(encodedTxSet, txByHash); - TransactionHistoryEntry hist; - hist.ledgerSeq = ledgerSeq; - hist.ext.v(1); - xdr_argpack_archive(unpacker, hist.ext.generalizedTxSet()); - - checkpointBuilder.appendTransactionSet(ledgerSeq, hist, results.txResultSet, - /* skipStartupCheck */ true); -} - -void -writeTxSetToStream( - Database& db, soci::session& sess, uint32 ledgerSeq, - std::vector>> const& encodedTxSets, - std::vector>>::const_iterator& - encodedTxSetIt, - std::vector const& txs, - TransactionHistoryResultEntry& results, - CheckpointBuilder& checkpointBuilder) -{ - ZoneScoped; - - // encodedTxSets may *only* contain generalized tx sets, so if the requested - // ledger is before the first generalized tx set ledger, then we still need - // to emit the legacy tx set. - // Starting with the first generalized tx set, all the tx sets will be - // considered generalized. - if (encodedTxSets.empty() || ledgerSeq < encodedTxSets.front().first) - { - writeNonGeneralizedTxSetToStream(db, sess, ledgerSeq, txs, results, - checkpointBuilder); - } - else - { - // Currently there can be gaps in this method calls when there are no - // transactions in the set. - while (encodedTxSetIt != encodedTxSets.end() && - ledgerSeq != encodedTxSetIt->first) - { - checkEncodedGeneralizedTxSetIsEmpty(encodedTxSetIt->second); - ++encodedTxSetIt; - } - if (encodedTxSetIt == encodedTxSets.end()) - { - throw std::runtime_error( - "Could not find tx set corresponding to the ledger."); - } - writeGeneralizedTxSetToStream(ledgerSeq, encodedTxSetIt->second, txs, - results, checkpointBuilder); - ++encodedTxSetIt; - } -} - -} // namespace - -size_t -populateCheckpointFilesFromDB(Application& app, soci::session& sess, - uint32_t ledgerSeq, uint32_t ledgerCount, - CheckpointBuilder& checkpointBuilder) -{ - ZoneScoped; - - auto encodedTxSets = readEncodedTxSets(app, sess, ledgerSeq, ledgerCount); - auto encodedTxSetIt = encodedTxSets.cbegin(); - - auto& db = app.getDatabase(); - auto timer = db.getSelectTimer("txhistory"); - std::string txBody, txResult, txMeta; - uint32_t begin = ledgerSeq, end = ledgerSeq + ledgerCount; - size_t n = 0; - - TransactionEnvelope tx; - uint32_t curLedgerSeq; - - releaseAssert(begin <= end); - soci::statement st = - (sess.prepare << "SELECT ledgerseq, txbody, txresult FROM txhistory " - "WHERE ledgerseq >= :begin AND ledgerseq < :end ORDER " - "BY ledgerseq ASC, txindex ASC", - soci::into(curLedgerSeq), soci::into(txBody), soci::into(txResult), - soci::use(begin), soci::use(end)); - - auto lh = LedgerHeaderUtils::loadBySequence(db, sess, ledgerSeq); - if (!lh) - { - throw std::runtime_error("Could not find ledger"); - } - - Hash h; - std::vector txs; - TransactionHistoryResultEntry results; - - st.execute(true); - - uint32_t lastLedgerSeq = curLedgerSeq; - results.ledgerSeq = curLedgerSeq; - - while (st.got_data()) - { - if (curLedgerSeq != lastLedgerSeq) - { - writeTxSetToStream(db, sess, lastLedgerSeq, encodedTxSets, - encodedTxSetIt, txs, results, checkpointBuilder); - // reset state - txs.clear(); - results.ledgerSeq = curLedgerSeq; - results.txResultSet.results.clear(); - lastLedgerSeq = curLedgerSeq; - } - - std::vector body; - decoder::decode_b64(txBody, body); - - std::vector result; - decoder::decode_b64(txResult, result); - - xdr::xdr_get g1(&body.front(), &body.back() + 1); - xdr_argpack_archive(g1, tx); - auto txFrame = TransactionFrameBase::makeTransactionFromWire( - app.getNetworkID(), tx); - txs.emplace_back(txFrame); - - xdr::xdr_get g2(&result.front(), &result.back() + 1); - results.txResultSet.results.emplace_back(); - - TransactionResultPair& p = results.txResultSet.results.back(); - xdr_argpack_archive(g2, p); - - if (p.transactionHash != txFrame->getContentsHash()) - { - throw std::runtime_error("transaction mismatch"); - } - - ++n; - st.fetch(); - } - - if (n != 0) - { - writeTxSetToStream(db, sess, lastLedgerSeq, encodedTxSets, - encodedTxSetIt, txs, results, checkpointBuilder); - } - return n; -} -} diff --git a/src/transactions/TransactionSQL.h b/src/transactions/TransactionSQL.h deleted file mode 100644 index d7ba9e55c2..0000000000 --- a/src/transactions/TransactionSQL.h +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2019 Stellar Development Foundation and contributors. Licensed -// under the Apache License, Version 2.0. See the COPYING file at the root -// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 - -#pragma once - -#include "database/Database.h" -#include "overlay/StellarXDR.h" - -namespace stellar -{ -class Application; -class CheckpointBuilder; - -size_t populateCheckpointFilesFromDB(Application& app, soci::session& sess, - uint32_t ledgerSeq, uint32_t ledgerCount, - CheckpointBuilder& checkpointBuilder); -}