WIP: State indexer

cmd/run.go

@@ -168,6 +168,9 @@ func init() {
 	// Tag transaction with node source
 	runCmd.PersistentFlags().Uint64("node-source", 0, "Node ID to tag transaction with. Maps to ../core/lib/nodes.go")
 
+	// State syncer
+	runCmd.PersistentFlags().String("state-change-dir", "", "Path to state syncer directory containing the state change file and index files")
+
 	// Public keys that need their balances monitored. Map of Label to Public key
 	runCmd.PersistentFlags().String("public-key-balances-to-monitor", "",
 		"Comma-separated string of 'label=publicKey'. These balances of the public key provided will be logged in DataDog with the label provided.")

routes/admin_fees.go

@@ -214,10 +214,10 @@ func (fes *APIServer) TxnFeeMapToResponse(skipProfileEntryResponses bool) map[st
 			// Get the ProfileEntryResponse if we need it. Save it in the profileEntryResponseMap to expedite lookup
 			// if we have duplicates.
 			if !skipProfileEntryResponses && utxoView != nil {
-				// Get the PKID
+				// Get the PublicKey
 				pkid := utxoView.GetPKIDForPublicKey(output.PublicKey)
 				var exists bool
-				// Check if the PKID exists in the map
+				// Check if the PublicKey exists in the map
 				profileEntryResponse, exists = profileEntryResponseMap[pkid.PKID]
 				// If it doesn't exist, try to get the ProfileEntry and convert it to a response and save it in the map.
 				if !exists {

routes/admin_jumio.go

@@ -66,7 +66,7 @@ func (fes *APIServer) AdminResetJumioForPublicKey(ww http.ResponseWriter, req *h
 
 	pkid := utxoView.GetPKIDForPublicKey(userMetadata.PublicKey)
 	if pkid == nil {
-		_AddBadRequestError(ww, fmt.Sprintf("AdminResetJumioForPublicKey: No PKID found for public key: %v", requestData.PublicKeyBase58Check))
+		_AddBadRequestError(ww, fmt.Sprintf("AdminResetJumioForPublicKey: No PublicKey found for public key: %v", requestData.PublicKeyBase58Check))
 		return
 	}
 	prefix := GlobalStatePrefixforPKIDTstampnanosToJumioTransaction(pkid.PKID)

routes/admin_referrals.go

@@ -193,7 +193,7 @@ func (fes *APIServer) AdminCreateReferralHash(ww http.ResponseWriter, req *http.
 		userPublicKeyBytes = profile.PublicKey
 	}
 
-	// Get the PKID for the pub key.
+	// Get the PublicKey for the pub key.
 	utxoView, err := fes.backendServer.GetMempool().GetAugmentedUniversalView()
 	if err != nil {
 		_AddBadRequestError(ww, fmt.Sprintf("AdminCreateReferralHash: Problem getting utxoView: %v", err))
@@ -202,7 +202,7 @@ func (fes *APIServer) AdminCreateReferralHash(ww http.ResponseWriter, req *http.
 	referrerPKID := utxoView.GetPKIDForPublicKey(userPublicKeyBytes)
 	if referrerPKID == nil {
 		_AddBadRequestError(ww, fmt.Sprintf(
-			"AdminCreateReferralHash: nil PKID for pubkey: %v", lib.PkToString(userPublicKeyBytes, fes.Params)))
+			"AdminCreateReferralHash: nil PublicKey for pubkey: %v", lib.PkToString(userPublicKeyBytes, fes.Params)))
 		return
 	}
 
@@ -294,7 +294,7 @@ func (fes *APIServer) AdminUpdateReferralHash(ww http.ResponseWriter, req *http.
 	}
 
 	// Make a copy of the referral info. Note that the referrerPKID is a pointer but it should
-	// be safe to leave them pointing to the same PKID in this endpoint.
+	// be safe to leave them pointing to the same PublicKey in this endpoint.
 	updatedReferralInfo := &ReferralInfo{}
 	*updatedReferralInfo = *referralInfo
 
@@ -361,18 +361,18 @@ type AdminGetAllReferralInfoForUserResponse struct {
 func (fes *APIServer) getReferralInfoResponsesForPubKey(pkBytes []byte, includeReferredUsers bool,
 ) (_referralInfoResponses []ReferralInfoResponse, _err error) {
 
-	// Get the PKID for the pub key passed in.
+	// Get the PublicKey for the pub key passed in.
 	utxoView, err := fes.backendServer.GetMempool().GetAugmentedUniversalView()
 	if err != nil {
 		return nil, fmt.Errorf("putReferralHashWithInfo: Problem getting utxoView: %v", err)
 	}
 	referrerPKID := utxoView.GetPKIDForPublicKey(pkBytes)
 	if referrerPKID == nil {
 		return nil, fmt.Errorf(
-			"putReferralHashWithInfo: nil PKID for pubkey: %v", lib.PkToString(pkBytes, fes.Params))
+			"putReferralHashWithInfo: nil PublicKey for pubkey: %v", lib.PkToString(pkBytes, fes.Params))
 	}
 
-	// Build a key to seek all of the referral hashes for this PKID.
+	// Build a key to seek all of the referral hashes for this PublicKey.
 	dbSeekKey := GlobalStateSeekKeyForPKIDReferralHashes(referrerPKID.PKID)
 	keysFound, valsFound, err := fes.GlobalState.Seek(
 		dbSeekKey, dbSeekKey, 0, 0, false /*reverse*/, true /*fetchValue*/)
@@ -670,7 +670,7 @@ func (fes *APIServer) updateOrCreateReferralInfoFromCSVRow(row []string) (_err e
 		referralInfo = *existingReferralInfo
 	}
 
-	// Decode and fill the PKID.
+	// Decode and fill the PublicKey.
 	var err error
 	pkBytes, _, err := lib.Base58CheckDecode(row[CSVColumnPKID])
 	if err != nil || len(pkBytes) != btcec.PubKeyBytesLenCompressed {

routes/admin_tutorial.go

@@ -47,7 +47,7 @@ func (fes *APIServer) AdminUpdateTutorialCreator(ww http.ResponseWriter, req *ht
 
 	pkid := utxoView.GetPKIDForPublicKey(userMetadata.PublicKey)
 	if pkid == nil {
-		_AddBadRequestError(ww, fmt.Sprintf("AdminUpdateTutorialCreator: No PKID found for public key: %v", requestData.PublicKeyBase58Check))
+		_AddBadRequestError(ww, fmt.Sprintf("AdminUpdateTutorialCreator: No PublicKey found for public key: %v", requestData.PublicKeyBase58Check))
 		return
 	}
 	var key []byte

routes/admin_user.go

@@ -482,8 +482,8 @@ func (fes *APIServer) UpdateUsernameVerificationAuditLog(usernameToVerify string
 		return errors.Wrap(fmt.Errorf("UpdateUsernameVerificationAuditLog: Failed to decode verifier public key bytes"), "")
 	}
 
-	// Get verifier's PKID and then get profile entry -- This is separated into two steps since we save the
-	// PKID in the VerificationUsernameAuditLog
+	// Get verifier's PublicKey and then get profile entry -- This is separated into two steps since we save the
+	// PublicKey in the VerificationUsernameAuditLog
 	verifierPKID := utxoView.GetPKIDForPublicKey(verifierPublicKeyBytes)
 	verifierProfileEntry := utxoView.GetProfileEntryForPKID(verifierPKID.PKID)
 	verifierUsername := ""
@@ -609,7 +609,7 @@ type VerifiedUsernameToPKID struct {
 type VerificationUsernameAuditLog struct {
 	// Time at which the verification was granted or removed.
 	TimestampNanos uint64
-	// Username and PKID of the admin who verified the user.
+	// Username and PublicKey of the admin who verified the user.
 	VerifierUsername string
 	VerifierPKID     *lib.PKID
 	// The user who was verified or had their verification removed.
@@ -633,7 +633,7 @@ type FilterAuditLog struct {
 	TimestampNanos uint64
 	// The filter type being updated
 	Filter FilterType
-	// Username and PKID of the admin who filtered the user.
+	// Username and PublicKey of the admin who filtered the user.
 	UpdaterUsername string
 	UpdaterPKID     *lib.PKID
 	// The user who was filtered or had their filter removed.
@@ -690,7 +690,7 @@ func (fes *APIServer) AdminGrantVerificationBadge(ww http.ResponseWriter, req *h
 	}
 	pkidEntryToVerify := utxoView.GetPKIDForPublicKey(pubKey)
 	if pkidEntryToVerify == nil {
-		_AddBadRequestError(ww, fmt.Sprintf("AdminGrantVerificationBadge: PKID not found for username: %s", usernameToVerify))
+		_AddBadRequestError(ww, fmt.Sprintf("AdminGrantVerificationBadge: PublicKey not found for username: %s", usernameToVerify))
 		return
 	}
 
@@ -713,8 +713,8 @@ func (fes *APIServer) AdminGrantVerificationBadge(ww http.ResponseWriter, req *h
 		_AddBadRequestError(ww, fmt.Sprintf("AdminGrantVerificationBadge: error updating audit log of username verification: %v", err))
 		return
 	}
-	// Add username -> PKID mapping
-	// A username must map to a specific PKID, as someone could change their username
+	// Add username -> PublicKey mapping
+	// A username must map to a specific PublicKey, as someone could change their username
 	// and impersonate someone else. For example:
 	// @elonmusk changes his username to @jeffbezos
 	// We verify the username still matches or else it would transfer over
@@ -760,7 +760,7 @@ type AdminRemoveVerificationBadgeResponse struct {
 
 // AdminRemoveVerificationBadge
 //
-// A valid verification mapping will have an element where map[PKID] = username.
+// A valid verification mapping will have an element where map[PublicKey] = username.
 // If the public key still has the same username, the user is considered verified.
 // In order to "delete" a user efficiently, we simply map their public key to an empty string.
 // Since their public key can never have an underlying username of "", it will never show up as verified.
@@ -795,7 +795,7 @@ func (fes *APIServer) AdminRemoveVerificationBadge(ww http.ResponseWriter, req *
 	}
 	pkidEntryToUnverify := utxoView.GetPKIDForPublicKey(pubKey)
 	if pkidEntryToUnverify == nil {
-		_AddBadRequestError(ww, fmt.Sprintf("AdminRemoveVerificationBadge: PKID not found for username: %s", usernameToRemove))
+		_AddBadRequestError(ww, fmt.Sprintf("AdminRemoveVerificationBadge: PublicKey not found for username: %s", usernameToRemove))
 		return
 	}
 
@@ -827,7 +827,7 @@ func (fes *APIServer) AdminRemoveVerificationBadge(ww http.ResponseWriter, req *
 		return
 	}
 
-	// We set the value for this username to an empty PKID to signify that it has been removed.
+	// We set the value for this username to an empty PublicKey to signify that it has been removed.
 	verifiedMapStruct.VerifiedUsernameToPKID[strings.ToLower(usernameToRemove)] = &lib.ZeroPKID
 
 	// Encode the updated entry and stick it in the database.

routes/associations.go

@@ -31,14 +31,15 @@ const MaxAssociationValuesPerQueryLimit = 12
 // ------------
 
 type CreateUserAssociationRequest struct {
-	TransactorPublicKeyBase58Check string            `safeForLogging:"true"`
-	TargetUserPublicKeyBase58Check string            `safeForLogging:"true"`
-	AppPublicKeyBase58Check        string            `safeForLogging:"true"`
-	AssociationType                string            `safeForLogging:"true"`
-	AssociationValue               string            `safeForLogging:"true"`
-	ExtraData                      map[string]string `safeForLogging:"true"`
-	MinFeeRateNanosPerKB           uint64            `safeForLogging:"true"`
-	TransactionFees                []TransactionFee  `safeForLogging:"true"`
+	TransactorPublicKeyBase58Check string `safeForLogging:"true"`
+	TargetUserPublicKeyBase58Check string `safeForLogging:"true"`
+	AppPublicKeyBase58Check        string `safeForLogging:"true"`
+	AssociationType                string `safeForLogging:"true"`
+	// blog-slug:introducing-the-deso-long-form-content-standard
+	AssociationValue     string            `safeForLogging:"true"`
+	ExtraData            map[string]string `safeForLogging:"true"`
+	MinFeeRateNanosPerKB uint64            `safeForLogging:"true"`
+	TransactionFees      []TransactionFee  `safeForLogging:"true"`
 }
 
 type UserAssociationQuery struct {
@@ -1301,7 +1302,7 @@ func (fes *APIServer) _parseAssociationQueryParams(
 	if transactorPublicKeyBase58Check != "" {
 		transactorPKID, err = fes.getPKIDFromPublicKeyBase58Check(utxoView, transactorPublicKeyBase58Check)
 		if err != nil {
-			return nil, nil, nil, nil, errors.New("problem getting PKID for the transactor")
+			return nil, nil, nil, nil, errors.New("problem getting PublicKey for the transactor")
 		}
 	}
 
@@ -1310,7 +1311,7 @@ func (fes *APIServer) _parseAssociationQueryParams(
 	if targetUserPublicKeyBase58Check != "" {
 		targetUserPKID, err = fes.getPKIDFromPublicKeyBase58Check(utxoView, targetUserPublicKeyBase58Check)
 		if err != nil {
-			return nil, nil, nil, nil, errors.New("problem getting PKID for the target user")
+			return nil, nil, nil, nil, errors.New("problem getting PublicKey for the target user")
 		}
 	}
 
@@ -1329,7 +1330,7 @@ func (fes *APIServer) _parseAssociationQueryParams(
 	if appPublicKeyBase58Check != "" {
 		appPKID, err = fes.getPKIDFromPublicKeyBase58Check(utxoView, appPublicKeyBase58Check)
 		if err != nil {
-			return nil, nil, nil, nil, errors.New("problem getting PKID for the app")
+			return nil, nil, nil, nil, errors.New("problem getting PublicKey for the app")
 		}
 	}
 	return transactorPKID, targetUserPKID, postHash, appPKID, nil

routes/dao_coin_exchange.go

@@ -843,7 +843,7 @@ func (fes *APIServer) validateTransactorSellingCoinBalance(
 		return errors.Errorf("Problem fetching UTXOView: %v", err)
 	}
 
-	// Get transactor PKID and public key from public key base58 check.
+	// Get transactor PublicKey and public key from public key base58 check.
 	transactorPKID, err := fes.getPKIDFromPublicKeyBase58Check(
 		utxoView, transactorPublicKeyBase58Check)
 	if err != nil {
@@ -854,7 +854,7 @@ func (fes *APIServer) validateTransactorSellingCoinBalance(
 		return errors.Errorf("Error decoding transactor public key: %v", err)
 	}
 
-	// If buying $DESO, the buying PKID is the ZeroPKID. Else it's the DAO coin's PKID.
+	// If buying $DESO, the buying PublicKey is the ZeroPKID. Else it's the DAO coin's PublicKey.
 	buyingCoinPKID := &lib.ZeroPKID
 	if buyingDAOCoinCreatorPublicKeyBase58Check != DESOCoinIdentifierString {
 		buyingCoinPKID, err = fes.getPKIDFromPublicKeyBase58Check(
@@ -864,7 +864,7 @@ func (fes *APIServer) validateTransactorSellingCoinBalance(
 		}
 	}
 
-	// If selling $DESO, the selling PKID is the ZeroPKID. We consider this the default
+	// If selling $DESO, the selling PublicKey is the ZeroPKID. We consider this the default
 	// case and update if the transactor is actually selling a DAO coin below.
 	sellingCoinPKID := &lib.ZeroPKID
 
@@ -878,7 +878,7 @@ func (fes *APIServer) validateTransactorSellingCoinBalance(
 		}
 		transactorSellingBalanceBaseUnits = uint256.NewInt().SetUint64(desoBalanceNanos)
 	} else {
-		// Get selling coin PKID and public key from public key base58 check.
+		// Get selling coin PublicKey and public key from public key base58 check.
 		sellingCoinPKID, err = fes.getPKIDFromPublicKeyBase58Check(
 			utxoView, sellingDAOCoinCreatorPublicKeyBase58Check)
 		if err != nil {

routes/exchange.go

@@ -1624,7 +1624,7 @@ func (fes *APIServer) GetPostsForFollowFeedForPublicKey(bav *lib.UtxoView, start
 		// the old logic.
 		pubKeyForPKID := bav.GetPublicKeyForPKID(followEntry.FollowedPKID)
 		if len(pubKeyForPKID) == 0 {
-			glog.Errorf("GetPostsForFollowFeedForPublicKey found PKID %v that "+
+			glog.Errorf("GetPostsForFollowFeedForPublicKey found PublicKey %v that "+
 				"does not have public key mapping; this should never happen",
 				lib.PkToString(followEntry.FollowedPKID[:], bav.Params))
 			continue

routes/expose_global_state.go

@@ -15,13 +15,13 @@ func (fes *APIServer) GetVerifiedUsernames(ww http.ResponseWriter, req *http.Req
 	fes.WriteGlobalStateDataToResponse(fes.VerifiedUsernameToPKIDMap, "GetVerifiedUsernames", ww)
 }
 
-// GetBlacklistedPublicKeys returns a map of PKID (as Base58 encoded string) to Blacklist state bytes if global state
+// GetBlacklistedPublicKeys returns a map of PublicKey (as Base58 encoded string) to Blacklist state bytes if global state
 // is exposed.
 func (fes *APIServer) GetBlacklistedPublicKeys(ww http.ResponseWriter, req *http.Request) {
 	fes.WriteGlobalStateDataToResponse(fes.BlacklistedResponseMap, "GetBlacklistedPublicKeys", ww)
 }
 
-// GetGraylistedPublicKeys returns a map of PKID (as Base58 encoded string) to Graylist state bytes if global state
+// GetGraylistedPublicKeys returns a map of PublicKey (as Base58 encoded string) to Graylist state bytes if global state
 // is exposed.
 func (fes *APIServer) GetGraylistedPublicKeys(ww http.ResponseWriter, req *http.Request) {
 	fes.WriteGlobalStateDataToResponse(fes.GraylistedResponseMap, "GetGraylistedPublicKeys", ww)
@@ -93,31 +93,31 @@ func (fes *APIServer) GetVerifiedUsernameMap() (
 	return verifiedUsernameMap, nil
 }
 
-// GetBlacklist returns both a slice of strings and a map of PKID to []byte representing the current state of
+// GetBlacklist returns both a slice of strings and a map of PublicKey to []byte representing the current state of
 // blacklisted users.
 func (fes *APIServer) GetBlacklist(utxoView *lib.UtxoView) (
 	_blacklistedPKIDMap map[lib.PKID][]byte, _err error,
 ) {
 	return fes.GetRestrictedPublicKeys(_GlobalStatePrefixPublicKeyToBlacklistState, utxoView, RoutePathGetBlacklistedPublicKeys)
 }
 
-// GetGraylist returns both a slice of strings and a map of PKID to []byte representing the current state of graylisted
+// GetGraylist returns both a slice of strings and a map of PublicKey to []byte representing the current state of graylisted
 // users.
 func (fes *APIServer) GetGraylist(utxoView *lib.UtxoView) (
 	_graylistedPKIDMap map[lib.PKID][]byte, _err error,
 ) {
 	return fes.GetRestrictedPublicKeys(_GlobalStatePrefixPublicKeyToGraylistState, utxoView, RoutePathGetGraylistedPublicKeys)
 }
 
-// GetUsernameBlacklist returns both a slice of strings and a map of PKID to []byte representing the current state of
+// GetUsernameBlacklist returns both a slice of strings and a map of PublicKey to []byte representing the current state of
 // blacklisted users.
 func (fes *APIServer) GetUsernameBlacklist() (
 	_blacklistedUsernameMap map[string][]byte, _err error,
 ) {
 	return fes.GetRestrictedUsernames(_GlobalStatePrefixUsernameToBlacklistState, RoutePathGetBlacklistedUsernames)
 }
 
-// GetUsernameGraylist returns both a slice of strings and a map of PKID to []byte representing the current state of
+// GetUsernameGraylist returns both a slice of strings and a map of PublicKey to []byte representing the current state of
 // graylisted users.
 func (fes *APIServer) GetUsernameGraylist() (
 	_blacklistedUsernameMap map[string][]byte, _err error,
@@ -126,7 +126,7 @@ func (fes *APIServer) GetUsernameGraylist() (
 }
 
 // GetRestrictedPublicKeys fetches the blacklisted or graylisted public keys from the configured external global state
-// (if available) and merges it with this node's global state. This returns a map of PKID to restricted bytes.
+// (if available) and merges it with this node's global state. This returns a map of PublicKey to restricted bytes.
 func (fes *APIServer) GetRestrictedPublicKeys(prefix []byte, utxoView *lib.UtxoView, routePath string) (
 	_pkidMap map[lib.PKID][]byte, _err error,
 ) {
@@ -138,8 +138,8 @@ func (fes *APIServer) GetRestrictedPublicKeys(prefix []byte, utxoView *lib.UtxoV
 		if err != nil {
 			return nil, err
 		}
-		// Decode the response into the appropriate struct. To use json encoding, we had to convert PKID to a string
-		// so we'll need to convert back from string to PKID here.
+		// Decode the response into the appropriate struct. To use json encoding, we had to convert PublicKey to a string
+		// so we'll need to convert back from string to PublicKey here.
 		stringifiedPKIDsMap := make(map[string][]byte)
 		decoder := json.NewDecoder(bytes.NewReader(restrictedPublicKeyMapBytes))
 		if err = decoder.Decode(&stringifiedPKIDsMap); err != nil {

routes/extra_data_utils.go

@@ -9,6 +9,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"io"
+	"math"
 	"strconv"
 	"strings"
 )
@@ -189,11 +190,20 @@ func DecodePubKeyToUint64MapString(bytes []byte, params *lib.DeSoParams, _ *lib.
 
 func DecodeTransactionSpendingLimit(spendingBytes []byte, params *lib.DeSoParams, utxoView *lib.UtxoView) string {
 	var transactionSpendingLimit lib.TransactionSpendingLimit
-	tipHeight, err := lib.GetBlockTipHeight(utxoView.Handle, false)
-	if err != nil {
-		glog.Errorf("Error getting block tip height from the db")
-		return ""
+	var tipHeight uint64
+	var err error
+
+	if utxoView.Handle != nil {
+		tipHeight, err = lib.GetBlockTipHeight(utxoView.Handle, false)
+		if err != nil {
+			glog.Errorf("Error getting block tip height from the db")
+			return ""
+		}
+	} else {
+		// If we don't have a db handle, we can't get the tip height, so we just set it to the max uint64.
+		tipHeight = math.MaxUint64
 	}
+
 	// Note: we will have to update this with every migration on transaction spending limits
 	blockHeights := []uint64{
 		tipHeight,