diff --git a/census.go b/census.go index a679a2b7..680c4335 100644 --- a/census.go +++ b/census.go @@ -1196,6 +1196,10 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan if weightRecord == "" { weightRecord = "1" } + if weightRecord == "0" { + log.Warnf("address %s has weight %s", address, weightRecord) + continue + } weight, ok = new(big.Int).SetString(weightRecord, 10) if !ok { log.Warnf("invalid weight for address %s: %s", address, weightRecord) @@ -1297,12 +1301,11 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan // the weight is the sum of the weights of all the addresses of the user weight := new(big.Int).SetUint64(0) for _, addr := range user.Addresses { - weightAddress, ok := addressMap[common.HexToAddress(addr).Hex()] + weightAddress, ok := addressMap[helpers.NormalizeAddressString(addr)] if ok { weight = weight.Add(weight, weightAddress) } } - for _, signer := range user.Signers { signerBytes, err := hex.DecodeString(strings.TrimPrefix(signer, "0x")) if err != nil { @@ -1327,7 +1330,6 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan // Fetch the remaining users from the farcaster API count := 0 - log.Debugw("fetching users from farcaster", "count", len(pendingAddresses)) for i := 0; i < len(pendingAddresses); i += neynar.MaxAddressesPerRequest { // Fetch the user data from the farcaster API ctx2, cancel := context.WithTimeout(ctx, 10*time.Second) @@ -1336,7 +1338,7 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan if to > len(pendingAddresses) { to = len(pendingAddresses) } - log.Debugw("fetching users from farcaster", "from", i, "to", to) + log.Debugw("fetching users from neynar", "from", i, "to", to, "total", len(pendingAddresses)) usersData, err := v.fcapi.UserDataByVerificationAddress(ctx2, pendingAddresses[i:to]) if err != nil { if errors.Is(err, farcasterapi.ErrNoDataFound) { @@ -1344,6 +1346,7 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan } log.Errorw(err, "error fetching users from Neynar API") } + log.Debugw("users found on neynar", "count", len(usersData)) for _, userData := range usersData { // Add or update the user on the database dbUser, err := v.db.User(userData.FID) @@ -1353,32 +1356,37 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan userData.FID, userData.Username, userData.Displayname, - userData.VerificationsAddresses, + helpers.NormalizeAddressStringSlice(userData.VerificationsAddresses), userData.Signers, - userData.CustodyAddress, + helpers.NormalizeAddressString(userData.CustodyAddress), 0, ); err != nil { return nil, 0, err } } else { log.Debugw("updating user on database", "fid", userData.FID) - dbUser.Addresses = userData.VerificationsAddresses + dbUser.Addresses = helpers.NormalizeAddressStringSlice(userData.VerificationsAddresses) dbUser.Username = userData.Username dbUser.Signers = userData.Signers - dbUser.CustodyAddress = userData.CustodyAddress + dbUser.CustodyAddress = helpers.NormalizeAddressString(userData.CustodyAddress) if err := v.db.UpdateUser(dbUser); err != nil { return nil, 0, err } } + // find the addres on the map to get the weight // the weight is the sum of the weights of all the addresses of the user weight := new(big.Int).SetUint64(0) for _, addr := range userData.VerificationsAddresses { - weightAddress, ok := addressMap[common.HexToAddress(addr).Hex()] + weightAddress, ok := addressMap[helpers.NormalizeAddressString(addr)] if ok { weight = weight.Add(weight, weightAddress) } } + if weight.Cmp(big.NewInt(0)) == 0 { + log.Warnw("user has no weight, skiping...", "fid", userData.FID, "address", userData.VerificationsAddresses) + continue + } // Add the user to the participants list (with all the signers) for _, signer := range userData.Signers { diff --git a/election_background.go b/election_background.go index 96d6c182..36c0e207 100644 --- a/election_background.go +++ b/election_background.go @@ -6,8 +6,6 @@ import ( "fmt" "time" - "github.com/vocdoni/vote-frame/helpers" - "github.com/vocdoni/vote-frame/mongo" "go.vocdoni.io/dvote/api" "go.vocdoni.io/dvote/apiclient" "go.vocdoni.io/dvote/log" @@ -86,63 +84,3 @@ func finalizeElectionsAtBackround(ctx context.Context, v *vocdoniHandler) { } } } - -// populateElectionsQuestionAtBackground checks for elections without question and populates them. -// Uses a list of API clients to retrieve the election metadata, extract the question and store it in the database. -// Once it finish checking all current elections, it stops. It must run in the background. -func populateElectionsQuestionAtBackground(ctx context.Context, db *mongo.MongoStorage) { - batchSize := int64(50) // Define the batch size - offset := int64(0) - - apiClients := createApiClientsForElectionRecovery() - if len(apiClients) == 0 { - log.Error("failed to create any API client, aborting") - return - } - - for { - select { - case <-ctx.Done(): - return - case <-time.After(30 * time.Second): - elections, total, err := db.LatestElections(batchSize, offset) - if err != nil { - log.Errorw(err, "failed to retrieve latest elections") - return - } - log.Infow("populating elections question", "offset", offset, "total", total) - for _, election := range elections { - if election.Question == "" { - electionIDbytes, err := hex.DecodeString(election.ElectionID) - if err != nil { - log.Errorw(err, fmt.Sprintf("failed to decode electionID %x", election.ElectionID)) - continue - } - apiElection := recoverElectionFromMultipleEndpoints(electionIDbytes, apiClients) - if apiElection == nil { - log.Warnw("failed to recover election metadata", "electionID", election.ElectionID) - continue - } - // Extract the question from the metadata and store it in the database - metadata := helpers.UnpackMetadata(apiElection.Metadata) - if metadata != nil && metadata.Title != nil { - question := metadata.Title["default"] - if err := db.SetElectionQuestion(types.HexBytes(electionIDbytes), question); err != nil { - log.Warnw("failed to set election question", "electionID", election.ElectionID, "error", err) - } - } else { - log.Warnw("missing election metadata", "electionID", election.ElectionID) - } - } - time.Sleep(1 * time.Second) - } - - if offset+batchSize >= total { - log.Info("finished populating elections question") - return - } - - offset += batchSize - } - } -} diff --git a/helpers/helpers.go b/helpers/helpers.go index 482b6f84..4c797126 100644 --- a/helpers/helpers.go +++ b/helpers/helpers.go @@ -4,6 +4,7 @@ import ( "encoding/json" "math/big" + "github.com/ethereum/go-ethereum/common" "go.vocdoni.io/dvote/api" "go.vocdoni.io/dvote/log" ) @@ -127,3 +128,20 @@ func UnpackMetadata(metadata any) *api.ElectionDescription { } return desc } + +// NormalizeAddressString converts an Ethereum address to its normalized form. +func NormalizeAddressString(address string) string { + return common.HexToAddress(address).Hex() +} + +// NormalizeAddressStringSlice converts a slice of Ethereum addresses to their normalized form. +func NormalizeAddressStringSlice(addresses []string) []string { + normalizedAddresses := make([]string, 0, len(addresses)) + for _, address := range addresses { + normalizedAddress := NormalizeAddressString(address) + if normalizedAddress != "" { + normalizedAddresses = append(normalizedAddresses, normalizedAddress) + } + } + return normalizedAddresses +} diff --git a/main.go b/main.go index 08fba327..20094ffc 100644 --- a/main.go +++ b/main.go @@ -678,10 +678,6 @@ func main() { log.Fatal(err) } - // Start the background routine to get missing election questions - // TODO: at some point this function could be removed - go populateElectionsQuestionAtBackground(mainCtx, db) - // if a bot FID is provided, start the bot background process if botFid > 0 { var botAPI farcasterapi.API diff --git a/mongo/mongo.go b/mongo/mongo.go index dec7a43b..5af614f8 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -129,11 +129,14 @@ func (ms *MongoStorage) createIndexes() error { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - // Index model for the 'addresses' field + // Create an index for the 'addresses' field on users addressesIndexModel := mongo.IndexModel{ Keys: bson.D{{Key: "addresses", Value: 1}}, // 1 for ascending order Options: nil, } + if _, err := ms.users.Indexes().CreateOne(ctx, addressesIndexModel); err != nil { + return fmt.Errorf("failed to create index on addresses for users: %w", err) + } // Index model for the 'signers' field signersIndexModel := mongo.IndexModel{ @@ -142,7 +145,7 @@ func (ms *MongoStorage) createIndexes() error { } // Create both indexes - _, err := ms.users.Indexes().CreateMany(ctx, []mongo.IndexModel{addressesIndexModel, signersIndexModel}) + _, err := ms.users.Indexes().CreateOne(ctx, signersIndexModel) if err != nil { return err } diff --git a/mongo/user.go b/mongo/user.go index b27db0be..53ba5f3d 100644 --- a/mongo/user.go +++ b/mongo/user.go @@ -3,9 +3,12 @@ package mongo import ( "context" "fmt" + "regexp" "time" + "github.com/vocdoni/vote-frame/helpers" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "go.vocdoni.io/dvote/log" ) @@ -161,16 +164,39 @@ func (ms *MongoStorage) UsersWithPendingProfile() ([]uint64, error) { return users, nil } +// UserByAddress returns the user that has the given address (case insensitive). If the user is not found, it returns an error. +// Warning, this is expensive and should be used with caution. +func (ms *MongoStorage) UserByAddressCaseInsensitive(address string) (*User, error) { + ms.keysLock.RLock() + defer ms.keysLock.RUnlock() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var userByAddress User + if err := ms.users.FindOne(ctx, bson.M{ + "addresses": bson.M{ + "$regex": "^" + regexp.QuoteMeta(address) + "$", + "$options": "i", + }, + }).Decode(&userByAddress); err != nil { + if err == mongo.ErrNoDocuments { + return nil, ErrUserUnknown + } + return nil, err + } + return &userByAddress, nil +} + // UserByAddress returns the user that has the given address. If the user is not found, it returns an error. func (ms *MongoStorage) UserByAddress(address string) (*User, error) { ms.keysLock.RLock() defer ms.keysLock.RUnlock() - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() var userByAddress User if err := ms.users.FindOne(ctx, bson.M{ - "addresses": bson.M{"$in": []string{address}}, + "addresses": bson.M{"$in": []string{helpers.NormalizeAddressString(address)}}, }).Decode(&userByAddress); err != nil { return nil, ErrUserUnknown }