do not include users with weight=0 on census

Signed-off-by: p4u <[email protected]>

census.go

@@ -1196,6 +1196,10 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan
 		if weightRecord == "" {
 			weightRecord = "1"
 		}
+		if weightRecord == "0" {
+			log.Warnf("address %s has weight %s", address, weightRecord)
+			continue
+		}
 		weight, ok = new(big.Int).SetString(weightRecord, 10)
 		if !ok {
 			log.Warnf("invalid weight for address %s: %s", address, weightRecord)
@@ -1297,12 +1301,11 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan
 			// the weight is the sum of the weights of all the addresses of the user
 			weight := new(big.Int).SetUint64(0)
 			for _, addr := range user.Addresses {
-				weightAddress, ok := addressMap[common.HexToAddress(addr).Hex()]
+				weightAddress, ok := addressMap[helpers.NormalizeAddressString(addr)]
 				if ok {
 					weight = weight.Add(weight, weightAddress)
 				}
 			}
-
 			for _, signer := range user.Signers {
 				signerBytes, err := hex.DecodeString(strings.TrimPrefix(signer, "0x"))
 				if err != nil {
@@ -1327,7 +1330,6 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan
 
 	// Fetch the remaining users from the farcaster API
 	count := 0
-	log.Debugw("fetching users from farcaster", "count", len(pendingAddresses))
 	for i := 0; i < len(pendingAddresses); i += neynar.MaxAddressesPerRequest {
 		// Fetch the user data from the farcaster API
 		ctx2, cancel := context.WithTimeout(ctx, 10*time.Second)
@@ -1336,14 +1338,15 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan
 		if to > len(pendingAddresses) {
 			to = len(pendingAddresses)
 		}
-		log.Debugw("fetching users from farcaster", "from", i, "to", to)
+		log.Debugw("fetching users from neynar", "from", i, "to", to, "total", len(pendingAddresses))
 		usersData, err := v.fcapi.UserDataByVerificationAddress(ctx2, pendingAddresses[i:to])
 		if err != nil {
 			if errors.Is(err, farcasterapi.ErrNoDataFound) {
 				break
 			}
 			log.Errorw(err, "error fetching users from Neynar API")
 		}
+		log.Debugw("users found on neynar", "count", len(usersData))
 		for _, userData := range usersData {
 			// Add or update the user on the database
 			dbUser, err := v.db.User(userData.FID)
@@ -1353,32 +1356,37 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan
 					userData.FID,
 					userData.Username,
 					userData.Displayname,
-					userData.VerificationsAddresses,
+					helpers.NormalizeAddressStringSlice(userData.VerificationsAddresses),
 					userData.Signers,
-					userData.CustodyAddress,
+					helpers.NormalizeAddressString(userData.CustodyAddress),
 					0,
 				); err != nil {
 					return nil, 0, err
 				}
 			} else {
 				log.Debugw("updating user on database", "fid", userData.FID)
-				dbUser.Addresses = userData.VerificationsAddresses
+				dbUser.Addresses = helpers.NormalizeAddressStringSlice(userData.VerificationsAddresses)
 				dbUser.Username = userData.Username
 				dbUser.Signers = userData.Signers
-				dbUser.CustodyAddress = userData.CustodyAddress
+				dbUser.CustodyAddress = helpers.NormalizeAddressString(userData.CustodyAddress)
 				if err := v.db.UpdateUser(dbUser); err != nil {
 					return nil, 0, err
 				}
 			}
+
 			// find the addres on the map to get the weight
 			// the weight is the sum of the weights of all the addresses of the user
 			weight := new(big.Int).SetUint64(0)
 			for _, addr := range userData.VerificationsAddresses {
-				weightAddress, ok := addressMap[common.HexToAddress(addr).Hex()]
+				weightAddress, ok := addressMap[helpers.NormalizeAddressString(addr)]
 				if ok {
 					weight = weight.Add(weight, weightAddress)
 				}
 			}
+			if weight.Cmp(big.NewInt(0)) == 0 {
+				log.Warnw("user has no weight, skiping...", "fid", userData.FID, "address", userData.VerificationsAddresses)
+				continue
+			}
 
 			// Add the user to the participants list (with all the signers)
 			for _, signer := range userData.Signers {

election_background.go

@@ -6,8 +6,6 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/vocdoni/vote-frame/helpers"
-	"github.com/vocdoni/vote-frame/mongo"
 	"go.vocdoni.io/dvote/api"
 	"go.vocdoni.io/dvote/apiclient"
 	"go.vocdoni.io/dvote/log"
@@ -86,63 +84,3 @@ func finalizeElectionsAtBackround(ctx context.Context, v *vocdoniHandler) {
 		}
 	}
 }
-
-// populateElectionsQuestionAtBackground checks for elections without question and populates them.
-// Uses a list of API clients to retrieve the election metadata, extract the question and store it in the database.
-// Once it finish checking all current elections, it stops. It must run in the background.
-func populateElectionsQuestionAtBackground(ctx context.Context, db *mongo.MongoStorage) {
-	batchSize := int64(50) // Define the batch size
-	offset := int64(0)
-
-	apiClients := createApiClientsForElectionRecovery()
-	if len(apiClients) == 0 {
-		log.Error("failed to create any API client, aborting")
-		return
-	}
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.After(30 * time.Second):
-			elections, total, err := db.LatestElections(batchSize, offset)
-			if err != nil {
-				log.Errorw(err, "failed to retrieve latest elections")
-				return
-			}
-			log.Infow("populating elections question", "offset", offset, "total", total)
-			for _, election := range elections {
-				if election.Question == "" {
-					electionIDbytes, err := hex.DecodeString(election.ElectionID)
-					if err != nil {
-						log.Errorw(err, fmt.Sprintf("failed to decode electionID %x", election.ElectionID))
-						continue
-					}
-					apiElection := recoverElectionFromMultipleEndpoints(electionIDbytes, apiClients)
-					if apiElection == nil {
-						log.Warnw("failed to recover election metadata", "electionID", election.ElectionID)
-						continue
-					}
-					// Extract the question from the metadata and store it in the database
-					metadata := helpers.UnpackMetadata(apiElection.Metadata)
-					if metadata != nil && metadata.Title != nil {
-						question := metadata.Title["default"]
-						if err := db.SetElectionQuestion(types.HexBytes(electionIDbytes), question); err != nil {
-							log.Warnw("failed to set election question", "electionID", election.ElectionID, "error", err)
-						}
-					} else {
-						log.Warnw("missing election metadata", "electionID", election.ElectionID)
-					}
-				}
-				time.Sleep(1 * time.Second)
-			}
-
-			if offset+batchSize >= total {
-				log.Info("finished populating elections question")
-				return
-			}
-
-			offset += batchSize
-		}
-	}
-}

helpers/helpers.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"math/big"
 
+	"github.com/ethereum/go-ethereum/common"
 	"go.vocdoni.io/dvote/api"
 	"go.vocdoni.io/dvote/log"
 )
@@ -127,3 +128,20 @@ func UnpackMetadata(metadata any) *api.ElectionDescription {
 	}
 	return desc
 }
+
+// NormalizeAddressString converts an Ethereum address to its normalized form.
+func NormalizeAddressString(address string) string {
+	return common.HexToAddress(address).Hex()
+}
+
+// NormalizeAddressStringSlice converts a slice of Ethereum addresses to their normalized form.
+func NormalizeAddressStringSlice(addresses []string) []string {
+	normalizedAddresses := make([]string, 0, len(addresses))
+	for _, address := range addresses {
+		normalizedAddress := NormalizeAddressString(address)
+		if normalizedAddress != "" {
+			normalizedAddresses = append(normalizedAddresses, normalizedAddress)
+		}
+	}
+	return normalizedAddresses
+}

main.go

@@ -678,10 +678,6 @@ func main() {
 		log.Fatal(err)
 	}
 
-	// Start the background routine to get missing election questions
-	// TODO: at some point this function could be removed
-	go populateElectionsQuestionAtBackground(mainCtx, db)
-
 	// if a bot FID is provided, start the bot background process
 	if botFid > 0 {
 		var botAPI farcasterapi.API

mongo/mongo.go

@@ -129,11 +129,14 @@ func (ms *MongoStorage) createIndexes() error {
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 
-	// Index model for the 'addresses' field
+	// Create an index for the 'addresses' field on users
 	addressesIndexModel := mongo.IndexModel{
 		Keys:    bson.D{{Key: "addresses", Value: 1}}, // 1 for ascending order
 		Options: nil,
 	}
+	if _, err := ms.users.Indexes().CreateOne(ctx, addressesIndexModel); err != nil {
+		return fmt.Errorf("failed to create index on addresses for users: %w", err)
+	}
 
 	// Index model for the 'signers' field
 	signersIndexModel := mongo.IndexModel{
@@ -142,7 +145,7 @@ func (ms *MongoStorage) createIndexes() error {
 	}
 
 	// Create both indexes
-	_, err := ms.users.Indexes().CreateMany(ctx, []mongo.IndexModel{addressesIndexModel, signersIndexModel})
+	_, err := ms.users.Indexes().CreateOne(ctx, signersIndexModel)
 	if err != nil {
 		return err
 	}

mongo/user.go

@@ -3,9 +3,12 @@ package mongo
 import (
 	"context"
 	"fmt"
+	"regexp"
 	"time"
 
+	"github.com/vocdoni/vote-frame/helpers"
 	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 	"go.vocdoni.io/dvote/log"
 )
@@ -161,16 +164,39 @@ func (ms *MongoStorage) UsersWithPendingProfile() ([]uint64, error) {
 	return users, nil
 }
 
+// UserByAddress returns the user that has the given address (case insensitive). If the user is not found, it returns an error.
+// Warning, this is expensive and should be used with caution.
+func (ms *MongoStorage) UserByAddressCaseInsensitive(address string) (*User, error) {
+	ms.keysLock.RLock()
+	defer ms.keysLock.RUnlock()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	var userByAddress User
+	if err := ms.users.FindOne(ctx, bson.M{
+		"addresses": bson.M{
+			"$regex":   "^" + regexp.QuoteMeta(address) + "$",
+			"$options": "i",
+		},
+	}).Decode(&userByAddress); err != nil {
+		if err == mongo.ErrNoDocuments {
+			return nil, ErrUserUnknown
+		}
+		return nil, err
+	}
+	return &userByAddress, nil
+}
+
 // UserByAddress returns the user that has the given address. If the user is not found, it returns an error.
 func (ms *MongoStorage) UserByAddress(address string) (*User, error) {
 	ms.keysLock.RLock()
 	defer ms.keysLock.RUnlock()
 
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	var userByAddress User
 	if err := ms.users.FindOne(ctx, bson.M{
-		"addresses": bson.M{"$in": []string{address}},
+		"addresses": bson.M{"$in": []string{helpers.NormalizeAddressString(address)}},
 	}).Decode(&userByAddress); err != nil {
 		return nil, ErrUserUnknown
 	}