debugging fetch fail at >600 registrations

README.md

@@ -2,6 +2,8 @@
 
 Check 📖 [The Waku Simulator Book](https://waku-org.github.io/waku-simulator/)📖
 
+Also see: [AI_CONTEXT.md](./AI_CONTEXT.md) for a concise, structured overview suitable for AI tools.
+
 ## Quickstart
 
 ```
@@ -29,6 +31,26 @@ docker compose up -d
 ```
 (tested with Docker Compose version v2.28.1. Notice that we don't support docker compose v1)
 
+### Timed run + log capture (optional helper)
+
+You can run the whole simulation for a fixed duration and automatically collect per-node logs using the Python helper:
+
+```
+python3 tools/run_simulation.py --duration 10m --logs-dir ./logs
+```
+
+Notes:
+- Requires docker compose v2.
+- Creates a timestamped folder under `--logs-dir` with `nwaku-<i>.log` per replica and `bootstrap.log`.
+- The script derives the Compose project name from the repo folder unless `--project-name` or `COMPOSE_PROJECT_NAME` is set.
+- Pass `--no-down` if you want to keep the stack running after log collection.
+
+Analyze existing logs only (no containers started):
+
+```
+python3 tools/run_simulation.py --analyze-only ./logs/run_YYYYmmdd_HHMMSS
+```
+
 ## Warning
 
 In case arp tables are overflowing:

deploy_rln_contract.sh

@@ -3,10 +3,10 @@
 set -e
 
 # 1. Install foundry and pnpm
-curl -L https://foundry.paradigm.xyz | bash && . /root/.bashrc && foundryup && export PATH=$PATH:$HOME/.foundry/bin
+curl -L https://foundry.paradigm.xyz | bash && . /root/.bashrc && foundryup --install 1.5.0 && export PATH=$PATH:$HOME/.foundry/bin
 
 echo "installing pnpm..."
-npm install -g pnpm
+npm i -g pnpm@10.23.0
 
 # 2. Clone and build the repository
 if [ ! -d "waku-rlnv2-contract" ]; then
@@ -20,7 +20,8 @@ fi
 
 cd /waku-rlnv2-contract
 git checkout $RLN_CONTRACT_REPO_COMMIT
-
+# git checkout temp-with-imt-dep
+ls
 # 3. Compile Contract Repo
 echo "forge install..."
 forge install
@@ -100,4 +101,145 @@ validate_address "$RLN_CONTRACT_ADDRESS" "RLN Proxy"
 printf "\nContract deployment completed successfully"
 printf "\nTOKEN_ADDRESS: $TOKEN_ADDRESS"
 printf "\nRLN_CONTRACT_ADDRESS: $RLN_CONTRACT_ADDRESS"
-printf "\nEach account registering a membership needs to first mint the token and approve the contract to spend it on their behalf."
+printf "\nEach account registering a membership needs to first mint the token and approve the contract to spend it on their behalf."
+
+
+
+# # 7. Run ETH-based minting for first N accounts (for debugging)
+# # Disable set -e for this section so errors don't exit the script
+# set +e
+# printf "\n\n=== Running ETH-based token minting (DEBUG) ===\n"
+# if [ -f "/shared/anvil-config.txt" ]; then
+#     # Configuration
+#     NUM_ACCOUNTS_TO_MINT=${NUM_ACCOUNTS_TO_MINT:-36}
+#     ETH_AMOUNT_PER_MINT=${ETH_AMOUNT_PER_MINT:-5000000000000000000}  # Default: 1 ETH in wei
+    
+#     echo "Token contract: $TOKEN_ADDRESS"
+#     echo "RPC URL: $RPC_URL"
+#     echo "Number of accounts to mint: $NUM_ACCOUNTS_TO_MINT"
+#     echo "ETH amount per mint: $ETH_AMOUNT_PER_MINT wei"
+#     echo ""
+    
+#     # Install jq if not present (needed to parse JSON)
+#     if ! command -v jq &> /dev/null; then
+#         echo "Installing jq..."
+#         apt-get update -qq && apt-get install -y -qq jq
+#     fi
+    
+#     # Extract private keys and addresses from anvil config
+#     PRIVATE_KEYS_FILE=$(mktemp)
+#     ADDRESSES_FILE=$(mktemp)
+#     cat /shared/anvil-config.txt | jq -r '.private_keys[]' | head -n $NUM_ACCOUNTS_TO_MINT > "$PRIVATE_KEYS_FILE"
+#     cat /shared/anvil-config.txt | jq -r '.available_accounts[]' | head -n $NUM_ACCOUNTS_TO_MINT > "$ADDRESSES_FILE"
+    
+#     # Store the deployer's private key before we start looping
+#     DEPLOYER_PRIVATE_KEY=$PRIVATE_KEY
+    
+#     # Create temporary files to store results
+#     SUCCESS_FILE=$(mktemp)
+#     FAIL_FILE=$(mktemp)
+#     echo "0" > "$SUCCESS_FILE"
+#     echo "0" > "$FAIL_FILE"
+    
+#     # Loop through each account and call mintWithETH
+#     ACCOUNT_NUM=1
+#     while [ $ACCOUNT_NUM -le $NUM_ACCOUNTS_TO_MINT ]; do
+#         TARGET_PRIVATE_KEY=$(sed -n "${ACCOUNT_NUM}p" "$PRIVATE_KEYS_FILE")
+#         TARGET_ADDRESS=$(sed -n "${ACCOUNT_NUM}p" "$ADDRESSES_FILE")
+        
+#         if [ -z "$TARGET_PRIVATE_KEY" ] || [ -z "$TARGET_ADDRESS" ]; then
+#             echo "Warning: Missing private key or address for account $ACCOUNT_NUM"
+#             ACCOUNT_NUM=$((ACCOUNT_NUM + 1))
+#             continue
+#         fi
+        
+#         echo "[$ACCOUNT_NUM/$NUM_ACCOUNTS_TO_MINT] Minting tokens with ETH for $TARGET_ADDRESS"
+        
+#         # Step 1: Mint tokens with ETH using the target account's private key
+#         #cast send format: mintWithETH(address) <TO_ACCOUNT> --value <ETH_AMOUNT> --from <MINTING_ACCOUNT>
+#         MINT_TX_OUTPUT=$(cast send $TOKEN_ADDRESS \
+#             "mintWithETH(address)" \
+#             $TARGET_ADDRESS \
+#             --value $ETH_AMOUNT_PER_MINT \
+#             --rpc-url $RPC_URL \
+#             --private-key $TARGET_PRIVATE_KEY \
+#             --from $TARGET_ADDRESS \
+#             2>&1)
+        
+#         MINT_EXIT_CODE=$?
+
+#         if [ $MINT_EXIT_CODE -eq 0 ]; then
+#             MINT_TX_HASH=$(echo "$MINT_TX_OUTPUT" | grep "transactionHash" | awk '{print $2}')
+#             echo "✓ [$ACCOUNT_NUM] Mint successful for $TARGET_ADDRESS"
+#             echo "  Mint Transaction: $MINT_TX_HASH"
+            
+#             # Step 2: Approve the RLN contract to spend tokens
+#             echo "  Approving RLN contract to spend tokens..."
+#             # We need to calculate the approval amount (typically same as or more than mint amount)
+#             # For RLN membership, we need enough tokens based on message limit
+#             # Default approval amount = mint amount (can be adjusted)
+#             APPROVAL_AMOUNT=$ETH_AMOUNT_PER_MINT
+            
+#             APPROVE_TX_OUTPUT=$(cast send $TOKEN_ADDRESS \
+#                 "approve(address,uint256)" \
+#                 $RLN_CONTRACT_ADDRESS \
+#                 $APPROVAL_AMOUNT \
+#                 --rpc-url $RPC_URL \
+#                 --private-key $TARGET_PRIVATE_KEY \
+#                 --from $TARGET_ADDRESS \
+#                 2>&1)
+
+#             APPROVE_EXIT_CODE=$?
+            
+#             if [ $APPROVE_EXIT_CODE -eq 0 ]; then
+#                 APPROVE_TX_HASH=$(echo "$APPROVE_TX_OUTPUT" | grep "transactionHash" | awk '{print $2}')
+#                 echo "  ✓ Approval successful"
+#                 echo "    Approve Transaction: $APPROVE_TX_HASH"
+#                 SUCCESS_COUNT=$(cat "$SUCCESS_FILE")
+#                 echo $((SUCCESS_COUNT + 1)) > "$SUCCESS_FILE"
+#             else
+#                 echo "  ✗ Approval failed for $TARGET_ADDRESS"
+#                 echo "    Error: $APPROVE_TX_OUTPUT"
+#                 FAIL_COUNT=$(cat "$FAIL_FILE")
+#                 echo $((FAIL_COUNT + 1)) > "$FAIL_FILE"
+#             fi
+#         else
+#             echo "✗ [$ACCOUNT_NUM] Mint failed for $TARGET_ADDRESS"
+#             echo "  Error: $MINT_TX_OUTPUT"
+#             FAIL_COUNT=$(cat "$FAIL_FILE")
+#             echo $((FAIL_COUNT + 1)) > "$FAIL_FILE"
+#         fi
+#         echo ""
+        
+#         # Small delay between transactions to avoid nonce issues
+#         if [ $ACCOUNT_NUM -lt $NUM_ACCOUNTS_TO_MINT ]; then
+#             sleep 2
+#         fi
+        
+#         ACCOUNT_NUM=$((ACCOUNT_NUM + 1))
+#     done
+    
+#     # Read final counts
+#     SUCCESS_COUNT=$(cat "$SUCCESS_FILE")
+#     FAIL_COUNT=$(cat "$FAIL_FILE")
+    
+#     # Cleanup temp files
+#     rm -f "$SUCCESS_FILE" "$FAIL_FILE" "$PRIVATE_KEYS_FILE" "$ADDRESSES_FILE"
+    
+#     echo "============================================================"
+#     echo "ETH minting completed: $SUCCESS_COUNT successful, $FAIL_COUNT failed"
+#     echo "============================================================"
+    
+#     if [ $FAIL_COUNT -eq 0 ]; then
+#         printf "\n✓ ETH-based token minting completed successfully\n"
+#     else
+#         printf "\n✗ Some ETH-based token mints failed ($FAIL_COUNT/$NUM_ACCOUNTS_TO_MINT)\n"
+#         printf "This is a debug feature and won't fail the deployment.\n"
+#     fi
+# else
+#     echo "Warning: anvil-config.txt not found at /shared/anvil-config.txt"
+#     echo "Skipping ETH-based token minting."
+# fi
+
+# # Re-enable set -e for any subsequent commands
+# set -e

docker-compose.yml

@@ -13,7 +13,7 @@ services:
 # Accounts are hardcoded to 520 with the idea that nwaku nodes use up to 500 for membership registration and the last 20 are used for ad-hoc testing.
 # The account number and private key pairs of the last 20 accounts can be found in the Register memberships section of the Waku-simulator book.
   foundry:
-    image: ghcr.io/foundry-rs/foundry:nightly-12be76ff47ae0f4fe1a18b74a47e5f0611bab5a0
+    image: ghcr.io/foundry-rs/foundry:v1.4.2
     user: root
     labels:
       com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}'
@@ -25,13 +25,15 @@ services:
         --host=0.0.0.0
         --accounts=520
         --allow-origin=*
-        --block-time=3
         --chain-id=1234
-        --gas-limit=1000000000000000
-        --gas-price=1
-        --balance=1000000000000000
-        --silent
+        --gas-limit=30000000
+        --gas-price=7
+        --base-fee=7
         --config-out=/shared/anvil-config.txt
+        --no-request-size-limit
+        --prune-history
+        --balance=1000000000000000
+        --disable-min-priority-fee
     volumes:
       - privatekeys-volume:/shared
     networks:

tools/token-mint-service/init_node_tokens.py

@@ -37,7 +37,7 @@ class NodeTokenInitializer:
         # Approver private key for adding accounts to approved minters list
         self.approver_private_key = os.getenv('PRIVATE_KEY')
 
-        self.mint_amount = int(os.getenv('MINT_AMOUNT', '5000000000000000000'))  # at least 5 tokens required for membership with RLN_RELAY_MSG_LIMIT=100
+        self.mint_amount = int(os.getenv('MINT_AMOUNT', '900000000000000000000'))  # at least 5 tokens required for membership with RLN_RELAY_MSG_LIMIT=100
         
         if not self.private_key:
             raise ValueError("NODE_PRIVATE_KEY (Ethereum account private key) environment variable is required")