Added initial throttling support (#139, #904, #926).

packages/providers/src.ts/alchemy-provider.ts

@@ -1,7 +1,9 @@
 "use strict";
 
 import { Network, Networkish } from "@ethersproject/networks";
+import { ConnectionInfo } from "@ethersproject/web";
 
+import { showThrottleMessage } from "./formatter";
 import { WebSocketProvider } from "./websocket-provider";
 
 import { Logger } from "@ethersproject/logger";
@@ -18,7 +20,6 @@ import { UrlJsonRpcProvider } from "./url-json-rpc-provider";
 const defaultApiKey = "_gg7wSSi0KMBsdKnGVfHDueq6xMB9EkC"
 
 export class AlchemyProvider extends UrlJsonRpcProvider {
-    readonly apiKey: string;
 
     static getWebSocketProvider(network?: Networkish, apiKey?: any): WebSocketProvider {
         const provider = new AlchemyProvider(network, apiKey);
@@ -37,7 +38,7 @@ export class AlchemyProvider extends UrlJsonRpcProvider {
         return apiKey;
     }
 
-    static getUrl(network: Network, apiKey: string): string {
+    static getUrl(network: Network, apiKey: string): ConnectionInfo {
         let host = null;
         switch (network.name) {
             case "homestead":
@@ -59,6 +60,14 @@ export class AlchemyProvider extends UrlJsonRpcProvider {
                logger.throwArgumentError("unsupported network", "network", arguments[0]);
         }
 
-        return ("https:/" + "/" + host + apiKey);
+        return {
+            url: ("https:/" + "/" + host + apiKey),
+            throttleCallback: (attempt: number, url: string) => {
+                if (apiKey === defaultApiKey) {
+                    showThrottleMessage();
+                }
+                return Promise.resolve(true);
+            }
+        };
     }
 }

packages/providers/src.ts/etherscan-provider.ts

@@ -6,6 +6,8 @@ import { Network, Networkish } from "@ethersproject/networks";
 import { deepCopy, defineReadOnly } from "@ethersproject/properties";
 import { fetchJson } from "@ethersproject/web";
 
+import { showThrottleMessage } from "./formatter";
+
 import { Logger } from "@ethersproject/logger";
 import { version } from "./_version";
 const logger = new Logger(version);
@@ -34,9 +36,11 @@ function getResult(result: { status?: number, message?: string, result?: any }):
     }
 
     if (result.status != 1 || result.message != "OK") {
-        // @TODO: not any
         const error: any = new Error("invalid response");
         error.result = JSON.stringify(result);
+        if ((result.result || "").toLowerCase().indexOf("rate limit") >= 0) {
+            error.throttleRetry = true;
+        }
         throw error;
     }
 
@@ -44,6 +48,14 @@ function getResult(result: { status?: number, message?: string, result?: any }):
 }
 
 function getJsonResult(result: { jsonrpc: string, result?: any, error?: { code?: number, data?: any, message?: string} } ): any {
+    // This response indicates we are being throttled
+    if (result && (<any>result).status == 0 && (<any>result).message == "NOTOK" && (result.result || "").toLowerCase().indexOf("rate limit") >= 0) {
+        const error: any = new Error("throttled response");
+        error.result = JSON.stringify(result);
+        error.throttleRetry = true;
+        throw error;
+    }
+
     if (result.jsonrpc != "2.0") {
         // @TODO: not any
         const error: any = new Error("invalid response");
@@ -76,6 +88,7 @@ const defaultApiKey = "9D13ZE7XSBTJ94N9BNJ2MA33VMAY2YPIRB";
 export class EtherscanProvider extends BaseProvider{
     readonly baseUrl: string;
     readonly apiKey: string;
+
     constructor(network?: Networkish, apiKey?: string) {
         logger.checkNew(new.target, EtherscanProvider);
 
@@ -126,7 +139,18 @@ export class EtherscanProvider extends BaseProvider{
                 provider: this
             });
 
-            const result = await fetchJson(url, null, procFunc || getJsonResult);
+
+            const connection = {
+                url: url,
+                throttleCallback: (attempt: number, url: string) => {
+                    if (this.apiKey === defaultApiKey) {
+                        showThrottleMessage();
+                    }
+                    return Promise.resolve(true);
+                }
+            };
+
+            const result = await fetchJson(connection, null, procFunc || getJsonResult);
 
             this.emit("debug", {
                 action: "response",
@@ -162,13 +186,13 @@ export class EtherscanProvider extends BaseProvider{
             case "getCode":
                 url += "/api?module=proxy&action=eth_getCode&address=" + params.address;
                 url += "&tag=" + params.blockTag + apiKey;
-                return get(url, getJsonResult);
+                return get(url);
 
             case "getStorageAt":
                 url += "/api?module=proxy&action=eth_getStorageAt&address=" + params.address;
                 url += "&position=" + params.position;
                 url += "&tag=" + params.blockTag + apiKey;
-                return get(url, getJsonResult);
+                return get(url);
 
 
             case "sendTransaction":
@@ -325,7 +349,17 @@ export class EtherscanProvider extends BaseProvider{
                 provider: this
             });
 
-            return fetchJson(url, null, getResult).then((result: Array<any>) => {
+            const connection = {
+                url: url,
+                throttleCallback: (attempt: number, url: string) => {
+                    if (this.apiKey === defaultApiKey) {
+                        showThrottleMessage();
+                    }
+                    return Promise.resolve(true);
+                }
+            }
+
+            return fetchJson(connection, null, getResult).then((result: Array<any>) => {
                 this.emit("debug", {
                     action: "response",
                     request: url,

packages/providers/src.ts/formatter.ts

@@ -455,3 +455,23 @@ export class Formatter {
     }
 }
 
+// Show the throttle message only once
+let throttleMessage = false;
+export function showThrottleMessage() {
+    if (throttleMessage) { return; }
+    throttleMessage = true;
+
+    console.log("========= NOTICE =========")
+    console.log("Request-Rate Exceeded  (this message will not be repeated)");
+    console.log("");
+    console.log("The default API keys for each service are provided as a highly-throttled,");
+    console.log("community resource for low-traffic projects and early prototyping.");
+    console.log("");
+    console.log("While your application will continue to function, we highly recommended");
+    console.log("signing up for your own API keys to improve performance, increase your");
+    console.log("request rate/limit and enable other perks, such as metrics and advanced APIs.");
+    console.log("");
+    console.log("For more details: https:/\/docs.ethers.io/api-keys/");
+    console.log("==========================");
+}
+

packages/providers/src.ts/infura-provider.ts

@@ -4,6 +4,7 @@ import { Network, Networkish } from "@ethersproject/networks";
 import { ConnectionInfo } from "@ethersproject/web";
 
 import { WebSocketProvider } from "./websocket-provider";
+import { showThrottleMessage } from "./formatter";
 
 import { Logger } from "@ethersproject/logger";
 import { version } from "./_version";
@@ -61,7 +62,7 @@ export class InfuraProvider extends UrlJsonRpcProvider {
         return apiKeyObj;
     }
 
-    static getUrl(network: Network, apiKey: any): string | ConnectionInfo {
+    static getUrl(network: Network, apiKey: any): ConnectionInfo {
         let host: string = null;
         switch(network ? network.name: "unknown") {
             case "homestead":
@@ -87,7 +88,13 @@ export class InfuraProvider extends UrlJsonRpcProvider {
         }
 
         const connection: ConnectionInfo = {
-             url: ("https:/" + "/" + host + "/v3/" + apiKey.projectId)
+             url: ("https:/" + "/" + host + "/v3/" + apiKey.projectId),
+             throttleCallback: (attempt: number, url: string) => {
+                 if (apiKey.projectId === defaultProjectId) {
+                     showThrottleMessage();
+                 }
+                 return Promise.resolve(true);
+             }
         };
 
         if (apiKey.projectSecret != null) {

packages/providers/src.ts/nodesmith-provider.ts

@@ -1,3 +1,5 @@
+/* istanbul ignore file */
+
 "use strict";
 
 import { Network } from "@ethersproject/networks";

packages/web/src.ts/index.ts

@@ -10,15 +10,26 @@ const logger = new Logger(version);
 
 import { getUrl, GetUrlResponse } from "./geturl";
 
+function staller(duration: number): Promise<void> {
+    return new Promise((resolve) => {
+        setTimeout(resolve, duration);
+    });
+}
+
 // Exported Types
 export type ConnectionInfo = {
     url: string,
+    headers?: { [key: string]: string | number }
+
     user?: string,
     password?: string,
+
     allowInsecureAuthentication?: boolean,
+
     throttleLimit?: number,
+    throttleCallback?: (attempt: number, url: string) => Promise<boolean>,
+
     timeout?: number,
-    headers?: { [key: string]: string | number }
 };
 
 export interface OnceBlockable {
@@ -48,6 +59,14 @@ export type FetchJsonResponse = {
 type Header = { key: string, value: string };
 
 export function fetchJson(connection: string | ConnectionInfo, json?: string, processFunc?: (value: any, response: FetchJsonResponse) => any): Promise<any> {
+
+    // How many times to retry in the event of a throttle
+    const attemptLimit = (typeof(connection) === "object" && connection.throttleLimit != null) ? connection.throttleLimit: 12;
+    logger.assertArgument((attemptLimit > 0 && (attemptLimit % 1) === 0),
+        "invalid connection throttle limit", "connection.throttleLimit", attemptLimit);
+
+    const throttleCallback = ((typeof(connection) === "object") ? connection.throttleCallback: null);
+
     const headers: { [key: string]: Header } = { };
 
     let url: string = null;
@@ -143,9 +162,26 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr
 
     const runningFetch = (async function() {
 
+        for (let attempt = 0; attempt < attemptLimit; attempt++) {
             let response: GetUrlResponse = null;
+
             try {
                 response = await getUrl(url, options);
+
+                // Exponential back-off throttling (interval = 100ms)
+                if (response.statusCode === 429 && attempt < attemptLimit) {
+                    let tryAgain = true;
+                    if (throttleCallback) {
+                        tryAgain = await throttleCallback(attempt, url);
+                    }
+
+                    if (tryAgain) {
+                        const timeout = 100 * parseInt(String(Math.random() * Math.pow(2, attempt)));
+                        await staller(timeout);
+                        continue;
+                    }
+                }
+
             } catch (error) {
                 response = (<any>error).response;
                 if (response == null) {
@@ -177,13 +213,12 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr
                 });
             }
 
-        runningTimeout.cancel();
-
             let json: any = null;
             if (body != null) {
                 try {
                     json = JSON.parse(body);
                 } catch (error) {
+                    runningTimeout.cancel();
                     logger.throwError("invalid JSON", Logger.errors.SERVER_ERROR, {
                         body: body,
                         error: error,
@@ -198,6 +233,21 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr
                 try {
                     json = await processFunc(json, response);
                 } catch (error) {
+                    // Allow the processFunc to trigger a throttle
+                    if (error.throttleRetry && attempt < attemptLimit) {
+                        let tryAgain = true;
+                        if (throttleCallback) {
+                            tryAgain = await throttleCallback(attempt, url);
+                        }
+
+                        if (tryAgain) {
+                            const timeout = 100 * parseInt(String(Math.random() * Math.pow(2, attempt)));
+                            await staller(timeout);
+                            continue;
+                        }
+                    }
+
+                    runningTimeout.cancel();
                     logger.throwError("processing response error", Logger.errors.SERVER_ERROR, {
                         body: json,
                         error: error,
@@ -208,7 +258,9 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr
                 }
             }
 
+            runningTimeout.cancel();
             return json;
+        }
     })();
 
     return Promise.race([ runningTimeout.promise, runningFetch ]);