package main import ( "bytes" "encoding/json" "flag" "fmt" "go/format" "io" "io/ioutil" "log" "net/http" "net/url" "os" "strconv" "strings" "text/template" "time" "github.com/ethereum/go-ethereum/common" "github.com/status-im/status-go/exchanges/cmd/scaffold" ) const ( templateFile = "cmd/exchanges.txt" // must include the cmd prefix because this code is called from the Makefile outputGoFile = "exchanges.go" outputJsonFile = "exchanges.json" baseUrl = "https://sapi.coincarp.com/api" walletUrl = baseUrl + "/v1/market/walletscreen/coin/wallet" walletAddressUrl = baseUrl + "/v1/market/walletscreen/coin/walletaddress" iconBaseUrl = "https://s1.coincarp.com" ethereumCode = "ethereum" mainnetChainType = "" initialPageSize = 30 maxRetries = 10 requestWaitTime = 1000 * time.Millisecond requestTimeout = 5 * time.Second zeroAddress = "0x0000000000000000000000000000000000000000" ) func main() { log.SetFlags(log.Lshortfile | log.LstdFlags) var fromJsonFile string flag.StringVar(&fromJsonFile, "from-json-file", "", "Path to JSON file to use instead of remote source") flag.Parse() var exchangesData []exchangeData if fromJsonFile == "" { log.Println("Fetching from external service...") exchangesData = getExchangesData() } else { log.Println("Fetching from JSON file...") exchangesData = loadExchangesDataFromJson(fromJsonFile) } log.Println("Generating files...") for _, gen := range generators { gen(exchangesData) } } func doRequest(url string) ([]byte, error) { req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:96.0) Gecko/20100101 Firefox/96.0") // Ensure wait time between requests time.Sleep(requestWaitTime) client := http.Client{ Timeout: requestTimeout, } res, err := client.Do(req) if err != nil { log.Fatal(err) } statusCode := res.StatusCode if statusCode != http.StatusOK { err := fmt.Errorf("unsuccessful request: %s - %d %s", url, statusCode, http.StatusText(statusCode)) log.Fatal(err) } b, err := ioutil.ReadAll(res.Body) if err != nil { log.Fatal(err) } defer res.Body.Close() var dataInfo scaffold.DataInfo if err = json.Unmarshal(b, &dataInfo); err != nil { fmt.Println("unmarshall error: ", url) log.Fatal(err) } if dataInfo.Code != http.StatusOK { err := fmt.Errorf("inconsistent response: %s - %d %s", url, dataInfo.Code, dataInfo.Msg) return nil, err } return b, nil } func getExchangesData() []exchangeData { log.Println("Fetching exchanges list...") exchanges, err := getLatestExchangeList() if err != nil { log.Fatalf("could not get list of exchanges: %v", err) } exchangesData := make([]exchangeData, 0, 128) for _, exchange := range exchanges { log.Println("Fetching address list for exchange:", exchange.Name) addresses, err := getLatestExchangeAddresses(exchange.Code) if err != nil { log.Fatalf("could not get list of addresses: %v", err) } exchangeData := buildExchangeData(exchange, addresses) exchangesData = append(exchangesData, exchangeData) } if len(exchangesData) == 0 { log.Fatalf("could not build exchanges list") } return exchangesData } func loadExchangesDataFromJson(filePath string) []exchangeData { file, err := ioutil.ReadFile(filePath) if err != nil { log.Fatal(err) } var data []exchangeData err = json.Unmarshal(file, &data) if err != nil { log.Fatalf("cannot unmarshal data: %v", err) } return data } func getLatestExchangeList() ([]*scaffold.Exchange, error) { page := 1 pageSize := initialPageSize retries := 0 exchanges := make([]*scaffold.Exchange, 0, 128) for { queryParams := url.Values{ "code": {ethereumCode}, "chainType": {mainnetChainType}, "page": {strconv.Itoa(page)}, "pageSize": {strconv.Itoa(pageSize)}, "isexchange": {"false"}, "lang": {"en-US"}, } url := walletUrl + "?" + queryParams.Encode() b, err := doRequest(url) if err != nil { fmt.Println("request error:", err) if retries < maxRetries { page = 1 pageSize++ retries++ exchanges = nil fmt.Println("retry", retries) continue } log.Fatal(err) } var data scaffold.ExchangesData if err = json.Unmarshal(b, &data); err != nil { fmt.Println("unmarshall error: ", url) log.Fatal(err) } exchanges = append(exchanges, data.Entries.List...) if page >= data.Entries.TotalPages { break } page++ } return exchanges, nil } func getLatestExchangeAddresses(exchangeCode string) ([]*scaffold.ExchangeAddress, error) { page := 1 pageSize := initialPageSize retries := 0 addresses := make([]*scaffold.ExchangeAddress, 0, 128) for { queryParams := url.Values{ "code": {ethereumCode}, "exchangecode": {exchangeCode}, "chainType": {mainnetChainType}, "page": {strconv.Itoa(page)}, "pageSize": {strconv.Itoa(pageSize)}, "lang": {"en-US"}, } url := walletAddressUrl + "?" + queryParams.Encode() b, err := doRequest(url) if err != nil { fmt.Println("request error:", err) if retries < maxRetries { page = 1 pageSize++ retries++ addresses = nil fmt.Println("retry", retries) continue } log.Fatal(err) } var data scaffold.ExchangeAddressesData if err = json.Unmarshal(b, &data); err != nil { fmt.Println("unmarshall error: ", url) log.Fatal(err) } addresses = append(addresses, data.Entries.List...) if page >= data.Entries.TotalPages { break } page++ } return addresses, nil } type exchangeData struct { Code string `json:"code"` Name string `json:"name"` Symbol string `json:"symbol"` Logo string `json:"logo"` Addresses []common.Address `json:"addresses"` } func buildExchangeData(exchange *scaffold.Exchange, addresses []*scaffold.ExchangeAddress) exchangeData { data := exchangeData{ Code: exchange.Code, Name: exchange.Name, Symbol: exchange.Symbol, Logo: iconBaseUrl + exchange.Logo, Addresses: []common.Address{}, } for _, exchangeAddress := range addresses { address := common.HexToAddress(exchangeAddress.Address) if address.Hex() == zeroAddress { continue } data.Addresses = append(data.Addresses, address) } return data } type generatorFunc func(exchangesData []exchangeData) var generators = []generatorFunc{ generateJsonFile, generateGoPackage, } func generateJsonFile(exchangesData []exchangeData) { file, err := json.MarshalIndent(exchangesData, "", " ") if err != nil { log.Fatalf("cannot marshal data: %v", err) } err = ioutil.WriteFile(outputJsonFile, file, 0644) if err != nil { log.Fatal(err) } } func toVariableName(input string) string { return "exchange_" + strings.ToLower(strings.Replace(input, "-", "_", -1)) } func addressesJoin(addresses []common.Address) string { list := make([]string, 0, len(addresses)) for _, address := range addresses { list = append(list, "common.HexToAddress(\""+address.String()+"\")") } return strings.Join(list, ", ") } func generateGoPackage(exchangesData []exchangeData) { tpl, err := ioutil.ReadFile(templateFile) if err != nil { log.Fatalf("cannot open template file: %v", err) } funcMap := template.FuncMap{ "toVariableName": toVariableName, "addressesJoin": addressesJoin, } t := template.Must(template.New("go").Funcs(funcMap).Parse(string(tpl))) buf := new(bytes.Buffer) err = t.Execute(buf, exchangesData) if err != nil { log.Fatal(err) } formatted, err := format.Source(buf.Bytes()) if err != nil { log.Fatal(err) } buf = bytes.NewBuffer(formatted) to, err := os.Create(outputGoFile) if err != nil { log.Fatal(err) } defer to.Close() _, err = io.Copy(to, buf) if err != nil { log.Fatal(err) } }