Skip to content

Commit 2a342e3

Browse files
iuwqyiriuwqyir
authored
create an endpoint to query token balances (#136)
### TL;DR Added a new API endpoint to fetch token balances by type (ERC20, ERC721, ERC1155) for a given address. ### What changed? - Added `/balances/:owner/:type` endpoint to retrieve token balances - Created materialized views in ClickHouse to track token balances from transfer events - Implemented balance querying with support for pagination and filtering - Added support for hiding zero balances - Introduced token balance data structures and query filters ### How to test? 1. Start the API server 2. Query the endpoint: `/balances/{address}/{token_type}` - `token_type`: "erc20", "erc721", or "erc1155" - Optional query params: - `hide_zero_balances`: true/false - `token_address`: specific token contract - `page` and `limit`: for pagination - `sort_by` and `sort_order`: for sorting ### Why make this change? To provide a convenient way to fetch token balances for addresses across different token standards, enabling better wallet integration and portfolio tracking capabilities.
2 parents 0bbbf18 + 38f40ce commit 2a342e3

File tree

8 files changed

+351
-0
lines changed

8 files changed

+351
-0
lines changed

‎api/api.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,3 +144,14 @@ func GetChainId(c *gin.Context) (*big.Int, error) {
144144
}
145145
return big.NewInt(int64(chainIdInt)), nil
146146
}
147+
148+
func ParseIntQueryParam(value string, defaultValue int) int {
149+
if value == "" {
150+
return defaultValue
151+
}
152+
parsed, err := strconv.Atoi(value)
153+
if err != nil {
154+
return defaultValue
155+
}
156+
return parsed
157+
}

‎cmd/api.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,9 @@ func RunApi(cmd *cobra.Command, args []string) {
8383

8484
// blocks table queries
8585
root.GET("/blocks", handlers.GetBlocks)
86+
87+
// token balance queries
88+
root.GET("/balances/:owner/:type", handlers.GetTokenBalancesByType)
8689
}
8790

8891
r.GET("/health", func(c *gin.Context) {

‎internal/common/balances.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package common
2+
3+
import (
4+
"math/big"
5+
)
6+
7+
type TokenBalance struct {
8+
ChainId *big.Int `json:"chain_id" ch:"chain_id"`
9+
TokenType string `json:"token_type" ch:"token_type"`
10+
TokenAddress string `json:"token_address" ch:"address"`
11+
Owner string `json:"owner" ch:"owner"`
12+
TokenId *big.Int `json:"token_id" ch:"token_id"`
13+
Balance *big.Int `json:"balance" ch:"balance"`
14+
}

‎internal/handlers/token_handlers.go‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
package handlers
2+
3+
import (
4+
"fmt"
5+
"math/big"
6+
"strings"
7+
8+
"github.com/gin-gonic/gin"
9+
"github.com/rs/zerolog/log"
10+
"github.com/thirdweb-dev/indexer/api"
11+
"github.com/thirdweb-dev/indexer/internal/storage"
12+
)
13+
14+
// BalanceModel return type for Swagger documentation
15+
type BalanceModel struct {
16+
ChainId string `json:"chain_id" ch:"chain_id"`
17+
TokenType string `json:"token_type" ch:"token_type"`
18+
TokenAddress string `json:"token_address" ch:"address"`
19+
Owner string `json:"owner" ch:"owner"`
20+
TokenId string `json:"token_id" ch:"token_id"`
21+
Balance *big.Int `json:"balance" ch:"balance"`
22+
}
23+
24+
// @Summary Get token balances of an address by type
25+
// @Description Retrieve token balances of an address by type
26+
// @Tags balances
27+
// @Accept json
28+
// @Produce json
29+
// @Security BasicAuth
30+
// @Param chainId path string true "Chain ID"
31+
// @Param owner path string true "Owner address"
32+
// @Param type path string true "Type of token balance"
33+
// @Param hide_zero_balances query bool true "Hide zero balances"
34+
// @Param page query int false "Page number for pagination"
35+
// @Param limit query int false "Number of items per page" default(5)
36+
// @Success 200 {object} api.QueryResponse{data=[]LogModel}
37+
// @Failure 400 {object} api.Error
38+
// @Failure 401 {object} api.Error
39+
// @Failure 500 {object} api.Error
40+
// @Router /{chainId}/events [get]
41+
func GetTokenBalancesByType(c *gin.Context) {
42+
chainId, err := api.GetChainId(c)
43+
if err != nil {
44+
api.BadRequestErrorHandler(c, err)
45+
return
46+
}
47+
tokenType := c.Param("type")
48+
if tokenType != "erc20" && tokenType != "erc1155" && tokenType != "erc721" {
49+
api.BadRequestErrorHandler(c, fmt.Errorf("invalid token type '%s'", tokenType))
50+
return
51+
}
52+
owner := strings.ToLower(c.Param("owner"))
53+
if !strings.HasPrefix(owner, "0x") {
54+
api.BadRequestErrorHandler(c, fmt.Errorf("invalid owner address '%s'", owner))
55+
return
56+
}
57+
tokenAddress := strings.ToLower(c.Query("token_address"))
58+
if tokenAddress != "" && !strings.HasPrefix(tokenAddress, "0x") {
59+
api.BadRequestErrorHandler(c, fmt.Errorf("invalid token address '%s'", tokenAddress))
60+
return
61+
}
62+
hideZeroBalances := c.Query("hide_zero_balances") != "false"
63+
qf := storage.BalancesQueryFilter{
64+
ChainId: chainId,
65+
Owner: owner,
66+
TokenType: tokenType,
67+
TokenAddress: tokenAddress,
68+
ZeroBalance: hideZeroBalances,
69+
SortBy: c.Query("sort_by"),
70+
SortOrder: c.Query("sort_order"),
71+
Page: api.ParseIntQueryParam(c.Query("page"), 0),
72+
Limit: api.ParseIntQueryParam(c.Query("limit"), 0),
73+
}
74+
75+
queryResult := api.QueryResponse{
76+
Meta: api.Meta{
77+
ChainId: chainId.Uint64(),
78+
Page: qf.Page,
79+
Limit: qf.Limit,
80+
},
81+
}
82+
83+
mainStorage, err = getMainStorage()
84+
if err != nil {
85+
log.Error().Err(err).Msg("Error getting main storage")
86+
api.InternalErrorHandler(c)
87+
return
88+
}
89+
90+
balancesResult, err := mainStorage.GetTokenBalances(qf)
91+
if err != nil {
92+
log.Error().Err(err).Msg("Error querying balances")
93+
// TODO: might want to choose BadRequestError if it's due to not-allowed functions
94+
api.InternalErrorHandler(c)
95+
return
96+
}
97+
queryResult.Data = balancesResult.Data
98+
sendJSONResponse(c, queryResult)
99+
}

‎internal/storage/clickhouse.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1369,3 +1369,50 @@ func (c *ClickHouseConnector) getTableName(chainId *big.Int, defaultTable string
13691369

13701370
return defaultTable
13711371
}
1372+
1373+
func (c *ClickHouseConnector) GetTokenBalances(qf BalancesQueryFilter) (QueryResult[common.TokenBalance], error) {
1374+
columns := "chain_id, token_type, address, owner, token_id, balance"
1375+
balanceCondition := ">="
1376+
if qf.ZeroBalance {
1377+
balanceCondition = ">"
1378+
}
1379+
query := fmt.Sprintf("SELECT %s FROM %s.token_balances FINAL WHERE chain_id = ? AND token_type = ? AND owner = ? AND balance %s 0", columns, c.cfg.Database, balanceCondition)
1380+
1381+
if qf.TokenAddress != "" {
1382+
query += fmt.Sprintf(" AND address = '%s'", qf.TokenAddress)
1383+
}
1384+
1385+
// Add ORDER BY clause
1386+
if qf.SortBy != "" {
1387+
query += fmt.Sprintf(" ORDER BY %s %s", qf.SortBy, qf.SortOrder)
1388+
}
1389+
1390+
// Add limit clause
1391+
if qf.Page > 0 && qf.Limit > 0 {
1392+
offset := (qf.Page - 1) * qf.Limit
1393+
query += fmt.Sprintf(" LIMIT %d OFFSET %d", qf.Limit, offset)
1394+
} else if qf.Limit > 0 {
1395+
query += fmt.Sprintf(" LIMIT %d", qf.Limit)
1396+
}
1397+
1398+
rows, err := c.conn.Query(context.Background(), query, qf.ChainId, qf.TokenType, qf.Owner)
1399+
if err != nil {
1400+
return QueryResult[common.TokenBalance]{}, err
1401+
}
1402+
defer rows.Close()
1403+
1404+
queryResult := QueryResult[common.TokenBalance]{
1405+
Data: []common.TokenBalance{},
1406+
}
1407+
1408+
for rows.Next() {
1409+
var tb common.TokenBalance
1410+
err := rows.ScanStruct(&tb)
1411+
if err != nil {
1412+
return QueryResult[common.TokenBalance]{}, err
1413+
}
1414+
queryResult.Data = append(queryResult.Data, tb)
1415+
}
1416+
1417+
return queryResult, nil
1418+
}

‎internal/storage/connector.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,20 @@ type QueryFilter struct {
2424
Signature string
2525
ForceConsistentData bool
2626
}
27+
28+
type BalancesQueryFilter struct {
29+
ChainId *big.Int
30+
TokenType string
31+
TokenAddress string
32+
Owner string
33+
ZeroBalance bool
34+
SortBy string
35+
SortOrder string
36+
Page int
37+
Limit int
38+
Offset int
39+
}
40+
2741
type QueryResult[T any] struct {
2842
// TODO: findout how to only allow Log/transaction arrays or split the result
2943
Data []T `json:"data"`
@@ -65,6 +79,8 @@ type IMainStorage interface {
6579
*/
6680
GetBlockHeadersDescending(chainId *big.Int, from *big.Int, to *big.Int) (blockHeaders []common.BlockHeader, err error)
6781
DeleteBlockData(chainId *big.Int, blockNumbers []*big.Int) error
82+
83+
GetTokenBalances(qf BalancesQueryFilter) (QueryResult[common.TokenBalance], error)
6884
}
6985

7086
func NewStorageConnector(cfg *config.StorageConfig) (IStorage, error) {

‎internal/tools/clickhouse_create_token_balances_mv.sql‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
CREATE TABLE IF NOT EXISTS token_balances
2+
(
3+
`token_type` String,
4+
`chain_id` UInt256,
5+
`owner` FixedString(42),
6+
`address` FixedString(42),
7+
`token_id` UInt256,
8+
`balance` Int256
9+
)
10+
ENGINE = SummingMergeTree
11+
ORDER BY (token_type, chain_id, owner, address, token_id);
12+
13+
CREATE MATERIALIZED VIEW IF NOT EXISTS single_token_transfers_mv TO token_balances AS
14+
SELECT chain_id, owner, address, token_type, token_id, sum(amount) as balance
15+
FROM
16+
(
17+
SELECT
18+
chain_id,
19+
address,
20+
(topic_0 = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' AND topic_3 = '') as is_erc20,
21+
(topic_0 = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' AND topic_3 != '') as is_erc721,
22+
(topic_0 = '0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62') as is_erc1155,
23+
if(is_erc1155, concat('0x', substring(topic_2, 27, 40)), concat('0x', substring(topic_1, 27, 40))) AS sender_address,
24+
if(is_erc1155, concat('0x', substring(topic_3, 27, 40)), concat('0x', substring(topic_2, 27, 40))) AS receiver_address,
25+
multiIf(is_erc20, 'erc20', is_erc721, 'erc721', 'erc1155') as token_type,
26+
multiIf(
27+
is_erc1155,
28+
reinterpretAsUInt256(reverse(unhex(substring(data, 3, 64)))),
29+
is_erc721,
30+
reinterpretAsUInt256(reverse(unhex(substring(topic_3, 3, 64)))),
31+
toUInt256(0) -- other
32+
) AS token_id,
33+
multiIf(
34+
is_erc20 AND length(data) = 66,
35+
reinterpretAsInt256(reverse(unhex(substring(data, 3)))),
36+
is_erc721 OR is_erc1155,
37+
toInt256(1),
38+
toInt256(0) -- unknown
39+
) as transfer_amount,
40+
(sign * transfer_amount) as amount
41+
FROM logs
42+
WHERE
43+
topic_0 IN (
44+
'0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef',
45+
'0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62'
46+
)
47+
)
48+
array join
49+
[chain_id, chain_id] AS chain_id,
50+
[sender_address, receiver_address] AS owner,
51+
[-amount, amount] as amount,
52+
[token_type, token_type] AS token_type,
53+
[token_id, token_id] AS token_id,
54+
[address, address] AS address
55+
GROUP BY chain_id, owner, address, token_type, token_id;
56+
57+
CREATE MATERIALIZED VIEW IF NOT EXISTS erc1155_batch_token_transfers_mv TO token_balances AS
58+
SELECT chain_id, owner, address, token_type, token_id, sum(amount) as balance
59+
FROM (
60+
WITH
61+
metadata as (
62+
SELECT
63+
*,
64+
3 + 2 * 64 as ids_length_idx,
65+
ids_length_idx + 64 as ids_values_idx,
66+
reinterpretAsUInt64(reverse(unhex(substring(data, ids_length_idx, 64)))) AS ids_length,
67+
ids_length_idx + 64 + (ids_length * 64) as amounts_length_idx,
68+
reinterpretAsUInt64(reverse(unhex(substring(data, amounts_length_idx, 64)))) AS amounts_length,
69+
amounts_length_idx + 64 as amounts_values_idx
70+
FROM logs
71+
WHERE topic_0 = '0x4a39dc06d4c0dbc64b70af90fd698a233a518aa5d07e595d983b8c0526c8f7fb'
72+
),
73+
decoded AS (
74+
SELECT
75+
*,
76+
arrayMap(
77+
x -> substring(data, ids_values_idx + (x - 1) * 64, 64),
78+
range(1, ids_length + 1)
79+
) AS ids_hex,
80+
arrayMap(
81+
x -> substring(data, amounts_values_idx + (x - 1) * 64, 64),
82+
range(1, amounts_length + 1)
83+
) AS amounts_hex
84+
FROM metadata
85+
)
86+
SELECT
87+
chain_id,
88+
address,
89+
concat('0x', substring(topic_2, 27, 40)) AS sender_address,
90+
concat('0x', substring(topic_3, 27, 40)) AS receiver_address,
91+
'erc1155' as token_type,
92+
reinterpretAsUInt256(reverse(unhex(substring(hex_id, 1, 64)))) AS token_id,
93+
reinterpretAsInt256(reverse(unhex(substring(hex_amount, 1, 64)))) AS transfer_amount,
94+
(sign * transfer_amount) as amount
95+
FROM decoded
96+
ARRAY JOIN ids_hex AS hex_id, amounts_hex AS hex_amount
97+
)
98+
array join
99+
[chain_id, chain_id] AS chain_id,
100+
[sender_address, receiver_address] AS owner,
101+
[-amount, amount] as amount,
102+
[token_type, token_type] AS token_type,
103+
[token_id, token_id] AS token_id,
104+
[address, address] AS address
105+
GROUP BY chain_id, owner, address, token_type, token_id;

‎test/mocks/MockIMainStorage.go‎

Lines changed: 56 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)