diff --git a/src/server/routes/admin/backfill.ts b/src/server/routes/admin/backfill.ts new file mode 100644 index 000000000..1a417e503 --- /dev/null +++ b/src/server/routes/admin/backfill.ts @@ -0,0 +1,110 @@ +import { type Static, Type } from "@sinclair/typebox"; +import type { FastifyInstance } from "fastify"; +import { StatusCodes } from "http-status-codes"; +import { TransactionDB } from "../../../shared/db/transactions/db"; +import { standardResponseSchema } from "../../schemas/shared-api-schemas"; + +// SPECIAL LOGIC FOR AMEX +// Two admin routes to backfill transaction data: +// - loadBackfillRoute: Load queueId to status/transactionHash mappings +// - clearBackfillRoute: Clear all backfill entries +// See https://github.com/thirdweb-dev/solutions-customer-scripts/blob/main/amex/scripts/load-backfill-via-api.ts + +const MinedEntrySchema = Type.Object({ + queueId: Type.String({ description: "Queue ID (UUID)" }), + status: Type.Literal("mined"), + transactionHash: Type.String({ description: "Transaction hash (0x...)" }), +}); + +const ErroredEntrySchema = Type.Object({ + queueId: Type.String({ description: "Queue ID (UUID)" }), + status: Type.Literal("errored"), +}); + +const loadRequestBodySchema = Type.Object({ + entries: Type.Array( + Type.Union([MinedEntrySchema, ErroredEntrySchema], { + description: "Entry with status 'mined' requires transactionHash; status 'errored' does not", + }), + { + description: "Array of queueId to status/transactionHash mappings", + maxItems: 10000, + }, + ), +}); + +const loadResponseBodySchema = Type.Object({ + result: Type.Object({ + inserted: Type.Integer({ description: "Number of entries inserted" }), + skipped: Type.Integer({ + description: "Number of entries skipped (already exist)", + }), + }), +}); + +const clearResponseBodySchema = Type.Object({ + result: Type.Object({ + deleted: Type.Integer({ description: "Number of entries deleted" }), + }), +}); + +export async function loadBackfillRoute(fastify: FastifyInstance) { + fastify.route<{ + Body: Static; + Reply: Static; + }>({ + method: "POST", + url: "/admin/backfill", + schema: { + summary: "Load backfill entries", + description: + "Load queueId to transactionHash mappings into the backfill table. Uses SETNX to never overwrite existing entries.", + tags: ["Admin"], + operationId: "loadBackfill", + body: loadRequestBodySchema, + response: { + ...standardResponseSchema, + [StatusCodes.OK]: loadResponseBodySchema, + }, + hide: true, + }, + handler: async (request, reply) => { + const { entries } = request.body; + + const { inserted, skipped } = + await TransactionDB.bulkSetBackfill(entries); + + reply.status(StatusCodes.OK).send({ + result: { inserted, skipped }, + }); + }, + }); +} + +export async function clearBackfillRoute(fastify: FastifyInstance) { + fastify.route<{ + Reply: Static; + }>({ + method: "DELETE", + url: "/admin/backfill", + schema: { + summary: "Clear backfill table", + description: + "Delete all entries from the backfill table. This action cannot be undone.", + tags: ["Admin"], + operationId: "clearBackfill", + response: { + ...standardResponseSchema, + [StatusCodes.OK]: clearResponseBodySchema, + }, + hide: true, + }, + handler: async (_request, reply) => { + const deleted = await TransactionDB.clearBackfill(); + + reply.status(StatusCodes.OK).send({ + result: { deleted }, + }); + }, + }); +} diff --git a/src/server/routes/index.ts b/src/server/routes/index.ts index 1150458dc..4483b70f0 100644 --- a/src/server/routes/index.ts +++ b/src/server/routes/index.ts @@ -1,4 +1,5 @@ import type { FastifyInstance } from "fastify"; +import { clearBackfillRoute, loadBackfillRoute } from "./admin/backfill"; import { getNonceDetailsRoute } from "./admin/nonces"; import { getTransactionDetails } from "./admin/transaction"; import { createAccessToken } from "./auth/access-tokens/create"; @@ -297,4 +298,6 @@ export async function withRoutes(fastify: FastifyInstance) { // Admin await fastify.register(getTransactionDetails); await fastify.register(getNonceDetailsRoute); + await fastify.register(loadBackfillRoute); + await fastify.register(clearBackfillRoute); } diff --git a/src/server/routes/transaction/blockchain/get-logs.ts b/src/server/routes/transaction/blockchain/get-logs.ts index 7f4610018..9ce20ef5f 100644 --- a/src/server/routes/transaction/blockchain/get-logs.ts +++ b/src/server/routes/transaction/blockchain/get-logs.ts @@ -8,6 +8,7 @@ import { eth_getTransactionReceipt, getContract, getRpcClient, + isHex, parseEventLogs, prepareEvent, } from "thirdweb"; @@ -15,6 +16,7 @@ import { resolveContractAbi } from "thirdweb/contract"; import type { TransactionReceipt } from "thirdweb/transaction"; import { TransactionDB } from "../../../../shared/db/transactions/db"; import { getChain } from "../../../../shared/utils/chain"; +import { env } from "../../../../shared/utils/env"; import { thirdwebClient } from "../../../../shared/utils/sdk"; import { createCustomError } from "../../../middleware/error"; import { AddressSchema, TransactionHashSchema } from "../../../schemas/address"; @@ -153,10 +155,23 @@ export async function getTransactionLogs(fastify: FastifyInstance) { // Get the transaction hash from the provided input. let hash: Hex | undefined; if (queueId) { + // Primary lookup const transaction = await TransactionDB.get(queueId); if (transaction?.status === "mined") { hash = transaction.transactionHash; } + + // SPECIAL LOGIC FOR AMEX + // AMEX uses this endpoint to get logs for transactions they didn't receive webhooks for + // the queue ID's were cleaned out of REDIS so we backfilled tx hashes to this backfill table + // see https://github.com/thirdweb-dev/solutions-customer-scripts/blob/main/amex/scripts/load-backfill-via-api.ts + // Fallback to backfill table if enabled and not found + if (!hash && env.ENABLE_TX_BACKFILL_FALLBACK) { + const backfill = await TransactionDB.getBackfill(queueId); + if (backfill?.status === "mined" && backfill.transactionHash && isHex(backfill.transactionHash)) { + hash = backfill.transactionHash as Hex; + } + } } else if (transactionHash) { hash = transactionHash as Hex; } diff --git a/src/server/routes/transaction/status.ts b/src/server/routes/transaction/status.ts index d18fa0333..91f286ceb 100644 --- a/src/server/routes/transaction/status.ts +++ b/src/server/routes/transaction/status.ts @@ -2,6 +2,7 @@ import { type Static, Type } from "@sinclair/typebox"; import type { FastifyInstance } from "fastify"; import { StatusCodes } from "http-status-codes"; import { TransactionDB } from "../../../shared/db/transactions/db"; +import { env } from "../../../shared/utils/env"; import { createCustomError } from "../../middleware/error"; import { standardResponseSchema } from "../../schemas/shared-api-schemas"; import { @@ -9,6 +10,77 @@ import { toTransactionSchema, } from "../../schemas/transaction"; +/** + * Creates a minimal transaction response from backfill data. + * Used when the transaction is not found in Redis but exists in the backfill table. + */ +const createBackfillResponse = ( + queueId: string, + backfill: { status: "mined" | "errored"; transactionHash?: string }, +): Static => { + const baseResponse: Static = { + queueId, + status: backfill.status, + chainId: null, + fromAddress: null, + toAddress: null, + data: null, + extension: null, + value: null, + nonce: null, + gasLimit: null, + gasPrice: null, + maxFeePerGas: null, + maxPriorityFeePerGas: null, + transactionType: null, + transactionHash: null, + queuedAt: null, + sentAt: null, + minedAt: null, + cancelledAt: null, + deployedContractAddress: null, + deployedContractType: null, + errorMessage: null, + sentAtBlockNumber: null, + blockNumber: null, + retryCount: 0, + retryGasValues: null, + retryMaxFeePerGas: null, + retryMaxPriorityFeePerGas: null, + signerAddress: null, + accountAddress: null, + accountSalt: null, + accountFactoryAddress: null, + target: null, + sender: null, + initCode: null, + callData: null, + callGasLimit: null, + verificationGasLimit: null, + preVerificationGas: null, + paymasterAndData: null, + userOpHash: null, + functionName: null, + functionArgs: null, + onChainTxStatus: null, + onchainStatus: null, + effectiveGasPrice: null, + cumulativeGasUsed: null, + batchOperations: null, + }; + + if (backfill.status === "mined" && backfill.transactionHash) { + return { + ...baseResponse, + transactionHash: backfill.transactionHash, + onchainStatus: "success", + onChainTxStatus: 1, + }; + } + + return baseResponse; +}; + // INPUT const requestSchema = Type.Object({ queueId: Type.String({ @@ -75,6 +147,20 @@ export async function getTransactionStatusRoute(fastify: FastifyInstance) { const transaction = await TransactionDB.get(queueId); if (!transaction) { + // SPECIAL LOGIC FOR AMEX + // AMEX uses this endpoint to check transaction status for queue IDs they didn't receive webhooks for. + // The queue ID's were cleaned out of Redis so we backfilled tx data to this backfill table. + // See https://github.com/thirdweb-dev/solutions-customer-scripts/blob/main/amex/scripts/load-backfill-via-api.ts + // Fallback to backfill table if enabled and not found + if (env.ENABLE_TX_BACKFILL_FALLBACK) { + const backfill = await TransactionDB.getBackfill(queueId); + if (backfill) { + return reply.status(StatusCodes.OK).send({ + result: createBackfillResponse(queueId, backfill), + }); + } + } + throw createCustomError( "Transaction not found.", StatusCodes.BAD_REQUEST, @@ -122,6 +208,20 @@ export async function getTransactionStatusQueryParamRoute( const transaction = await TransactionDB.get(queueId); if (!transaction) { + // SPECIAL LOGIC FOR AMEX + // AMEX uses this endpoint to check transaction status for queue IDs they didn't receive webhooks for. + // The queue ID's were cleaned out of Redis so we backfilled tx data to this backfill table. + // See https://github.com/thirdweb-dev/solutions-customer-scripts/blob/main/amex/scripts/load-backfill-via-api.ts + // Fallback to backfill table if enabled and not found + if (env.ENABLE_TX_BACKFILL_FALLBACK) { + const backfill = await TransactionDB.getBackfill(queueId); + if (backfill) { + return reply.status(StatusCodes.OK).send({ + result: createBackfillResponse(queueId, backfill), + }); + } + } + throw createCustomError( "Transaction not found.", StatusCodes.BAD_REQUEST, diff --git a/src/shared/db/transactions/db.ts b/src/shared/db/transactions/db.ts index 3250e5cb2..605ae2a97 100644 --- a/src/shared/db/transactions/db.ts +++ b/src/shared/db/transactions/db.ts @@ -2,6 +2,15 @@ import superjson from "superjson"; import { MAX_REDIS_BATCH_SIZE, redis } from "../../utils/redis/redis"; import type { AnyTransaction } from "../../utils/transaction/types"; +/** + * Backfill entry stored as JSON in Redis. + * Used for transaction status and logs fallback lookup. + */ +export interface BackfillEntry { + status: "mined" | "errored"; + transactionHash?: string; // Only present for mined transactions +} + /** * Schemas * @@ -37,6 +46,7 @@ export class TransactionDB { private static minedTransactionsKey = "transaction:mined"; private static cancelledTransactionsKey = "transaction:cancelled"; private static erroredTransactionsKey = "transaction:errored"; + private static backfillKey = (queueId: string) => `backfill:${queueId}`; /** * Inserts or replaces a transaction details. @@ -208,6 +218,104 @@ export class TransactionDB { return numPruned; }; + + /** + * Gets backfill entry from backfill table. + * Returns parsed JSON or handles backwards compatibility for plain string tx hashes. + */ + static getBackfill = async (queueId: string): Promise => { + const val = await redis.get(this.backfillKey(queueId)); + if (!val) return null; + try { + return JSON.parse(val) as BackfillEntry; + } catch { + // Backwards compatibility: treat plain string as mined tx hash + return { status: "mined", transactionHash: val }; + } + }; + + /** + * @deprecated Use getBackfill instead + * Gets transaction hash from backfill table. + */ + static getBackfillHash = async (queueId: string): Promise => { + const backfill = await this.getBackfill(queueId); + if (backfill?.status === "mined" && backfill.transactionHash) { + return backfill.transactionHash; + } + return null; + }; + + /** + * Sets a backfill entry. Uses SETNX to never overwrite. + * @returns true if set, false if already exists + */ + static setBackfill = async ( + queueId: string, + transactionHash: string, + ): Promise => { + const entry: BackfillEntry = { status: "mined", transactionHash }; + const result = await redis.setnx( + this.backfillKey(queueId), + JSON.stringify(entry), + ); + return result === 1; + }; + + /** + * Bulk set backfill entries. + * @returns { inserted: number, skipped: number } + */ + static bulkSetBackfill = async ( + entries: Array<{ queueId: string; status: "mined" | "errored"; transactionHash?: string }>, + ): Promise<{ inserted: number; skipped: number }> => { + let inserted = 0; + let skipped = 0; + + const pipeline = redis.pipeline(); + for (const { queueId, status, transactionHash } of entries) { + const entry: BackfillEntry = { status, transactionHash }; + pipeline.setnx(this.backfillKey(queueId), JSON.stringify(entry)); + } + + const results = await pipeline.exec(); + for (const [err, result] of results ?? []) { + if (!err && result === 1) { + inserted++; + } else { + skipped++; + } + } + + return { inserted, skipped }; + }; + + /** + * Clears all backfill entries. + * @returns number - The number of entries deleted. + */ + static clearBackfill = async (): Promise => { + let totalDeleted = 0; + let cursor = "0"; + + do { + const [nextCursor, keys] = await redis.scan( + cursor, + "MATCH", + "backfill:*", + "COUNT", + 1000, + ); + cursor = nextCursor; + + if (keys.length > 0) { + const deleted = await redis.unlink(...keys); + totalDeleted += deleted; + } + } while (cursor !== "0"); + + return totalDeleted; + }; } const toSeconds = (timestamp: Date) => timestamp.getTime() / 1000; diff --git a/src/shared/utils/env.ts b/src/shared/utils/env.ts index 3bbbe593d..758cdd7a3 100644 --- a/src/shared/utils/env.ts +++ b/src/shared/utils/env.ts @@ -98,6 +98,8 @@ export const env = createEnv({ SEND_WEBHOOK_QUEUE_CONCURRENCY: z.coerce.number().default(10), + ENABLE_TX_BACKFILL_FALLBACK: boolEnvSchema(false), + /** * Experimental env vars. These may be renamed or removed in future non-major releases. */ @@ -177,6 +179,7 @@ export const env = createEnv({ EXPERIMENTAL__RETRY_PREPARE_USEROP_ERRORS: process.env.EXPERIMENTAL__RETRY_PREPARE_USEROP_ERRORS, SEND_WEBHOOK_QUEUE_CONCURRENCY: process.env.SEND_WEBHOOK_QUEUE_CONCURRENCY, + ENABLE_TX_BACKFILL_FALLBACK: process.env.ENABLE_TX_BACKFILL_FALLBACK, }, onValidationError: (error: ZodError) => { console.error(