From f27ce612219671f0e2e9bfaa11368cf73f2c2a70 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 4 Feb 2026 14:09:42 -0500 Subject: [PATCH 1/8] feat: Add files for major AI tools, point all to AGENTS.md --- .cursorrules | 1 + .github/copilot-instructions.md | 1 + .windsurfrules | 1 + AGENTS.md | 229 +++++++++++++++++++++++++++++++ CLAUDE.md | 230 +------------------------------- 5 files changed, 233 insertions(+), 229 deletions(-) create mode 100644 .cursorrules create mode 100644 .github/copilot-instructions.md create mode 100644 .windsurfrules create mode 100644 AGENTS.md diff --git a/.cursorrules b/.cursorrules new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/.cursorrules @@ -0,0 +1 @@ +@AGENTS.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/.windsurfrules b/.windsurfrules new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/.windsurfrules @@ -0,0 +1 @@ +@AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..c8c9099 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,229 @@ +# AGENTS.md + +This file provides guidance to AI coding assistants when working with code in this repository. + +## Overview + +This is a CLI tool for migrating users from various authentication platforms (Clerk, Auth0, Supabase, AuthJS) to a Clerk instance. It handles rate limiting, validates user data with Zod schemas, and provides comprehensive logging of successes and failures. + +## Common Commands + +### Development Commands + +- `bun migrate` - Start the migration process (interactive CLI) +- `bun delete` - Delete all migrated users (uses externalId to identify users) +- `bun clean-logs` - Remove all log files from the `./logs` folder +- `bun convert-logs` - Convert NDJSON log files to JSON array format for easier analysis +- `bun run test` - Run all tests with Vitest +- `bun lint` - Run ESLint +- `bun lint:fix` - Auto-fix ESLint issues +- `bun format` - Format code with Prettier +- `bun format:test` - Check formatting without making changes + +### Testing + +- `bun run test` - Run all test files +- `bun run test ` - Run a specific test file (e.g., `bun run test validator.test.ts`) +- `bun run test --watch` - Run tests in watch mode + +## Architecture + +### Transformer System + +The migration tool uses a **transformer pattern** to support different source platforms. Each transformer defines: + +1. **Field Transformer**: Maps source platform fields to Clerk's schema + - Example: Auth0's `_id.$oid` → Clerk's `userId` + - Example: Supabase's `encrypted_password` → Clerk's `password` + - Handles nested field flattening (see `flattenObjectSelectively` in `src/migrate/functions.ts`) + +2. **Optional Default Fields**: Applied to all users from that platform + - Example: Supabase defaults `passwordHasher` to `"bcrypt"` + +3. **Optional Post-Transform**: Custom logic applied after field mapping + - Example: Auth0 converts metadata from string to objects + +**Transformer locations**: `src/migrate/transformers/` + +- `clerk.ts` - Clerk-to-Clerk migrations +- `auth0.ts` - Auth0 migrations +- `supabase.ts` - Supabase migrations +- `authjs.ts` - AuthJS migrations +- `index.ts` - Exports all transformers as array + +**Adding a new transformer**: + +1. Create a new file in `src/migrate/transformers/` with transformer config +2. Export it in `src/migrate/transformers/index.ts` +3. The CLI will automatically include it in the platform selection + +### Data Flow + +``` +User File (CSV/JSON) + ↓ +loadUsersFromFile (functions.ts) + ↓ Parse file + ↓ Apply transformer defaults + ↓ +transformUsers (functions.ts) + ↓ Transform field names via transformer + ↓ Apply transformer postTransform + ↓ Validate with Zod schema + ↓ Log validation errors + ↓ +importUsers (import-users.ts) + ↓ Process sequentially with rate limiting + ↓ +createUser (import-users.ts) + ↓ Create user with primary email/phone + ↓ Add additional emails/phones + ↓ Handle errors and logging +``` + +### Schema Validation + +User validation is centralized in `src/migrate/validator.ts`: + +- Uses Zod for schema validation +- Enforces: at least one verified identifier (email or phone) +- Enforces: passwordHasher required when password is present +- Fields can be single values or arrays (e.g., `email: string | string[]`) +- All fields except `userId` are optional + +**Adding a new field**: Edit `userSchema` in `src/migrate/validator.ts` + +### Rate Limiting + +Rate limits are auto-configured based on instance type (detected from `CLERK_SECRET_KEY`): + +- **Production** (`sk_live_*`): 100 requests/second (Clerk's limit: 1000 req/10s) +- **Development** (`sk_test_*`): 10 requests/second (Clerk's limit: 100 req/10s) + +Configuration in `src/envs-constants.ts`: + +- `RATE_LIMIT` - Requests per second (auto-configured based on instance type) +- `CONCURRENCY_LIMIT` - Number of concurrent requests (defaults to ~95% of rate limit) + - Production: 9 concurrent (assumes 100ms API latency → ~90-95 req/s throughput) + - Development: 1 concurrent (assumes 100ms API latency → ~9-10 req/s throughput) +- Override defaults via `.env` file with `RATE_LIMIT` or `CONCURRENCY_LIMIT` + +The script uses **p-limit for concurrency control** across all API calls: + +- Limits the number of simultaneously executing API calls +- Formula: `CONCURRENCY_LIMIT = RATE_LIMIT * 0.095` (assumes 100ms latency) +- With X concurrent requests and 100ms latency: throughput ≈ X \* 10 req/s +- Shared limiter across ALL operations (user creation, email creation, phone creation) + +**Performance**: + +- Production: ~3,500 users in ~35 seconds (assuming 1 email per user) +- Development: ~3,500 users in ~350 seconds +- Users can increase `CONCURRENCY_LIMIT` for faster processing (may hit some rate limits) + +**Retry logic**: + +- If a 429 occurs, uses Retry-After value from API response +- Falls back to 10 second default if Retry-After not available +- Centralized in `getRetryDelay()` function in `src/utils.ts` +- The script automatically retries up to 5 times (configurable via MAX_RETRIES) + +### Logging System + +All operations create timestamped logs in `./logs/` using NDJSON (Newline-Delimited JSON) format: + +- `{timestamp}-migration.log` - Combined log with all import entries (success, error, validation failures) +- `{timestamp}-user-deletion.log` - Combined log with all deletion entries + +**Log Format**: NDJSON (Newline-Delimited JSON) + +- Each line is a valid JSON object +- Optimized for streaming and crash-safety +- Can append entries without rewriting entire file +- Memory-efficient for large datasets + +**Log Entry Types**: + +1. **Success Entry**: `{ userId: "user_123", status: "success", clerkUserId: "clerk_abc" }` +2. **Error Entry**: `{ userId: "user_456", status: "error", error: "Email already exists", code: "422" }` +3. **Validation Failure**: `{ userId: "user_789", status: "fail", error: "User must have at least one identifier (email, phone, or username)", path: ["email"], row: 5 }` +4. **Additional Identifier Error**: `{ type: "User Creation Error", userId: "user_abc", status: "additional_email_error", error: "..." }` (logged when adding extra emails/phones fails, but user creation succeeded) + +**Converting Logs**: + +- Use `bun convert-logs` to convert NDJSON logs to JSON arrays for easier analysis +- Converted files are saved as `{timestamp}-migration.json` or `{timestamp}-user-deletion.json` +- Useful for importing into spreadsheets or analysis tools + +**Logger functions** in `src/logger.ts`: + +- `importLogger()` - Log import attempt (success/error with optional error code) +- `errorLogger()` - Log additional identifier errors (emails/phones) and retry attempts +- `validationLogger()` - Log validation failures during data transformation +- `deleteLogger()` - Log deletion attempt (success/error with optional error code) +- `deleteErrorLogger()` - Log retry attempts during deletion + +### CLI Analysis Features + +The CLI (in `src/migrate/cli.ts`) analyzes the import file before migration and provides: + +1. **Identifier Analysis**: Shows which users have emails, phones, usernames +2. **Password Analysis**: Prompts whether to migrate users without passwords +3. **User Model Analysis**: Shows first/last name coverage +4. **Dashboard Configuration Guidance**: Tells user which fields to enable/require in Clerk Dashboard +5. **Instance Type Detection**: Prevents importing >500 users to dev instances + +**Key CLI functions**: + +- `runCLI()` - Main CLI orchestrator +- `analyzeFields()` - Analyzes user data for field coverage +- `displayIdentifierAnalysis()` - Shows identifier stats + Dashboard guidance +- `displayPasswordAnalysis()` - Shows password stats + prompts for skipPasswordRequirement +- `loadSettings()` / `saveSettings()` - Persists CLI choices in `.settings` file + +### Error Handling + +The codebase uses a consistent error handling pattern: + +- `tryCatch()` utility (in `src/utils.ts`) - Returns `[result, error]` (error is null on success) +- Used extensively to make additional emails/phones non-fatal +- Rate limit errors (429) trigger automatic retry with delay +- Validation errors are logged but don't stop the migration + +## Important Implementation Notes + +### Clerk-to-Clerk Migrations + +When migrating from Clerk to Clerk (`key === "clerk"`), the transformer consolidates email and phone arrays: + +- Merges `email`, `emailAddresses`, `unverifiedEmailAddresses` into single array +- Merges `phone`, `phoneNumbers`, `unverifiedPhoneNumbers` into single array +- First item becomes primary, rest are added as additional identifiers +- See `transformUsers()` in `src/migrate/functions.ts` around line 129 + +### Password Hasher Validation + +Invalid password hashers cause immediate failure: + +- Valid hashers are defined in `PASSWORD_HASHERS` constant (`src/types.ts`) +- Detection logic in `transformUsers()` checks if hasher exists but is invalid +- Throws detailed error with user ID, row number, and list of valid hashers + +### User Creation Multi-Step Process + +Creating a user involves multiple API calls, all managed by the shared concurrency limiter: + +1. Create user with primary email/phone + core fields (rate-limited) +2. Add additional emails (each rate-limited individually, non-fatal) +3. Add additional phones (each rate-limited individually, non-fatal) + +This is necessary because Clerk's API only accepts one primary identifier per creation call. All API calls share the same concurrency pool, maximizing throughput across all operations. + +### Environment Variable Detection + +The script auto-detects instance type from `CLERK_SECRET_KEY`: + +- Checks if key contains `"live"` → production +- Otherwise → development +- Used to set default delays and enforce user limits +- See `detectInstanceType()` and `createEnvSchema()` in `src/envs-constants.ts` diff --git a/CLAUDE.md b/CLAUDE.md index 3c9a3d1..43c994c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,229 +1 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Overview - -This is a CLI tool for migrating users from various authentication platforms (Clerk, Auth0, Supabase, AuthJS) to a Clerk instance. It handles rate limiting, validates user data with Zod schemas, and provides comprehensive logging of successes and failures. - -## Common Commands - -### Development Commands - -- `bun migrate` - Start the migration process (interactive CLI) -- `bun delete` - Delete all migrated users (uses externalId to identify users) -- `bun clean-logs` - Remove all log files from the `./logs` folder -- `bun convert-logs` - Convert NDJSON log files to JSON array format for easier analysis -- `bun run test` - Run all tests with Vitest -- `bun lint` - Run ESLint -- `bun lint:fix` - Auto-fix ESLint issues -- `bun format` - Format code with Prettier -- `bun format:test` - Check formatting without making changes - -### Testing - -- `bun run test` - Run all test files -- `bun run test ` - Run a specific test file (e.g., `bun run test validator.test.ts`) -- `bun run test --watch` - Run tests in watch mode - -## Architecture - -### Transformer System - -The migration tool uses a **transformer pattern** to support different source platforms. Each transformer defines: - -1. **Field Transformer**: Maps source platform fields to Clerk's schema - - Example: Auth0's `_id.$oid` → Clerk's `userId` - - Example: Supabase's `encrypted_password` → Clerk's `password` - - Handles nested field flattening (see `flattenObjectSelectively` in `src/migrate/functions.ts`) - -2. **Optional Default Fields**: Applied to all users from that platform - - Example: Supabase defaults `passwordHasher` to `"bcrypt"` - -3. **Optional Post-Transform**: Custom logic applied after field mapping - - Example: Auth0 converts metadata from string to objects - -**Transformer locations**: `src/migrate/transformers/` - -- `clerk.ts` - Clerk-to-Clerk migrations -- `auth0.ts` - Auth0 migrations -- `supabase.ts` - Supabase migrations -- `authjs.ts` - AuthJS migrations -- `index.ts` - Exports all transformers as array - -**Adding a new transformer**: - -1. Create a new file in `src/migrate/transformers/` with transformer config -2. Export it in `src/migrate/transformers/index.ts` -3. The CLI will automatically include it in the platform selection - -### Data Flow - -``` -User File (CSV/JSON) - ↓ -loadUsersFromFile (functions.ts) - ↓ Parse file - ↓ Apply transformer defaults - ↓ -transformUsers (functions.ts) - ↓ Transform field names via transformer - ↓ Apply transformer postTransform - ↓ Validate with Zod schema - ↓ Log validation errors - ↓ -importUsers (import-users.ts) - ↓ Process sequentially with rate limiting - ↓ -createUser (import-users.ts) - ↓ Create user with primary email/phone - ↓ Add additional emails/phones - ↓ Handle errors and logging -``` - -### Schema Validation - -User validation is centralized in `src/migrate/validator.ts`: - -- Uses Zod for schema validation -- Enforces: at least one verified identifier (email or phone) -- Enforces: passwordHasher required when password is present -- Fields can be single values or arrays (e.g., `email: string | string[]`) -- All fields except `userId` are optional - -**Adding a new field**: Edit `userSchema` in `src/migrate/validator.ts` - -### Rate Limiting - -Rate limits are auto-configured based on instance type (detected from `CLERK_SECRET_KEY`): - -- **Production** (`sk_live_*`): 100 requests/second (Clerk's limit: 1000 req/10s) -- **Development** (`sk_test_*`): 10 requests/second (Clerk's limit: 100 req/10s) - -Configuration in `src/envs-constants.ts`: - -- `RATE_LIMIT` - Requests per second (auto-configured based on instance type) -- `CONCURRENCY_LIMIT` - Number of concurrent requests (defaults to ~95% of rate limit) - - Production: 9 concurrent (assumes 100ms API latency → ~90-95 req/s throughput) - - Development: 1 concurrent (assumes 100ms API latency → ~9-10 req/s throughput) -- Override defaults via `.env` file with `RATE_LIMIT` or `CONCURRENCY_LIMIT` - -The script uses **p-limit for concurrency control** across all API calls: - -- Limits the number of simultaneously executing API calls -- Formula: `CONCURRENCY_LIMIT = RATE_LIMIT * 0.095` (assumes 100ms latency) -- With X concurrent requests and 100ms latency: throughput ≈ X \* 10 req/s -- Shared limiter across ALL operations (user creation, email creation, phone creation) - -**Performance**: - -- Production: ~3,500 users in ~35 seconds (assuming 1 email per user) -- Development: ~3,500 users in ~350 seconds -- Users can increase `CONCURRENCY_LIMIT` for faster processing (may hit some rate limits) - -**Retry logic**: - -- If a 429 occurs, uses Retry-After value from API response -- Falls back to 10 second default if Retry-After not available -- Centralized in `getRetryDelay()` function in `src/utils.ts` -- The script automatically retries up to 5 times (configurable via MAX_RETRIES) - -### Logging System - -All operations create timestamped logs in `./logs/` using NDJSON (Newline-Delimited JSON) format: - -- `{timestamp}-migration.log` - Combined log with all import entries (success, error, validation failures) -- `{timestamp}-user-deletion.log` - Combined log with all deletion entries - -**Log Format**: NDJSON (Newline-Delimited JSON) - -- Each line is a valid JSON object -- Optimized for streaming and crash-safety -- Can append entries without rewriting entire file -- Memory-efficient for large datasets - -**Log Entry Types**: - -1. **Success Entry**: `{ userId: "user_123", status: "success", clerkUserId: "clerk_abc" }` -2. **Error Entry**: `{ userId: "user_456", status: "error", error: "Email already exists", code: "422" }` -3. **Validation Failure**: `{ userId: "user_789", status: "fail", error: "User must have at least one identifier (email, phone, or username)", path: ["email"], row: 5 }` -4. **Additional Identifier Error**: `{ type: "User Creation Error", userId: "user_abc", status: "additional_email_error", error: "..." }` (logged when adding extra emails/phones fails, but user creation succeeded) - -**Converting Logs**: - -- Use `bun convert-logs` to convert NDJSON logs to JSON arrays for easier analysis -- Converted files are saved as `{timestamp}-migration.json` or `{timestamp}-user-deletion.json` -- Useful for importing into spreadsheets or analysis tools - -**Logger functions** in `src/logger.ts`: - -- `importLogger()` - Log import attempt (success/error with optional error code) -- `errorLogger()` - Log additional identifier errors (emails/phones) and retry attempts -- `validationLogger()` - Log validation failures during data transformation -- `deleteLogger()` - Log deletion attempt (success/error with optional error code) -- `deleteErrorLogger()` - Log retry attempts during deletion - -### CLI Analysis Features - -The CLI (in `src/migrate/cli.ts`) analyzes the import file before migration and provides: - -1. **Identifier Analysis**: Shows which users have emails, phones, usernames -2. **Password Analysis**: Prompts whether to migrate users without passwords -3. **User Model Analysis**: Shows first/last name coverage -4. **Dashboard Configuration Guidance**: Tells user which fields to enable/require in Clerk Dashboard -5. **Instance Type Detection**: Prevents importing >500 users to dev instances - -**Key CLI functions**: - -- `runCLI()` - Main CLI orchestrator -- `analyzeFields()` - Analyzes user data for field coverage -- `displayIdentifierAnalysis()` - Shows identifier stats + Dashboard guidance -- `displayPasswordAnalysis()` - Shows password stats + prompts for skipPasswordRequirement -- `loadSettings()` / `saveSettings()` - Persists CLI choices in `.settings` file - -### Error Handling - -The codebase uses a consistent error handling pattern: - -- `tryCatch()` utility (in `src/utils.ts`) - Returns `[result, error]` (error is null on success) -- Used extensively to make additional emails/phones non-fatal -- Rate limit errors (429) trigger automatic retry with delay -- Validation errors are logged but don't stop the migration - -## Important Implementation Notes - -### Clerk-to-Clerk Migrations - -When migrating from Clerk to Clerk (`key === "clerk"`), the transformer consolidates email and phone arrays: - -- Merges `email`, `emailAddresses`, `unverifiedEmailAddresses` into single array -- Merges `phone`, `phoneNumbers`, `unverifiedPhoneNumbers` into single array -- First item becomes primary, rest are added as additional identifiers -- See `transformUsers()` in `src/migrate/functions.ts` around line 129 - -### Password Hasher Validation - -Invalid password hashers cause immediate failure: - -- Valid hashers are defined in `PASSWORD_HASHERS` constant (`src/types.ts`) -- Detection logic in `transformUsers()` checks if hasher exists but is invalid -- Throws detailed error with user ID, row number, and list of valid hashers - -### User Creation Multi-Step Process - -Creating a user involves multiple API calls, all managed by the shared concurrency limiter: - -1. Create user with primary email/phone + core fields (rate-limited) -2. Add additional emails (each rate-limited individually, non-fatal) -3. Add additional phones (each rate-limited individually, non-fatal) - -This is necessary because Clerk's API only accepts one primary identifier per creation call. All API calls share the same concurrency pool, maximizing throughput across all operations. - -### Environment Variable Detection - -The script auto-detects instance type from `CLERK_SECRET_KEY`: - -- Checks if key contains `"live"` → production -- Otherwise → development -- Used to set default delays and enforce user limits -- See `detectInstanceType()` and `createEnvSchema()` in `src/envs-constants.ts` +@AGENTS.md From f69b78c9180e93475298928bd89bb01aeec38922 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 4 Feb 2026 16:26:04 -0500 Subject: [PATCH 2/8] chore: Add .claude/settings.json to repo --- .claude/settings.json | 21 +++++++++++++++++++++ .gitignore | 3 ++- 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 .claude/settings.json diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..c824b70 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,21 @@ +{ + "permissions": { + "allow": [ + "Bash(bun run test)", + "Bash(bun run test:*)", + "Bash(bun format:test:debug:*)", + "Bash(bun lint:*)", + "Bash(bun lint:fix:*)", + "Bash(bun test-normalization.ts:*)", + "Bash(bun type-check:*)", + "Bash(awk:*)", + "Bash(ls:*)", + "WebSearch", + "Bash(tree:*)", + "Bash(bun format:*)", + "Bash(npx tsc:*)", + "Bash(node --check:*)", + "Bash(bun run:*)" + ] + } +} diff --git a/.gitignore b/.gitignore index 2230592..b1b4744 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ pnpm-lock.yaml logs tmp/ testing/ -.claude +.claude/settings.local.json +.claude/*.md From 605870d7fdb707d093820c8e7ea46f8a119bc5a6 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 4 Feb 2026 16:35:08 -0500 Subject: [PATCH 3/8] chore: Moved transformers to src/ and moved all types to src/types.ts --- src/delete/index.ts | 9 +-- src/migrate/cli.ts | 34 +++------ src/migrate/functions.ts | 14 +--- src/{migrate => }/transformers/auth0.ts | 0 src/{migrate => }/transformers/authjs.ts | 0 src/{migrate => }/transformers/clerk.ts | 0 src/{migrate => }/transformers/firebase.ts | 23 ++---- src/{migrate => }/transformers/index.ts | 0 src/{migrate => }/transformers/supabase.ts | 0 src/types.ts | 83 +++++++++++++++++++++- tests/migrate/functions.test.ts | 2 +- 11 files changed, 99 insertions(+), 66 deletions(-) rename src/{migrate => }/transformers/auth0.ts (100%) rename src/{migrate => }/transformers/authjs.ts (100%) rename src/{migrate => }/transformers/clerk.ts (100%) rename src/{migrate => }/transformers/firebase.ts (92%) rename src/{migrate => }/transformers/index.ts (100%) rename src/{migrate => }/transformers/supabase.ts (100%) diff --git a/src/delete/index.ts b/src/delete/index.ts index f88ab14..ed7c51e 100644 --- a/src/delete/index.ts +++ b/src/delete/index.ts @@ -17,6 +17,7 @@ import * as fs from 'fs'; import * as path from 'path'; import csvParser from 'csv-parser'; import pLimit from 'p-limit'; +import type { SettingsResult } from '../types'; const LIMIT = 500; const users: User[] = []; @@ -25,14 +26,6 @@ let total: number; let count = 0; let failed = 0; -/** - * Settings returned from readSettings - */ -type SettingsResult = { - file: string; - key?: string; -}; - /** * Reads the .settings file to get the migration source file path and transformer key * @returns The file path and transformer key from the migration settings diff --git a/src/migrate/cli.ts b/src/migrate/cli.ts index f065d01..a00777e 100644 --- a/src/migrate/cli.ts +++ b/src/migrate/cli.ts @@ -3,13 +3,12 @@ import color from 'picocolors'; import fs from 'fs'; import path from 'path'; import csvParser from 'csv-parser'; -import { transformers } from './transformers'; +import { transformers } from '../transformers'; import { - type FirebaseHashConfig, firebaseHashConfig, isFirebaseHashConfigComplete, setFirebaseHashConfig, -} from './transformers/firebase'; +} from '../transformers/firebase'; import { checkIfFileExists, createImportFilePath, @@ -18,15 +17,15 @@ import { tryCatch, } from '../utils'; import { env } from '../envs-constants'; +import type { + FieldAnalysis, + FirebaseHashConfig, + IdentifierCounts, + Settings, +} from '../types'; const SETTINGS_FILE = '.settings'; -type Settings = { - key?: string; - file?: string; - firebaseHashConfig?: FirebaseHashConfig; -}; - const DEV_USER_LIMIT = 500; const DASHBOARD_CONFIGURATION = color.bold( @@ -54,23 +53,6 @@ const ANALYZED_FIELDS = [ { key: 'totpSecret', label: 'TOTP Secret' }, ]; -type IdentifierCounts = { - verifiedEmails: number; - unverifiedEmails: number; - verifiedPhones: number; - unverifiedPhones: number; - username: number; - hasAnyIdentifier: number; -}; - -type FieldAnalysis = { - presentOnAll: string[]; - presentOnSome: string[]; - identifiers: IdentifierCounts; - totalUsers: number; - fieldCounts: Record; -}; - /** * Loads saved settings from the .settings file in the current directory * diff --git a/src/migrate/functions.ts b/src/migrate/functions.ts index 982fd1e..6ae365e 100644 --- a/src/migrate/functions.ts +++ b/src/migrate/functions.ts @@ -2,7 +2,7 @@ import fs from 'fs'; import csvParser from 'csv-parser'; import * as p from '@clack/prompts'; import { validationLogger } from '../logger'; -import { transformers } from './transformers'; +import { transformers } from '../transformers'; import { userSchema } from './validator'; import type { TransformerMapKeys, User } from '../types'; import { PASSWORD_HASHERS } from '../types'; @@ -13,16 +13,8 @@ import { transformKeys, } from '../utils'; -/** - * Result of a preTransform operation - * - * @property filePath - The file path to use (may be modified, e.g., temp file with headers) - * @property data - Pre-extracted user data (e.g., extracted from JSON wrapper) - */ -export type PreTransformResult = { - filePath: string; - data?: User[]; -}; +// Re-export for backwards compatibility +export type { PreTransformResult } from '../types'; const s = p.spinner(); diff --git a/src/migrate/transformers/auth0.ts b/src/transformers/auth0.ts similarity index 100% rename from src/migrate/transformers/auth0.ts rename to src/transformers/auth0.ts diff --git a/src/migrate/transformers/authjs.ts b/src/transformers/authjs.ts similarity index 100% rename from src/migrate/transformers/authjs.ts rename to src/transformers/authjs.ts diff --git a/src/migrate/transformers/clerk.ts b/src/transformers/clerk.ts similarity index 100% rename from src/migrate/transformers/clerk.ts rename to src/transformers/clerk.ts diff --git a/src/migrate/transformers/firebase.ts b/src/transformers/firebase.ts similarity index 92% rename from src/migrate/transformers/firebase.ts rename to src/transformers/firebase.ts index c85165f..03159d7 100644 --- a/src/migrate/transformers/firebase.ts +++ b/src/transformers/firebase.ts @@ -1,7 +1,9 @@ import fs from 'fs'; import path from 'path'; -import type { PreTransformResult } from '../functions'; -import type { User } from '../../types'; +import type { FirebaseHashConfig, PreTransformResult, User } from '../types'; + +// Re-export for backwards compatibility +export type { FirebaseHashConfig } from '../types'; /** * Transformer for migrating users from Firebase @@ -20,23 +22,6 @@ import type { User } from '../../types'; * - Splits displayName into firstName and lastName */ -/** - * Firebase scrypt hash configuration - * - * These values are required to verify Firebase passwords in Clerk. - * You can find them in Firebase Console: - * Authentication → Users → (⋮ menu) → Password hash parameters - * - * They can be set directly here in the transformer, or via the CLI - * which will save them to the .settings file. - */ -export type FirebaseHashConfig = { - base64_signer_key: string | undefined; - base64_salt_separator: string | undefined; - rounds: number | undefined; - mem_cost: number | undefined; -}; - /** * Hash configuration - can be set directly or via CLI * Values set here take precedence over .settings file diff --git a/src/migrate/transformers/index.ts b/src/transformers/index.ts similarity index 100% rename from src/migrate/transformers/index.ts rename to src/transformers/index.ts diff --git a/src/migrate/transformers/supabase.ts b/src/transformers/supabase.ts similarity index 100% rename from src/migrate/transformers/supabase.ts rename to src/transformers/supabase.ts diff --git a/src/types.ts b/src/types.ts index e3adf3a..b505c08 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,5 +1,5 @@ import type { ClerkAPIError } from '@clerk/types'; -import type { transformers } from './migrate/transformers'; +import type { transformers } from './transformers'; import type { userSchema } from './migrate/validator'; import * as z from 'zod'; @@ -144,3 +144,84 @@ export type DeleteLogEntry = { export const passwordHasherEnum = z.enum( PASSWORD_HASHERS as unknown as [string, ...string[]] ); + +/** + * Result of a preTransform operation + * + * @property filePath - The file path to use (may be modified, e.g., temp file with headers) + * @property data - Pre-extracted user data (e.g., extracted from JSON wrapper) + */ +export type PreTransformResult = { + filePath: string; + data?: User[]; +}; + +/** + * Firebase scrypt hash configuration + * + * These values are required to verify Firebase passwords in Clerk. + * You can find them in Firebase Console: + * Authentication → Users → (⋮ menu) → Password hash parameters + * + * They can be set directly in the transformer, or via the CLI + * which will save them to the .settings file. + */ +export type FirebaseHashConfig = { + base64_signer_key: string | undefined; + base64_salt_separator: string | undefined; + rounds: number | undefined; + mem_cost: number | undefined; +}; + +/** + * CLI settings persisted to .settings file + * + * @property key - Transformer key for the source platform + * @property file - Path to the user data file + * @property firebaseHashConfig - Firebase hash parameters (if using Firebase transformer) + */ +export type Settings = { + key?: string; + file?: string; + firebaseHashConfig?: FirebaseHashConfig; +}; + +/** + * Counts of users with each identifier type + */ +export type IdentifierCounts = { + verifiedEmails: number; + unverifiedEmails: number; + verifiedPhones: number; + unverifiedPhones: number; + username: number; + hasAnyIdentifier: number; +}; + +/** + * Analysis of user data fields for CLI display + * + * @property presentOnAll - Fields present on all users + * @property presentOnSome - Fields present on some but not all users + * @property identifiers - Counts of identifier types + * @property totalUsers - Total number of users analyzed + * @property fieldCounts - Count of users with each field + */ +export type FieldAnalysis = { + presentOnAll: string[]; + presentOnSome: string[]; + identifiers: IdentifierCounts; + totalUsers: number; + fieldCounts: Record; +}; + +/** + * Settings result from reading .settings file for deletion + * + * @property file - Path to the migration file + * @property key - Transformer key (optional) + */ +export type SettingsResult = { + file: string; + key?: string; +}; diff --git a/tests/migrate/functions.test.ts b/tests/migrate/functions.test.ts index 08d6dfd..8be83b4 100644 --- a/tests/migrate/functions.test.ts +++ b/tests/migrate/functions.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from 'vitest'; import { loadUsersFromFile } from '../../src/migrate/functions'; import { transformKeys } from '../../src/utils'; -import { transformers } from '../../src/migrate/transformers'; +import { transformers } from '../../src/transformers'; test('Clerk - loadUsersFromFile - JSON', async () => { const { users: usersFromClerk } = await loadUsersFromFile( From 331914259ac73305fcc031c66462b22be849beb5 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 4 Feb 2026 16:35:57 -0500 Subject: [PATCH 4/8] docs: Split docs to make more readable, added prompt for creasting transformer --- AGENTS.md | 136 +++++++-------- README.md | 315 ++++++---------------------------- docs/creating-transformers.md | 256 +++++++++++++++++++++++++++ docs/schema-fields.md | 82 +++++++++ docs/transformer-prompt.md | 160 +++++++++++++++++ 5 files changed, 611 insertions(+), 338 deletions(-) create mode 100644 docs/creating-transformers.md create mode 100644 docs/schema-fields.md create mode 100644 docs/transformer-prompt.md diff --git a/AGENTS.md b/AGENTS.md index c8c9099..434991a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,33 @@ This file provides guidance to AI coding assistants when working with code in th ## Overview -This is a CLI tool for migrating users from various authentication platforms (Clerk, Auth0, Supabase, AuthJS) to a Clerk instance. It handles rate limiting, validates user data with Zod schemas, and provides comprehensive logging of successes and failures. +This is a CLI tool for migrating users from various authentication platforms (Clerk, Auth0, Supabase, AuthJS, Firebase) to a Clerk instance. It handles rate limiting, validates user data with Zod schemas, and provides comprehensive logging of successes and failures. + +## Project Structure + +``` +src/ +├── clean-logs/ # Log cleanup utility +├── convert-logs/ # NDJSON to JSON converter +├── delete/ # User deletion functionality +├── migrate/ # Main migration logic +│ ├── cli.ts # Interactive CLI +│ ├── functions.ts # Data loading and transformation +│ ├── import-users.ts # User creation with Clerk API +│ ├── index.ts # Entry point +│ └── validator.ts # Zod schema validation +├── transformers/ # Platform-specific transformers +│ ├── auth0.ts +│ ├── authjs.ts +│ ├── clerk.ts +│ ├── firebase.ts +│ ├── supabase.ts +│ └── index.ts +├── envs-constants.ts # Environment configuration +├── logger.ts # NDJSON logging +├── types.ts # TypeScript types +└── utils.ts # Shared utilities +``` ## Common Commands @@ -26,6 +52,16 @@ This is a CLI tool for migrating users from various authentication platforms (Cl - `bun run test ` - Run a specific test file (e.g., `bun run test validator.test.ts`) - `bun run test --watch` - Run tests in watch mode +## After Making Changes + +Always run after code changes: + +- `bun run test` - Run all tests +- `bun lint:fix` - Fix linting issues +- `bun format` - Format code + +When adding/modifying features, add or update tests in the corresponding test files. + ## Architecture ### Transformer System @@ -40,21 +76,18 @@ The migration tool uses a **transformer pattern** to support different source pl 2. **Optional Default Fields**: Applied to all users from that platform - Example: Supabase defaults `passwordHasher` to `"bcrypt"` -3. **Optional Post-Transform**: Custom logic applied after field mapping - - Example: Auth0 converts metadata from string to objects +3. **Optional Pre-Transform**: Pre-processing before field transformation + - Example: Firebase adds CSV headers or extracts users from JSON wrapper -**Transformer locations**: `src/migrate/transformers/` +4. **Optional Post-Transform**: Custom logic applied after field mapping + - Example: Auth0 converts metadata from string to objects -- `clerk.ts` - Clerk-to-Clerk migrations -- `auth0.ts` - Auth0 migrations -- `supabase.ts` - Supabase migrations -- `authjs.ts` - AuthJS migrations -- `index.ts` - Exports all transformers as array +**Transformer locations**: `src/transformers/` **Adding a new transformer**: -1. Create a new file in `src/migrate/transformers/` with transformer config -2. Export it in `src/migrate/transformers/index.ts` +1. Create a new file in `src/transformers/` with transformer config +2. Export it in `src/transformers/index.ts` 3. The CLI will automatically include it in the platform selection ### Data Flow @@ -63,6 +96,7 @@ The migration tool uses a **transformer pattern** to support different source pl User File (CSV/JSON) ↓ loadUsersFromFile (functions.ts) + ↓ Run preTransform (if defined) ↓ Parse file ↓ Apply transformer defaults ↓ @@ -86,7 +120,7 @@ createUser (import-users.ts) User validation is centralized in `src/migrate/validator.ts`: - Uses Zod for schema validation -- Enforces: at least one verified identifier (email or phone) +- Enforces: at least one identifier (email, phone, or username) - Enforces: passwordHasher required when password is present - Fields can be single values or arrays (e.g., `email: string | string[]`) - All fields except `userId` are optional @@ -104,82 +138,30 @@ Configuration in `src/envs-constants.ts`: - `RATE_LIMIT` - Requests per second (auto-configured based on instance type) - `CONCURRENCY_LIMIT` - Number of concurrent requests (defaults to ~95% of rate limit) - - Production: 9 concurrent (assumes 100ms API latency → ~90-95 req/s throughput) - - Development: 1 concurrent (assumes 100ms API latency → ~9-10 req/s throughput) - Override defaults via `.env` file with `RATE_LIMIT` or `CONCURRENCY_LIMIT` -The script uses **p-limit for concurrency control** across all API calls: - -- Limits the number of simultaneously executing API calls -- Formula: `CONCURRENCY_LIMIT = RATE_LIMIT * 0.095` (assumes 100ms latency) -- With X concurrent requests and 100ms latency: throughput ≈ X \* 10 req/s -- Shared limiter across ALL operations (user creation, email creation, phone creation) - -**Performance**: - -- Production: ~3,500 users in ~35 seconds (assuming 1 email per user) -- Development: ~3,500 users in ~350 seconds -- Users can increase `CONCURRENCY_LIMIT` for faster processing (may hit some rate limits) +The script uses **p-limit for concurrency control** across all API calls. **Retry logic**: - If a 429 occurs, uses Retry-After value from API response - Falls back to 10 second default if Retry-After not available - Centralized in `getRetryDelay()` function in `src/utils.ts` -- The script automatically retries up to 5 times (configurable via MAX_RETRIES) +- Automatically retries up to 5 times (configurable via MAX_RETRIES) ### Logging System All operations create timestamped logs in `./logs/` using NDJSON (Newline-Delimited JSON) format: -- `{timestamp}-migration.log` - Combined log with all import entries (success, error, validation failures) +- `{timestamp}-migration.log` - Combined log with all import entries - `{timestamp}-user-deletion.log` - Combined log with all deletion entries -**Log Format**: NDJSON (Newline-Delimited JSON) +**Log Entry Types** (defined in `src/types.ts`): -- Each line is a valid JSON object -- Optimized for streaming and crash-safety -- Can append entries without rewriting entire file -- Memory-efficient for large datasets - -**Log Entry Types**: - -1. **Success Entry**: `{ userId: "user_123", status: "success", clerkUserId: "clerk_abc" }` -2. **Error Entry**: `{ userId: "user_456", status: "error", error: "Email already exists", code: "422" }` -3. **Validation Failure**: `{ userId: "user_789", status: "fail", error: "User must have at least one identifier (email, phone, or username)", path: ["email"], row: 5 }` -4. **Additional Identifier Error**: `{ type: "User Creation Error", userId: "user_abc", status: "additional_email_error", error: "..." }` (logged when adding extra emails/phones fails, but user creation succeeded) - -**Converting Logs**: - -- Use `bun convert-logs` to convert NDJSON logs to JSON arrays for easier analysis -- Converted files are saved as `{timestamp}-migration.json` or `{timestamp}-user-deletion.json` -- Useful for importing into spreadsheets or analysis tools - -**Logger functions** in `src/logger.ts`: - -- `importLogger()` - Log import attempt (success/error with optional error code) -- `errorLogger()` - Log additional identifier errors (emails/phones) and retry attempts -- `validationLogger()` - Log validation failures during data transformation -- `deleteLogger()` - Log deletion attempt (success/error with optional error code) -- `deleteErrorLogger()` - Log retry attempts during deletion - -### CLI Analysis Features - -The CLI (in `src/migrate/cli.ts`) analyzes the import file before migration and provides: - -1. **Identifier Analysis**: Shows which users have emails, phones, usernames -2. **Password Analysis**: Prompts whether to migrate users without passwords -3. **User Model Analysis**: Shows first/last name coverage -4. **Dashboard Configuration Guidance**: Tells user which fields to enable/require in Clerk Dashboard -5. **Instance Type Detection**: Prevents importing >500 users to dev instances - -**Key CLI functions**: - -- `runCLI()` - Main CLI orchestrator -- `analyzeFields()` - Analyzes user data for field coverage -- `displayIdentifierAnalysis()` - Shows identifier stats + Dashboard guidance -- `displayPasswordAnalysis()` - Shows password stats + prompts for skipPasswordRequirement -- `loadSettings()` / `saveSettings()` - Persists CLI choices in `.settings` file +- `ImportLogEntry` - Success/error for user imports +- `DeleteLogEntry` - Success/error for user deletions +- `ValidationErrorPayload` - Validation failures with path and row +- `ErrorLog` - Additional identifier errors ### Error Handling @@ -199,7 +181,7 @@ When migrating from Clerk to Clerk (`key === "clerk"`), the transformer consolid - Merges `email`, `emailAddresses`, `unverifiedEmailAddresses` into single array - Merges `phone`, `phoneNumbers`, `unverifiedPhoneNumbers` into single array - First item becomes primary, rest are added as additional identifiers -- See `transformUsers()` in `src/migrate/functions.ts` around line 129 +- See `transformUsers()` in `src/migrate/functions.ts` ### Password Hasher Validation @@ -217,7 +199,7 @@ Creating a user involves multiple API calls, all managed by the shared concurren 2. Add additional emails (each rate-limited individually, non-fatal) 3. Add additional phones (each rate-limited individually, non-fatal) -This is necessary because Clerk's API only accepts one primary identifier per creation call. All API calls share the same concurrency pool, maximizing throughput across all operations. +This is necessary because Clerk's API only accepts one primary identifier per creation call. ### Environment Variable Detection @@ -227,3 +209,9 @@ The script auto-detects instance type from `CLERK_SECRET_KEY`: - Otherwise → development - Used to set default delays and enforce user limits - See `detectInstanceType()` and `createEnvSchema()` in `src/envs-constants.ts` + +## Additional Documentation + +- [docs/schema-fields.md](docs/schema-fields.md) - Complete field reference +- [docs/creating-transformers.md](docs/creating-transformers.md) - Transformer development guide +- [docs/transformer-prompt.md](docs/transformer-prompt.md) - AI prompt for generating transformers diff --git a/README.md b/README.md index e8f16e3..b6ef943 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,21 @@ This repository contains a script that takes a JSON file as input, containing a list of users, and creates a user in Clerk using Clerk's backend API. The script respects rate limits and handles errors. +## Table of Contents + +- [Getting Started](#getting-started) +- [Migrating OAuth Connections](#migrating-oauth-connections) +- [Handle Existing User IDs and Foreign Key Constraints](#handle-existing-user-ids-and-foreign-key-constraints) +- [Configuration](#configuration) +- [Commands](#commands) +- [Convert Logs Utility](#convert-logs-utility) + +### Documentation + +- [Schema Fields Reference](docs/schema-fields.md) +- [Creating Custom Transformers](docs/creating-transformers.md) +- [AI Transformer Generation Prompt](docs/transformer-prompt.md) + ## Getting Started Clone the repository and install the dependencies. @@ -54,62 +69,13 @@ The script can be run on the same data multiple times. Clerk automatically uses bun migrate --resume-after="user_xxx" ``` -### Configuration - -The script can be configured through the following environment variables: - -| Variable | Description | -| ------------------- | ------------------------------------------------------------------------- | -| `CLERK_SECRET_KEY` | Your Clerk secret key | -| `RATE_LIMIT` | Rate limit in requests/second (auto-configured: 100 for prod, 10 for dev) | -| `CONCURRENCY_LIMIT` | Number of concurrent requests (auto-configured: ~9 for prod, ~1 for dev) | - -The script automatically detects production vs development instances from your `CLERK_SECRET_KEY` and sets appropriate rate limits and concurrency: - -- **Production** (`sk_live_*`): - - Rate limit: 100 requests/second (Clerk's limit: 1000 requests per 10 seconds) - - Concurrency: 9 concurrent requests (~95% of rate limit with 100ms API latency) - - Typical migration speed: ~3,500 users in ~35 seconds -- **Development** (`sk_test_*`): - - Rate limit: 10 requests/second (Clerk's limit: 100 requests per 10 seconds) - - Concurrency: 1 concurrent request (~95% of rate limit with 100ms API latency) - - Typical migration speed: ~3,500 users in ~350 seconds - -You can override these values by setting `RATE_LIMIT` or `CONCURRENCY_LIMIT` in your `.env` file. - -**Tuning Concurrency**: If you want faster migrations, you can increase `CONCURRENCY_LIMIT` (e.g., `CONCURRENCY_LIMIT=15` for ~150 req/s). Note that higher concurrency may trigger rate limit errors (429), which are automatically retried. - -## Other commands - -### Delete users - -```bash -bun delete -``` - -This will delete all migrated users from the instance. It should not delete pre-existing users, but it is not recommended to use this with a production instance that has pre-existing users. Please use caution with this command. - -### Clean logs - -```bash -bun clean-logs -``` - -All migrations and deletions will create logs in the `./logs` folder. This command will delete those logs. - -### Convert logs from NDJSON to JSON - -```bash -bun convert-logs -``` - -## Migrating OAuth connections +## Migrating OAuth Connections OAuth connections can not be directly migrated. The creation of the connection requires the user to consent, which can't happen on a migration like this. Instead you can rely on Clerk's [Account Linking](https://clerk.com/docs/guides/configure/auth-strategies/social-connections/account-linking) to handle this. -## Handling the Foreign Key constraint +## Handle Existing User IDs and Foreign Key Constraints -If you were using a database, you will have data tied to your previous auth system's userIDs. You will need to handle this in some way to maintain data consistency as you move to Clerk. Below are a few strategies you can use. +When migrating from another authentication system, you likely have data in your database tied to your previous system's user IDs. To maintain data consistency as you move to Clerk, you'll need a strategy to handle these foreign key relationships. Below are several approaches. ### Custom session claims @@ -150,240 +116,61 @@ You could continue to generate unique ids for the database as done previously, a You could add a column in your user table inside of your database called `ClerkId`. Use that column to store the userId from Clerk directly into your database. -## Supported Schema Fields - -The migration script validates all user data against a Zod schema defined in `src/migrate/validator.ts`. Below is a complete list of supported fields. - -### Required Fields - -| Field | Type | Description | -| -------- | -------- | ------------------------------------------------------------------ | -| `userId` | `string` | Unique identifier for the user (required for tracking and logging) | - -### Identifier Fields - -At least one verified identifier (email or phone) is required. +## Configuration -| Field | Type | Description | -| -------------------------- | -------------------- | ----------------------------------- | -| `email` | `string \| string[]` | Primary verified email address(es) | -| `emailAddresses` | `string \| string[]` | Additional verified email addresses | -| `unverifiedEmailAddresses` | `string \| string[]` | Unverified email addresses | -| `phone` | `string \| string[]` | Primary verified phone number(s) | -| `phoneNumbers` | `string \| string[]` | Additional verified phone numbers | -| `unverifiedPhoneNumbers` | `string \| string[]` | Unverified phone numbers | -| `username` | `string` | Username for the user | - -### User Information - -| Field | Type | Description | -| ----------- | -------- | ----------------- | -| `firstName` | `string` | User's first name | -| `lastName` | `string` | User's last name | - -### Password Fields - -| Field | Type | Description | -| ---------------- | -------- | ----------------------------------------------------------- | -| `password` | `string` | Hashed password from source platform | -| `passwordHasher` | `enum` | Hashing algorithm used (required when password is provided) | - -**Supported Password Hashers:** - -- `argon2i`, `argon2id` -- `bcrypt`, `bcrypt_peppered`, `bcrypt_sha256_django` -- `hmac_sha256_utf16_b64` -- `md5`, `md5_salted`, `md5_phpass` -- `pbkdf2_sha1`, `pbkdf2_sha256`, `pbkdf2_sha256_django`, `pbkdf2_sha512` -- `scrypt_firebase`, `scrypt_werkzeug` -- `sha256`, `sha256_salted`, `sha512_symfony` -- `ldap_ssha` - -### Two-Factor Authentication - -| Field | Type | Description | -| -------------------- | ---------- | -------------------------------- | -| `totpSecret` | `string` | TOTP secret for 2FA | -| `backupCodesEnabled` | `boolean` | Whether backup codes are enabled | -| `backupCodes` | `string[]` | Array of backup codes | - -### Metadata +The script can be configured through the following environment variables: -| Field | Type | Description | -| ----------------- | ----- | ------------------------------------------------------------ | -| `unsafeMetadata` | `any` | Publicly accessible metadata (readable by client and server) | -| `publicMetadata` | `any` | Publicly accessible metadata (readable by client and server) | -| `privateMetadata` | `any` | Server-side only metadata (not accessible to client) | +| Variable | Description | +| ------------------- | ------------------------------------------------------------------------- | +| `CLERK_SECRET_KEY` | Your Clerk secret key | +| `RATE_LIMIT` | Rate limit in requests/second (auto-configured: 100 for prod, 10 for dev) | +| `CONCURRENCY_LIMIT` | Number of concurrent requests (auto-configured: ~9 for prod, ~1 for dev) | -### Clerk API Configuration Fields +The script automatically detects production vs development instances from your `CLERK_SECRET_KEY` and sets appropriate rate limits and concurrency: -| Field | Type | Description | -| --------------------------- | --------- | ----------------------------------------------- | -| `bypassClientTrust` | `boolean` | Skip client trust verification | -| `createOrganizationEnabled` | `boolean` | Whether user can create organizations | -| `createOrganizationsLimit` | `number` | Maximum number of organizations user can create | -| `createdAt` | `string` | Custom creation timestamp | -| `deleteSelfEnabled` | `boolean` | Whether user can delete their own account | -| `legalAcceptedAt` | `string` | Timestamp when legal terms were accepted | -| `skipLegalChecks` | `boolean` | Skip legal acceptance checks | -| `skipPasswordChecks` | `boolean` | Skip password requirements during import | +- **Production** (`sk_live_*`): + - Rate limit: 100 requests/second (Clerk's limit: 1000 requests per 10 seconds) + - Concurrency: 9 concurrent requests (~95% of rate limit with 100ms API latency) + - Typical migration speed: ~3,500 users in ~35 seconds +- **Development** (`sk_test_*`): + - Rate limit: 10 requests/second (Clerk's limit: 100 requests per 10 seconds) + - Concurrency: 1 concurrent request (~95% of rate limit with 100ms API latency) + - Typical migration speed: ~3,500 users in ~350 seconds -## Creating a Custom Transformer +You can override these values by setting `RATE_LIMIT` or `CONCURRENCY_LIMIT` in your `.env` file. -Transformers map your source platform's user data format to Clerk's expected schema. Each transformer is defined in `src/migrate/transformers/`. +**Tuning Concurrency**: If you want faster migrations, you can increase `CONCURRENCY_LIMIT` (e.g., `CONCURRENCY_LIMIT=15` for ~150 req/s). Note that higher concurrency may trigger rate limit errors (429), which are automatically retried. -### Transformer Structure +## Commands -A transformer is an object with the following properties: +### Run migration -```typescript -{ - key: string, // Unique identifier for CLI selection - value: string, // Internal value (usually same as key) - label: string, // Display name shown in CLI - description: string, // Detailed description shown in CLI - transformer: object, // Field mapping configuration - postTransform?: function, // Optional: Custom transformation logic - defaults?: object // Optional: Default values for all users -} +```bash +bun migrate ``` -### Example: Basic Transformer - -Here's a simple transformer for a fictional platform: - -```typescript -// src/migrate/transformers/myplatform.ts -const myPlatformTransformer = { - key: 'myplatform', - value: 'myplatform', - label: 'My Platform', - description: - 'Use this transformer when migrating from My Platform. It handles standard user fields and bcrypt passwords.', - transformer: { - // Source field → Target Clerk field - user_id: 'userId', - email_address: 'email', - first: 'firstName', - last: 'lastName', - phone_number: 'phone', - hashed_password: 'password', - }, - defaults: { - passwordHasher: 'bcrypt', - }, -}; +### Delete users -export default myPlatformTransformer; +```bash +bun delete ``` -### Example: Advanced Transformer with Nested Fields - -For platforms with nested data structures: - -```typescript -const advancedTransformer = { - key: 'advanced', - value: 'advanced', - label: 'Advanced Platform', - description: - 'Use this for platforms with nested user data structures. Supports dot notation for extracting nested fields.', - transformer: { - // Supports dot notation for nested fields - 'user._id.$oid': 'userId', // Extracts user._id.$oid - 'profile.email': 'email', // Extracts profile.email - 'profile.name.first': 'firstName', - 'profile.name.last': 'lastName', - 'auth.passwordHash': 'password', - 'metadata.public': 'publicMetadata', - }, - defaults: { - passwordHasher: 'bcrypt', - }, -}; - -export default advancedTransformer; -``` +This will delete all migrated users from the instance. It should not delete pre-existing users, but it is not recommended to use this with a production instance that has pre-existing users. Please use caution with this command. -### Example: Transformer with Post-Transform Logic - -For complex transformations like handling verification status: - -```typescript -const verificationTransformer = { - key: 'verification', - value: 'verification', - label: 'Platform with Verification', - description: - 'Use this for platforms that track email verification status. Automatically routes emails to verified or unverified fields.', - transformer: { - id: 'userId', - email: 'email', - email_verified: 'emailVerified', - password_hash: 'password', - }, - postTransform: (user: Record) => { - // Route email based on verification status - const emailVerified = user.emailVerified as boolean | undefined; - const email = user.email as string | undefined; - - if (email) { - if (emailVerified === true) { - // Keep verified email in email field - user.email = email; - } else { - // Move unverified email to unverifiedEmailAddresses - user.unverifiedEmailAddresses = email; - delete user.email; - } - } - - // Clean up temporary field - delete user.emailVerified; - }, - defaults: { - passwordHasher: 'sha256', - }, -}; +### Clean logs -export default verificationTransformer; +```bash +bun clean-logs ``` -### Registering Your Transformer - -After creating your transformer file: - -1. Create the transformer file in `src/migrate/transformers/myplatform.ts` -2. Export it in `src/migrate/transformers/index.ts`: +All migrations and deletions will create logs in the `./logs` folder. This command will delete those logs. -```typescript -import clerkTransformer from './clerk'; -import auth0Transformer from './auth0'; -import supabaseTransformer from './supabase'; -import authjsTransformer from './authjs'; -import myPlatformTransformer from './myplatform'; // Add your import +### Convert logs from NDJSON to JSON -export const transformers = [ - clerkTransformer, - auth0Transformer, - supabaseTransformer, - authjsTransformer, - myPlatformTransformer, // Add to array -]; +```bash +bun convert-logs ``` -The CLI will automatically detect and display your transformer in the platform selection menu. - -### Transformer Best Practices - -1. **Field Mapping**: Map source fields to valid Clerk schema fields (see Supported Schema Fields above) -2. **Nested Fields**: Use dot notation (e.g., `'user.profile.email'`) for nested source data -3. **Verification Status**: Use `postTransform` to route emails/phones to verified or unverified arrays -4. **Password Hashers**: Always specify the correct `passwordHasher` in defaults if passwords are included -5. **Metadata**: Map platform-specific data to `publicMetadata` or `privateMetadata` -6. **Required Identifier**: Ensure at least one verified email or phone is mapped -7. **Cleanup**: Remove temporary fields in `postTransform` that aren't part of the schema - ## Convert Logs Utility Converts NDJSON (Newline-Delimited JSON) log files to standard JSON array format for easier analysis in spreadsheets, databases, or other tools. diff --git a/docs/creating-transformers.md b/docs/creating-transformers.md new file mode 100644 index 0000000..b4cb18f --- /dev/null +++ b/docs/creating-transformers.md @@ -0,0 +1,256 @@ +# Creating a Custom Transformer + +Transformers map your source platform's user data format to Clerk's expected schema. Each transformer is defined in `src/transformers/`. + +## Transformer Structure + +A transformer is an object with the following properties: + +```typescript +{ + key: string, // Unique identifier for CLI selection + value: string, // Internal value (usually same as key) + label: string, // Display name shown in CLI + description: string, // Detailed description shown in CLI + transformer: object, // Field mapping configuration + preTransform?: function, // Optional: Pre-processing before field mapping + postTransform?: function, // Optional: Custom transformation logic after mapping + defaults?: object // Optional: Default values for all users +} +``` + +## Transformer Functions + +### preTransform (Optional) + +The `preTransform` function runs before field mapping and is useful for: + +- Adding headers to CSV files that lack them +- Extracting user arrays from JSON wrapper objects +- Any preprocessing needed before the standard transformation + +**Function signature:** + +```typescript +preTransform: (filePath: string, fileType: string) => PreTransformResult; + +type PreTransformResult = { + filePath: string; // The file path to use (may be a temp file) + data?: User[]; // Pre-extracted user data (skips file parsing) +}; +``` + +**Example: Firebase preTransform** + +```typescript +preTransform: (filePath: string, fileType: string): PreTransformResult => { + if (fileType === 'text/csv') { + // Firebase CSV exports don't have headers - create temp file with headers + const originalContent = fs.readFileSync(filePath, 'utf-8'); + const newFilePath = path.join('tmp', 'users-with-headers.csv'); + fs.writeFileSync(newFilePath, `${CSV_HEADERS}\n${originalContent}`); + return { filePath: newFilePath }; + } + + if (fileType === 'application/json') { + // Firebase JSON wraps users in { users: [...] } + const parsed = JSON.parse(fs.readFileSync(filePath, 'utf-8')); + if (parsed.users && Array.isArray(parsed.users)) { + return { filePath, data: parsed.users }; + } + } + + return { filePath }; +}; +``` + +### transformer (Required) + +The `transformer` object maps source field names to Clerk schema field names: + +```typescript +transformer: { + // Source field → Target Clerk field + 'user_id': 'userId', + 'email_address': 'email', + 'first': 'firstName', + 'last': 'lastName', + // Supports dot notation for nested fields + 'user._id.$oid': 'userId', + 'profile.email': 'email', +} +``` + +### postTransform (Optional) + +The `postTransform` function runs after field mapping and is useful for: + +- Handling email/phone verification status +- Splitting combined fields (e.g., full name into first/last) +- Converting metadata formats +- Cleaning up temporary fields + +**Function signature:** + +```typescript +postTransform: (user: Record) => void +``` + +### defaults (Optional) + +Default values applied to all users: + +```typescript +defaults: { + passwordHasher: 'bcrypt', +} +``` + +## Example: Basic Transformer + +Here's a simple transformer for a fictional platform: + +```typescript +// src/transformers/myplatform.ts +const myPlatformTransformer = { + key: 'myplatform', + value: 'myplatform', + label: 'My Platform', + description: + 'Use this transformer when migrating from My Platform. It handles standard user fields and bcrypt passwords.', + transformer: { + user_id: 'userId', + email_address: 'email', + first: 'firstName', + last: 'lastName', + phone_number: 'phone', + hashed_password: 'password', + }, + defaults: { + passwordHasher: 'bcrypt', + }, +}; + +export default myPlatformTransformer; +``` + +## Example: Advanced Transformer with Nested Fields + +For platforms with nested data structures: + +```typescript +const advancedTransformer = { + key: 'advanced', + value: 'advanced', + label: 'Advanced Platform', + description: + 'Use this for platforms with nested user data structures. Supports dot notation for extracting nested fields.', + transformer: { + 'user._id.$oid': 'userId', + 'profile.email': 'email', + 'profile.name.first': 'firstName', + 'profile.name.last': 'lastName', + 'auth.passwordHash': 'password', + 'metadata.public': 'publicMetadata', + }, + defaults: { + passwordHasher: 'bcrypt', + }, +}; + +export default advancedTransformer; +``` + +## Example: Transformer with Verification Handling + +For platforms that track email verification status: + +```typescript +const verificationTransformer = { + key: 'verification', + value: 'verification', + label: 'Platform with Verification', + description: + 'Use this for platforms that track email verification status. Automatically routes emails to verified or unverified fields.', + transformer: { + id: 'userId', + email: 'email', + email_verified: 'emailVerified', + password_hash: 'password', + }, + postTransform: (user: Record) => { + const emailVerified = user.emailVerified as boolean | undefined; + const email = user.email as string | undefined; + + if (email) { + if (emailVerified === true) { + user.email = email; + } else { + user.unverifiedEmailAddresses = email; + delete user.email; + } + } + + // Clean up temporary field + delete user.emailVerified; + }, + defaults: { + passwordHasher: 'sha256', + }, +}; + +export default verificationTransformer; +``` + +## Registering Your Transformer + +After creating your transformer file: + +1. Create the transformer file in `src/transformers/myplatform.ts` +2. Export it in `src/transformers/index.ts`: + +```typescript +import clerkTransformer from './clerk'; +import auth0Transformer from './auth0'; +import supabaseTransformer from './supabase'; +import authjsTransformer from './authjs'; +import firebaseTransformer from './firebase'; +import myPlatformTransformer from './myplatform'; // Add your import + +export const transformers = [ + clerkTransformer, + auth0Transformer, + supabaseTransformer, + authjsTransformer, + firebaseTransformer, + myPlatformTransformer, // Add to array +]; +``` + +The CLI will automatically detect and display your transformer in the platform selection menu. + +## Best Practices + +1. **Field Mapping**: Map source fields to valid Clerk schema fields (see [Schema Fields Reference](schema-fields.md)) +2. **Nested Fields**: Use dot notation (e.g., `'user.profile.email'`) for nested source data +3. **Verification Status**: Use `postTransform` to route emails/phones to verified or unverified arrays +4. **Password Hashers**: Always specify the correct `passwordHasher` in defaults if passwords are included +5. **Metadata**: Map platform-specific data to `publicMetadata` or `privateMetadata` +6. **Required Identifier**: Ensure at least one identifier (email, phone, or username) is mapped +7. **Cleanup**: Remove temporary fields in `postTransform` that aren't part of the schema +8. **preTransform**: Use for file preprocessing (adding headers, extracting from wrappers) + +## Testing Your Transformer + +After creating a transformer, test it with sample data: + +```bash +# Run the migration CLI with your new transformer +bun migrate + +# Run tests to ensure validation still passes +bun run test + +# Check for linting issues +bun lint +``` diff --git a/docs/schema-fields.md b/docs/schema-fields.md new file mode 100644 index 0000000..2ee6ac0 --- /dev/null +++ b/docs/schema-fields.md @@ -0,0 +1,82 @@ +# Supported Schema Fields + +The migration script validates all user data against a Zod schema defined in `src/migrate/validator.ts`. Below is a complete list of supported fields. + +## Required Fields + +| Field | Type | Description | +| -------- | -------- | ------------------------------------------------------------------ | +| `userId` | `string` | Unique identifier for the user (required for tracking and logging) | + +## Identifier Fields + +At least one identifier (email, phone, or username) is required. + +| Field | Type | Description | +| -------------------------- | -------------------- | ----------------------------------- | +| `email` | `string \| string[]` | Primary verified email address(es) | +| `emailAddresses` | `string \| string[]` | Additional verified email addresses | +| `unverifiedEmailAddresses` | `string \| string[]` | Unverified email addresses | +| `phone` | `string \| string[]` | Primary verified phone number(s) | +| `phoneNumbers` | `string \| string[]` | Additional verified phone numbers | +| `unverifiedPhoneNumbers` | `string \| string[]` | Unverified phone numbers | +| `username` | `string` | Username for the user | + +## User Information + +| Field | Type | Description | +| ----------- | -------- | ----------------- | +| `firstName` | `string` | User's first name | +| `lastName` | `string` | User's last name | + +## Password Fields + +| Field | Type | Description | +| ---------------- | -------- | ----------------------------------------------------------- | +| `password` | `string` | Hashed password from source platform | +| `passwordHasher` | `enum` | Hashing algorithm used (required when password is provided) | + +### Supported Password Hashers + +- `argon2i`, `argon2id` +- `bcrypt`, `bcrypt_peppered`, `bcrypt_sha256_django` +- `hmac_sha256_utf16_b64` +- `md5`, `md5_salted`, `md5_phpass` +- `pbkdf2_sha1`, `pbkdf2_sha256`, `pbkdf2_sha256_django`, `pbkdf2_sha512` +- `scrypt_firebase`, `scrypt_werkzeug` +- `sha256`, `sha256_salted`, `sha512_symfony` +- `ldap_ssha` +- `awscognito` + +## Two-Factor Authentication + +| Field | Type | Description | +| -------------------- | ---------- | -------------------------------- | +| `totpSecret` | `string` | TOTP secret for 2FA | +| `backupCodesEnabled` | `boolean` | Whether backup codes are enabled | +| `backupCodes` | `string[]` | Array of backup codes | + +## Metadata + +| Field | Type | Description | +| ----------------- | ----- | ------------------------------------------------------------ | +| `unsafeMetadata` | `any` | Publicly accessible metadata (readable by client and server) | +| `publicMetadata` | `any` | Publicly accessible metadata (readable by client and server) | +| `privateMetadata` | `any` | Server-side only metadata (not accessible to client) | + +## Clerk API Configuration Fields + +| Field | Type | Description | +| --------------------------- | --------- | ----------------------------------------------- | +| `bypassClientTrust` | `boolean` | Skip client trust verification | +| `createOrganizationEnabled` | `boolean` | Whether user can create organizations | +| `createOrganizationsLimit` | `number` | Maximum number of organizations user can create | +| `createdAt` | `string` | Custom creation timestamp | +| `deleteSelfEnabled` | `boolean` | Whether user can delete their own account | +| `legalAcceptedAt` | `string` | Timestamp when legal terms were accepted | +| `skipLegalChecks` | `boolean` | Skip legal acceptance checks | +| `skipPasswordChecks` | `boolean` | Skip password requirements during import | + +## Modifying the Schema + +To add new fields to the schema, edit `userSchema` in `src/migrate/validator.ts`. diff --git a/docs/transformer-prompt.md b/docs/transformer-prompt.md new file mode 100644 index 0000000..69b59dc --- /dev/null +++ b/docs/transformer-prompt.md @@ -0,0 +1,160 @@ +# AI Prompt for Generating Custom Transformers + +Use this prompt with an AI assistant to generate a custom transformer from your sample user data. + +--- + +## Prompt Template + +Copy and paste the following prompt, replacing `[YOUR SAMPLE DATA]` with a sample of your user JSON or CSV data: + +```` +I need to create a custom transformer for the Clerk user migration script. Please analyze my sample user data and generate a transformer file. + +## Sample User Data + +[YOUR SAMPLE DATA] + +## Requirements + +1. Analyze the JSON/CSV structure to identify: + - User ID field (maps to `userId`) + - Email field(s) and verification status + - Phone field(s) and verification status + - Name fields (first name, last name, or combined name) + - Password field and hash algorithm + - Any metadata fields + +2. Generate a complete transformer file following this structure: + +```typescript +// src/transformers/[platform-name].ts +const [platformName]Transformer = { + key: '[platform-key]', + value: '[platform-key]', + label: '[Platform Name]', + description: '[Description of what this transformer handles]', + preTransform?: (filePath, fileType) => PreTransformResult, // if needed + transformer: { + // field mappings + }, + postTransform?: (user) => void, // if needed + defaults?: { + // default values + }, +}; + +export default [platformName]Transformer; +```` + +## Questions to Answer + +Before generating the transformer, please ask me about: + +1. **Email verification**: Does the data include an email verification status field? If not, should emails be treated as verified or unverified? + +2. **Phone verification**: Does the data include a phone verification status field? If not, should phones be treated as verified or unverified? + +3. **Password hasher**: If there's a password field, what hashing algorithm was used? (bcrypt, argon2, sha256, etc.) + +4. **Data preprocessing**: Does the data require any preprocessing? + - Is the JSON wrapped in an object (e.g., `{ users: [...] }` or `{ data: [...] }`)? + - Is the CSV missing headers? + - Any other preprocessing needs? + +5. **Metadata mapping**: Should any fields be mapped to `publicMetadata` or `privateMetadata`? + +After I answer these questions, generate the complete transformer file with: + +- All necessary imports +- Field mappings +- preTransform function (if data needs preprocessing) +- postTransform function (if verification handling or field splitting is needed) +- Appropriate defaults +- JSDoc comments explaining the transformer + +```` + +--- + +## Example Conversation + +**User provides sample data:** + +```json +{ + "users": [ + { + "_id": { "$oid": "507f1f77bcf86cd799439011" }, + "email": "user@example.com", + "email_confirmed": false, + "password_digest": "$2a$10$...", + "full_name": "John Doe", + "phone": "+1234567890", + "created_at": "2024-01-15T10:30:00Z", + "app_metadata": { "role": "admin" } + } + ] +} +```` + +**AI asks clarifying questions:** + +1. I see `email_confirmed: false` - should unconfirmed emails go to `unverifiedEmailAddresses`? +2. The phone field has no verification status - should it be treated as verified or unverified? +3. The `password_digest` appears to be bcrypt (`$2a$` prefix) - is that correct? +4. The data is wrapped in `{ users: [...] }` - should I add a preTransform to extract it? +5. Should `app_metadata` be mapped to `publicMetadata` or `privateMetadata`? + +**User answers:** + +1. Yes, unconfirmed emails should be unverified +2. Treat phones as verified +3. Yes, it's bcrypt +4. Yes, add preTransform +5. Map to privateMetadata + +**AI generates complete transformer with all handling.** + +--- + +## Field Reference + +When generating transformers, map to these Clerk schema fields: + +| Clerk Field | Description | +| -------------------------- | -------------------------------- | +| `userId` | Required unique identifier | +| `email` | Verified email address(es) | +| `unverifiedEmailAddresses` | Unverified email addresses | +| `phone` | Verified phone number(s) | +| `unverifiedPhoneNumbers` | Unverified phone numbers | +| `username` | Username | +| `firstName` | First name | +| `lastName` | Last name | +| `password` | Hashed password | +| `passwordHasher` | Algorithm used (set in defaults) | +| `publicMetadata` | Client-readable metadata | +| `privateMetadata` | Server-only metadata | +| `createdAt` | ISO 8601 timestamp | + +See [Schema Fields Reference](schema-fields.md) for the complete list. + +--- + +## Testing Commands + +After generating your transformer: + +```bash +# Add the transformer to src/transformers/index.ts + +# Test with the CLI +bun migrate + +# Run validation tests +bun run test + +# Check for lint errors +bun lint +``` From 1c9888ed0785acfb888a24c58d47fadf560cfb23 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 6 Feb 2026 14:21:08 -0500 Subject: [PATCH 5/8] chore: Improved tests, added banned field --- AGENTS.md | 6 +- docs/schema-fields.md | 1 + src/delete/index.ts | 58 ++++++++++++++--- src/logger.ts | 4 +- src/migrate/import-users.ts | 1 + src/migrate/validator.ts | 1 + tests/delete.test.ts | 108 ++++++++++++++++++++++++++++---- tests/migrate/validator.test.ts | 33 ++++++++++ tests/setup.ts | 14 +++++ vitest.config.ts | 1 + 10 files changed, 204 insertions(+), 23 deletions(-) create mode 100644 tests/setup.ts diff --git a/AGENTS.md b/AGENTS.md index 434991a..770935a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,14 +54,16 @@ src/ ## After Making Changes -Always run after code changes: +Always run after changes: -- `bun run test` - Run all tests - `bun lint:fix` - Fix linting issues - `bun format` - Format code +- `bun run test` - Run all tests When adding/modifying features, add or update tests in the corresponding test files. +Always perform these checks after **any** change. + ## Architecture ### Transformer System diff --git a/docs/schema-fields.md b/docs/schema-fields.md index 2ee6ac0..602656d 100644 --- a/docs/schema-fields.md +++ b/docs/schema-fields.md @@ -68,6 +68,7 @@ At least one identifier (email, phone, or username) is required. | Field | Type | Description | | --------------------------- | --------- | ----------------------------------------------- | +| `banned` | `boolean` | Whether the user is banned | | `bypassClientTrust` | `boolean` | Skip client trust verification | | `createOrganizationEnabled` | `boolean` | Whether user can create organizations | | `createOrganizationsLimit` | `number` | Maximum number of organizations user can create | diff --git a/src/delete/index.ts b/src/delete/index.ts index ed7c51e..73cbda6 100644 --- a/src/delete/index.ts +++ b/src/delete/index.ts @@ -18,6 +18,7 @@ import * as path from 'path'; import csvParser from 'csv-parser'; import pLimit from 'p-limit'; import type { SettingsResult } from '../types'; +import { transformers } from '../transformers'; const LIMIT = 500; const users: User[] = []; @@ -96,9 +97,40 @@ const FIREBASE_CSV_HEADERS = [ 'providerUserInfo', ]; +/** + * Finds the source field name that maps to 'userId' in a transformer's field mapping + * + * For example, Auth0's transformer maps `user_id` → `userId`, so this returns `'user_id'`. + * Supabase maps `id` → `userId`, so this returns `'id'`. + * + * @param transformerKey - The transformer key (e.g., 'auth0', 'firebase') + * @returns The source field name, or undefined if no transformer or mapping found + */ +export const getSourceUserIdField = ( + transformerKey?: string +): string | undefined => { + if (!transformerKey) return undefined; + const transformer = transformers.find((t) => t.key === transformerKey); + if (!transformer) return undefined; + + for (const [sourceField, targetField] of Object.entries( + transformer.transformer + )) { + if (targetField === 'userId') { + return sourceField; + } + } + return undefined; +}; + /** * Reads a migration file and extracts user IDs * Supports both JSON and CSV files + * + * Uses the transformer's field mapping to determine which source field contains the user ID. + * For example, Auth0 uses 'user_id', Supabase uses 'id', Firebase uses 'localId'. + * Falls back to checking 'userId', 'localId', and 'id' if no transformer mapping is found. + * * @param filePath - The relative path to the migration file * @param transformerKey - The transformer key used for migration (e.g., 'firebase') * @returns A Promise that resolves to a Set of user IDs from the migration file @@ -118,6 +150,9 @@ export const readMigrationFile = async ( const type = getFileType(fullPath); const userIds = new Set(); + // Resolve the source field name that maps to 'userId' in the transformer + const sourceUserIdField = getSourceUserIdField(transformerKey); + // Handle CSV files if (type === 'text/csv') { // Firebase CSV files don't have headers, so we need to provide them @@ -131,9 +166,13 @@ export const readMigrationFile = async ( return new Promise((resolve, reject) => { fs.createReadStream(fullPath) .pipe(csvParser(parserOptions)) - .on('data', (data: { id?: string; localId?: string }) => { + .on('data', (data: Record) => { + // Check transformer-mapped source field first + if (sourceUserIdField && data[sourceUserIdField]) { + userIds.add(data[sourceUserIdField]); + } // Firebase uses 'localId' for user IDs - if (data.localId) { + else if (data.localId) { userIds.add(data.localId); } // Other CSV files have 'id' column for user IDs @@ -154,8 +193,8 @@ export const readMigrationFile = async ( // Handle JSON files const fileContent = fs.readFileSync(fullPath, 'utf-8'); const parsed = JSON.parse(fileContent) as - | Array<{ userId?: string; id?: string; localId?: string }> - | { users?: Array<{ userId?: string; id?: string; localId?: string }> }; + | Array> + | { users?: Array> }; // Handle both direct array and { users: [...] } wrapper (Firebase format) const users = Array.isArray(parsed) ? parsed : parsed.users; @@ -171,16 +210,21 @@ export const readMigrationFile = async ( // Extract user IDs from the migration file for (const user of users) { + // Check transformer-mapped source field first (e.g., 'user_id' for Auth0) + const sourceValue = sourceUserIdField ? user[sourceUserIdField] : undefined; + if (typeof sourceValue === 'string' && sourceValue) { + userIds.add(sourceValue); + } // JSON files have 'userId' property - if (user.userId) { + else if (typeof user.userId === 'string' && user.userId) { userIds.add(user.userId); } // Firebase uses 'localId' for user IDs - else if (user.localId) { + else if (typeof user.localId === 'string' && user.localId) { userIds.add(user.localId); } // Also check for 'id' property as fallback - else if (user.id) { + else if (typeof user.id === 'string' && user.id) { userIds.add(user.id); } } diff --git a/src/logger.ts b/src/logger.ts index 4688011..151e0ae 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -14,9 +14,7 @@ import type { */ const confirmOrCreateFolder = (folderPath: string) => { try { - if (!fs.existsSync(folderPath)) { - fs.mkdirSync(folderPath); - } + fs.mkdirSync(folderPath, { recursive: true }); } catch (err) { // Logger infrastructure error - fallback when file system fails // eslint-disable-next-line no-console diff --git a/src/migrate/import-users.ts b/src/migrate/import-users.ts index 469f3ed..d809a22 100644 --- a/src/migrate/import-users.ts +++ b/src/migrate/import-users.ts @@ -90,6 +90,7 @@ async function createUser( userParams.publicMetadata = userData.publicMetadata; // Additional Clerk API fields + if (userData.banned !== undefined) userParams.banned = userData.banned; if (userData.bypassClientTrust !== undefined) userParams.bypassClientTrust = userData.bypassClientTrust; if (userData.createOrganizationEnabled !== undefined) diff --git a/src/migrate/validator.ts b/src/migrate/validator.ts index 2183458..f30a964 100644 --- a/src/migrate/validator.ts +++ b/src/migrate/validator.ts @@ -54,6 +54,7 @@ export const userSchema = z publicMetadata: z.any().optional(), privateMetadata: z.any().optional(), // Additional Clerk API fields + banned: z.boolean().optional(), bypassClientTrust: z.boolean().optional(), createOrganizationEnabled: z.boolean().optional(), createOrganizationsLimit: z.number().int().optional(), diff --git a/tests/delete.test.ts b/tests/delete.test.ts index cc95135..fe5ec3f 100644 --- a/tests/delete.test.ts +++ b/tests/delete.test.ts @@ -40,7 +40,7 @@ vi.mock('picocolors', () => ({ })); // Mock utils -vi.mock('../../src/utils', () => ({ +vi.mock('../src/utils', () => ({ getDateTimeStamp: vi.fn(() => '2024-01-01T12:00:00'), createImportFilePath: vi.fn((file: string) => file), getFileType: vi.fn(() => 'application/json'), @@ -66,7 +66,7 @@ vi.mock('../../src/utils', () => ({ })); // Mock env constants -vi.mock('../../src/envs-constants', () => ({ +vi.mock('../src/envs-constants', () => ({ env: { CLERK_SECRET_KEY: 'test_secret_key', RATE_LIMIT: 10, @@ -76,14 +76,28 @@ vi.mock('../../src/envs-constants', () => ({ RETRY_DELAY_MS: 10000, })); -// Mock fs module -vi.mock('fs', () => ({ - existsSync: vi.fn(), - readFileSync: vi.fn(), -})); +// Mock fs module - need both named exports and default export +vi.mock('fs', () => { + const existsSync = vi.fn(); + const readFileSync = vi.fn(); + const appendFileSync = vi.fn(); + const mkdirSync = vi.fn(); + return { + existsSync, + readFileSync, + appendFileSync, + mkdirSync, + default: { + existsSync, + readFileSync, + appendFileSync, + mkdirSync, + }, + }; +}); // Mock logger module -vi.mock('../../src/logger', () => ({ +vi.mock('../src/logger', () => ({ errorLogger: vi.fn(), importLogger: vi.fn(), deleteErrorLogger: vi.fn(), @@ -92,9 +106,12 @@ vi.mock('../../src/logger', () => ({ })); // Import after mocks are set up -import { deleteErrorLogger, deleteLogger } from '../../src/logger'; +import { deleteErrorLogger, deleteLogger } from '../src/logger'; import * as fs from 'fs'; -import { normalizeErrorMessage } from '../../src/delete/index'; +import { + getSourceUserIdField, + normalizeErrorMessage, +} from '../src/delete/index'; // Get reference to mocked functions - cast to mock type since vi.mocked is not available const _mockDeleteErrorLogger = deleteErrorLogger as ReturnType; @@ -107,6 +124,7 @@ describe('delete-users', () => { let readMigrationFile: any; let findIntersection: any; + // Get references to the mocked fs functions const mockExistsSync = fs.existsSync as ReturnType; const mockReadFileSync = fs.readFileSync as ReturnType; @@ -128,7 +146,7 @@ describe('delete-users', () => { }); // Import the module to get functions - note: vi.resetModules() is not available in Bun's Vitest - const deleteUsersModule = await import('../../src/delete/index'); + const deleteUsersModule = await import('../src/delete/index'); fetchUsers = deleteUsersModule.fetchUsers; deleteUsers = deleteUsersModule.deleteUsers; readSettings = deleteUsersModule.readSettings; @@ -519,6 +537,74 @@ describe('delete-users', () => { expect(result.has('1')).toBe(true); expect(result.has('3')).toBe(true); }); + + test('reads Auth0 JSON file using transformer-mapped user_id field', async () => { + const mockUsers = [ + { + user_id: 'auth0|abc123', + email: 'user1@example.com', + }, + { + user_id: 'google-oauth2|def456', + email: 'user2@example.com', + }, + ]; + + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockReturnValue(JSON.stringify(mockUsers)); + + const result = await readMigrationFile('samples/auth0.json', 'auth0'); + + expect(result.size).toBe(2); + expect(result.has('auth0|abc123')).toBe(true); + expect(result.has('google-oauth2|def456')).toBe(true); + }); + + test('falls back to userId/id when transformer key is not provided', async () => { + const mockUsers = [ + { user_id: 'auth0|abc123', userId: 'fallback_1' }, + { id: 'fallback_2' }, + ]; + + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockReturnValue(JSON.stringify(mockUsers)); + + const result = await readMigrationFile('samples/users.json'); + + expect(result.size).toBe(2); + expect(result.has('fallback_1')).toBe(true); + expect(result.has('fallback_2')).toBe(true); + }); + }); + + describe('getSourceUserIdField', () => { + test('returns user_id for auth0 transformer', () => { + expect(getSourceUserIdField('auth0')).toBe('user_id'); + }); + + test('returns id for clerk transformer', () => { + expect(getSourceUserIdField('clerk')).toBe('id'); + }); + + test('returns id for supabase transformer', () => { + expect(getSourceUserIdField('supabase')).toBe('id'); + }); + + test('returns id for authjs transformer', () => { + expect(getSourceUserIdField('authjs')).toBe('id'); + }); + + test('returns localId for firebase transformer', () => { + expect(getSourceUserIdField('firebase')).toBe('localId'); + }); + + test('returns undefined when no transformer key is provided', () => { + expect(getSourceUserIdField()).toBeUndefined(); + }); + + test('returns undefined for unknown transformer key', () => { + expect(getSourceUserIdField('nonexistent')).toBeUndefined(); + }); }); describe('findIntersection', () => { diff --git a/tests/migrate/validator.test.ts b/tests/migrate/validator.test.ts index 95ee83c..902d327 100644 --- a/tests/migrate/validator.test.ts +++ b/tests/migrate/validator.test.ts @@ -191,6 +191,39 @@ describe('userSchema', () => { }); expect(result.success).toBe(true); }); + + test('passes with banned true', () => { + const result = userSchema.safeParse({ + userId: 'user_123', + email: 'test@example.com', + banned: true, + }); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.banned).toBe(true); + } + }); + + test('passes with banned false', () => { + const result = userSchema.safeParse({ + userId: 'user_123', + email: 'test@example.com', + banned: false, + }); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.banned).toBe(false); + } + }); + + test('fails with banned as non-boolean', () => { + const result = userSchema.safeParse({ + userId: 'user_123', + email: 'test@example.com', + banned: 'true', + }); + expect(result.success).toBe(false); + }); }); describe('full user object', () => { diff --git a/tests/setup.ts b/tests/setup.ts new file mode 100644 index 0000000..30f0141 --- /dev/null +++ b/tests/setup.ts @@ -0,0 +1,14 @@ +import { existsSync, mkdirSync } from 'node:fs'; +import { join } from 'node:path'; + +export default function setup() { + // Set mock CLERK_SECRET_KEY if not already set (required by envs-constants.ts) + if (!process.env.CLERK_SECRET_KEY) { + process.env.CLERK_SECRET_KEY = 'sk_test_mock_key_for_testing'; + } + + const logsDir = join(process.cwd(), 'logs'); + if (!existsSync(logsDir)) { + mkdirSync(logsDir); + } +} diff --git a/vitest.config.ts b/vitest.config.ts index 9f183b8..e0000df 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,5 +3,6 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { include: ['tests/**/*.test.ts'], + globalSetup: ['tests/setup.ts'], }, }); From 605bde834ea40ca539779932e4a9f6ea85869515 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 6 Feb 2026 14:53:09 -0500 Subject: [PATCH 6/8] feat: Added CLI params so script can be run non-interactively --- README.md | 101 ++++++- docs/transformer-prompt.md | 23 ++ src/envs-constants.ts | 81 +++++- src/migrate/cli.ts | 562 ++++++++++++++++++++++++++++++++++++- src/migrate/index.ts | 18 +- 5 files changed, 762 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index b6ef943..02e4323 100644 --- a/README.md +++ b/README.md @@ -47,12 +47,33 @@ Some sample users have passwords. The password is `Kk4aPMeiaRpAs2OeX1NE`. ### Secret Key -Create a `.env` file in the root of the folder and add your `CLERK_SECRET_KEY` to it. You can find your secret key in the [Clerk dashboard](https://dashboard.clerk.dev/). +You have several options for providing your Clerk secret key: + +**Option 1: Create a `.env` file** (recommended for repeated use) ```bash CLERK_SECRET_KEY=your-secret-key ``` +**Option 2: Pass via command line** (useful for automation/AI agents) + +```bash +bun migrate --clerk-secret-key sk_test_xxx +``` + +**Option 3: Set environment variable** + +```bash +export CLERK_SECRET_KEY=sk_test_xxx +bun migrate +``` + +**Option 4: Enter interactively** + +If no key is found, the interactive CLI will prompt you to enter one and optionally save it to a `.env` file. + +You can find your secret key in the [Clerk Dashboard](https://dashboard.clerk.dev/) under **API Keys**. + ### Run the script ```bash @@ -69,6 +90,84 @@ The script can be run on the same data multiple times. Clerk automatically uses bun migrate --resume-after="user_xxx" ``` +## CLI Reference + +The migration script supports both interactive and non-interactive modes. + +### Usage + +```bash +bun migrate [OPTIONS] +``` + +### Options + +| Option | Description | +| ----------------------------- | ---------------------------------------------------------- | +| `-p, --platform ` | Source platform (clerk, auth0, authjs, firebase, supabase) | +| `-f, --file ` | Path to the user data file (JSON or CSV) | +| `-r, --resume-after ` | Resume migration after this user ID | +| `--skip-password-requirement` | Migrate users even if they don't have passwords | +| `-y, --yes` | Non-interactive mode (skip all confirmations) | +| `-h, --help` | Show help message | + +### Authentication Options + +| Option | Description | +| -------------------------- | ------------------------------------------- | +| `--clerk-secret-key ` | Clerk secret key (alternative to .env file) | + +### Firebase Options + +Required when `--platform` is `firebase`: + +| Option | Description | +| --------------------------------- | --------------------------------- | +| `--firebase-signer-key ` | Firebase hash signer key (base64) | +| `--firebase-salt-separator ` | Firebase salt separator (base64) | +| `--firebase-rounds ` | Firebase hash rounds | +| `--firebase-mem-cost ` | Firebase memory cost | + +### Examples + +```bash +# Interactive mode (default) +bun migrate + +# Non-interactive mode with required options +bun migrate -y -p auth0 -f users.json + +# Non-interactive with secret key (no .env needed) +bun migrate -y -p clerk -f users.json --clerk-secret-key sk_test_xxx + +# Resume a failed migration +bun migrate -y -p clerk -f users.json -r user_abc123 + +# Firebase migration with hash config +bun migrate -y -p firebase -f users.csv \ + --firebase-signer-key "abc123..." \ + --firebase-salt-separator "Bw==" \ + --firebase-rounds 8 \ + --firebase-mem-cost 14 +``` + +### Non-Interactive Mode + +For automation and AI agent usage, use the `-y` flag with required options: + +```bash +bun migrate -y \ + --platform clerk \ + --file users.json \ + --clerk-secret-key sk_test_xxx +``` + +**Required in non-interactive mode:** + +- `--platform` (or `-p`) +- `--file` (or `-f`) +- `CLERK_SECRET_KEY` (via `--clerk-secret-key`, environment variable, or `.env` file) + ## Migrating OAuth Connections OAuth connections can not be directly migrated. The creation of the connection requires the user to consent, which can't happen on a migration like this. Instead you can rely on Clerk's [Account Linking](https://clerk.com/docs/guides/configure/auth-strategies/social-connections/account-linking) to handle this. diff --git a/docs/transformer-prompt.md b/docs/transformer-prompt.md index 69b59dc..61213ba 100644 --- a/docs/transformer-prompt.md +++ b/docs/transformer-prompt.md @@ -11,6 +11,14 @@ Copy and paste the following prompt, replacing `[YOUR SAMPLE DATA]` with a sampl ```` I need to create a custom transformer for the Clerk user migration script. Please analyze my sample user data and generate a transformer file. +## Environment Setup + +Before we begin, please check: +1. Does a `.env` file exist in the project root? +2. If not, would you like me to help you create one with your CLERK_SECRET_KEY? + +You can find your secret key in the Clerk Dashboard under API Keys → Secret keys. + ## Sample User Data [YOUR SAMPLE DATA] @@ -73,6 +81,21 @@ After I answer these questions, generate the complete transformer file with: - Appropriate defaults - JSDoc comments explaining the transformer +## Environment Check + +Before generating the transformer, also verify: + +1. Check if a `.env` file exists in the project root +2. If it doesn't exist, ask if I want to create one +3. If yes, ask for my CLERK_SECRET_KEY and create the `.env` file with: + ``` + CLERK_SECRET_KEY= + ``` +4. If no, remind me that I'll need to either: + - Create a `.env` file manually, or + - Pass `--clerk-secret-key` when running the migration, or + - Set the CLERK_SECRET_KEY environment variable + ```` --- diff --git a/src/envs-constants.ts b/src/envs-constants.ts index feb7a92..3c73461 100644 --- a/src/envs-constants.ts +++ b/src/envs-constants.ts @@ -88,15 +88,68 @@ const envSchema = createEnvSchema(); */ export type EnvSchema = z.infer; -const parsed = envSchema.safeParse(process.env); - -if (!parsed.success) { - // Infrastructure error at module load time - occurs before CLI is initialized - // eslint-disable-next-line no-console - console.error('❌ Invalid environment variables:'); - // eslint-disable-next-line no-console - console.error(JSON.stringify(parsed.error.issues, null, 2)); - process.exit(1); +// Lazy validation - don't exit immediately, allow CLI to handle missing key +let _env: EnvSchema | null = null; +let _validationError: z.ZodError | null = null; + +/** + * Attempts to validate environment variables + * @returns true if valid, false if invalid + */ +function tryValidateEnv(): boolean { + const parsed = envSchema.safeParse(process.env); + if (parsed.success) { + _env = parsed.data; + _validationError = null; + return true; + } + _validationError = parsed.error; + return false; +} + +// Initial validation attempt +tryValidateEnv(); + +/** + * Checks if CLERK_SECRET_KEY is set in the environment + * @returns true if the key is set (even if not validated yet) + */ +export function hasClerkSecretKey(): boolean { + return !!process.env.CLERK_SECRET_KEY; +} + +/** + * Sets the CLERK_SECRET_KEY in process.env and re-validates + * @param key - The Clerk secret key to set + * @returns true if validation succeeds after setting the key + */ +export function setClerkSecretKey(key: string): boolean { + process.env.CLERK_SECRET_KEY = key; + return tryValidateEnv(); +} + +/** + * Gets the validation error if env validation failed + * @returns The Zod error or null if validation succeeded + */ +export function getEnvValidationError(): z.ZodError | null { + return _validationError; +} + +/** + * Validates environment and exits if invalid + * Call this after giving the user a chance to provide missing values + */ +export function requireValidEnv(): void { + if (!_env) { + // eslint-disable-next-line no-console + console.error('❌ Invalid environment variables:'); + if (_validationError) { + // eslint-disable-next-line no-console + console.error(JSON.stringify(_validationError.issues, null, 2)); + } + process.exit(1); + } } /** @@ -106,7 +159,15 @@ if (!parsed.success) { * @property RATE_LIMIT - Rate limit in requests per second (auto-configured based on instance type) * @property CONCURRENCY_LIMIT - Number of concurrent requests (defaults to ~95% of rate limit, can be overridden in .env) */ -export const env = parsed.data; +export const env: EnvSchema = new Proxy({} as EnvSchema, { + get(_, prop: keyof EnvSchema) { + if (!_env) { + requireValidEnv(); + } + // _env is guaranteed to be defined here since requireValidEnv exits if null + return (_env as EnvSchema)[prop]; + }, +}); /** * Maximum number of retries for rate limit (429) errors diff --git a/src/migrate/cli.ts b/src/migrate/cli.ts index a00777e..f567ca0 100644 --- a/src/migrate/cli.ts +++ b/src/migrate/cli.ts @@ -16,7 +16,12 @@ import { transformKeys as transformKeysFromFunctions, tryCatch, } from '../utils'; -import { env } from '../envs-constants'; +import { + env, + hasClerkSecretKey, + requireValidEnv, + setClerkSecretKey, +} from '../envs-constants'; import type { FieldAnalysis, FirebaseHashConfig, @@ -24,10 +29,535 @@ import type { Settings, } from '../types'; +/** + * Parsed command-line arguments for the migration script + */ +export type CLIArgs = { + platform?: string; + file?: string; + resumeAfter?: string; + skipPasswordRequirement: boolean; + nonInteractive: boolean; + help: boolean; + // Authentication + clerkSecretKey?: string; + // Firebase-specific options + firebaseSignerKey?: string; + firebaseSaltSeparator?: string; + firebaseRounds?: number; + firebaseMemCost?: number; +}; + const SETTINGS_FILE = '.settings'; const DEV_USER_LIMIT = 500; +/** + * Displays help information for the CLI + */ +function showHelp(): void { + const validPlatforms = transformers.map((t) => t.key).join(', '); + + // eslint-disable-next-line no-console + console.log(` +Clerk User Migration Utility + +USAGE: + bun migrate [OPTIONS] + +OPTIONS: + -p, --platform Source platform (${validPlatforms}) + -f, --file Path to the user data file (JSON or CSV) + -r, --resume-after Resume migration after this user ID + --skip-password-requirement Migrate users even if they don't have passwords + -y, --yes Non-interactive mode (skip all confirmations) + -h, --help Show this help message + +AUTHENTICATION: + --clerk-secret-key Clerk secret key (alternative to .env file) + Can also be set via CLERK_SECRET_KEY env var + +FIREBASE OPTIONS (required when platform is 'firebase'): + --firebase-signer-key Firebase hash signer key (base64) + --firebase-salt-separator Firebase salt separator (base64) + --firebase-rounds Firebase hash rounds + --firebase-mem-cost Firebase memory cost + +EXAMPLES: + # Interactive mode (default) + bun migrate + + # Non-interactive mode with all options + bun migrate -y -p auth0 -f users.json + + # Non-interactive with secret key (no .env needed) + bun migrate -y -p clerk -f users.json --clerk-secret-key sk_test_xxx + + # Resume a failed migration + bun migrate -y -p clerk -f users.json -r user_abc123 + + # Firebase migration with hash config + bun migrate -y -p firebase -f users.csv \\ + --firebase-signer-key "abc123..." \\ + --firebase-salt-separator "Bw==" \\ + --firebase-rounds 8 \\ + --firebase-mem-cost 14 + +ENVIRONMENT VARIABLES: + CLERK_SECRET_KEY Your Clerk secret key (required, or use --clerk-secret-key) + RATE_LIMIT Override requests per second (default: 100 prod, 10 dev) + CONCURRENCY_LIMIT Override concurrent requests (default: ~9 prod, ~1 dev) + +NOTES: + - In non-interactive mode (-y), --platform and --file are required + - Firebase migrations require all four --firebase-* options + - The script auto-detects dev/prod instance from CLERK_SECRET_KEY +`); +} + +/** + * Prompts the user to provide the CLERK_SECRET_KEY if it's missing + * + * In interactive mode, prompts the user to enter the key directly. + * In non-interactive mode, shows an error message with instructions. + * + * @param nonInteractive - Whether running in non-interactive mode + * @param cliProvidedKey - Optional key provided via --clerk-secret-key flag + * @returns true if the key was provided and validated, false otherwise + */ +async function ensureClerkSecretKey( + nonInteractive: boolean, + cliProvidedKey?: string +): Promise { + // If key was provided via CLI flag, use it + if (cliProvidedKey) { + const isValid = setClerkSecretKey(cliProvidedKey); + if (!isValid) { + if (nonInteractive) { + // eslint-disable-next-line no-console + console.error('Error: Invalid CLERK_SECRET_KEY provided.'); + } else { + p.log.error('Invalid CLERK_SECRET_KEY provided.'); + } + return false; + } + return true; + } + + if (hasClerkSecretKey()) { + requireValidEnv(); + return true; + } + + if (nonInteractive) { + // eslint-disable-next-line no-console + console.error('Error: CLERK_SECRET_KEY is not set.\n'); + // eslint-disable-next-line no-console + console.error('To fix this, either:'); + // eslint-disable-next-line no-console + console.error(' 1. Create a .env file with: CLERK_SECRET_KEY=sk_test_...'); + // eslint-disable-next-line no-console + console.error( + ' 2. Set the environment variable: export CLERK_SECRET_KEY=sk_test_...\n' + ); + // eslint-disable-next-line no-console + console.error( + 'You can find your secret key in the Clerk Dashboard under API Keys.' + ); + return false; + } + + // Interactive mode - prompt for the key + p.note( + `${color.yellow('CLERK_SECRET_KEY is not set.')}\n\n` + + `You can find your secret key in the Clerk Dashboard:\n` + + `${color.cyan('Dashboard → API Keys → Secret keys')}\n\n` + + `Alternatively, create a ${color.bold('.env')} file with:\n` + + `${color.dim('CLERK_SECRET_KEY=sk_test_...')}`, + 'Missing API Key' + ); + + const secretKey = await p.text({ + message: 'Enter your Clerk Secret Key', + placeholder: 'sk_test_... or sk_live_...', + validate: (value) => { + if (!value || value.trim() === '') { + return 'Secret key is required'; + } + if (!value.startsWith('sk_test_') && !value.startsWith('sk_live_')) { + return 'Secret key must start with sk_test_ or sk_live_'; + } + }, + }); + + if (p.isCancel(secretKey)) { + p.cancel('Migration cancelled.'); + process.exit(0); + } + + const trimmedKey = secretKey.trim(); + const isValid = setClerkSecretKey(trimmedKey); + if (!isValid) { + p.log.error('Failed to validate the secret key.'); + return false; + } + + p.log.success('Secret key validated successfully.'); + + // Ask if user wants to save the key to .env file + const envPath = path.join(process.cwd(), '.env'); + const envExists = fs.existsSync(envPath); + + const saveToEnv = await p.confirm({ + message: envExists + ? 'Would you like to add CLERK_SECRET_KEY to your existing .env file?' + : 'Would you like to create a .env file with your secret key?', + initialValue: true, + }); + + if (p.isCancel(saveToEnv)) { + // User cancelled, but key is still valid for this session + return true; + } + + if (saveToEnv) { + try { + const envLine = `CLERK_SECRET_KEY=${trimmedKey}\n`; + if (envExists) { + // Check if CLERK_SECRET_KEY already exists in the file + const existingContent = fs.readFileSync(envPath, 'utf-8'); + if (existingContent.includes('CLERK_SECRET_KEY=')) { + // Replace existing key + const updatedContent = existingContent.replace( + /CLERK_SECRET_KEY=.*/, + `CLERK_SECRET_KEY=${trimmedKey}` + ); + fs.writeFileSync(envPath, updatedContent); + p.log.success('Updated CLERK_SECRET_KEY in .env file.'); + } else { + // Append to existing file + fs.appendFileSync(envPath, envLine); + p.log.success('Added CLERK_SECRET_KEY to .env file.'); + } + } else { + // Create new .env file + fs.writeFileSync(envPath, envLine); + p.log.success('Created .env file with CLERK_SECRET_KEY.'); + } + } catch { + p.log.warn( + 'Could not save to .env file. The key will still work for this session.' + ); + } + } + + return true; +} + +/** + * Parses command-line arguments into a CLIArgs object + * + * @param argv - Array of command-line arguments (without node/bun and script path) + * @returns Parsed CLI arguments + */ +export function parseArgs(argv: string[]): CLIArgs { + const args: CLIArgs = { + skipPasswordRequirement: false, + nonInteractive: false, + help: false, + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const nextArg = argv[i + 1]; + + switch (arg) { + case '-h': + case '--help': + args.help = true; + break; + case '-y': + case '--yes': + args.nonInteractive = true; + break; + case '-p': + case '--platform': + args.platform = nextArg; + i++; + break; + case '-f': + case '--file': + args.file = nextArg; + i++; + break; + case '-r': + case '--resume-after': + args.resumeAfter = nextArg; + i++; + break; + case '--skip-password-requirement': + args.skipPasswordRequirement = true; + break; + case '--clerk-secret-key': + args.clerkSecretKey = nextArg; + i++; + break; + case '--firebase-signer-key': + args.firebaseSignerKey = nextArg; + i++; + break; + case '--firebase-salt-separator': + args.firebaseSaltSeparator = nextArg; + i++; + break; + case '--firebase-rounds': + args.firebaseRounds = parseInt(nextArg, 10); + i++; + break; + case '--firebase-mem-cost': + args.firebaseMemCost = parseInt(nextArg, 10); + i++; + break; + } + } + + return args; +} + +/** + * Validates CLI arguments for non-interactive mode + * + * @param args - Parsed CLI arguments + * @returns Error message if validation fails, null if valid + */ +function validateNonInteractiveArgs(args: CLIArgs): string | null { + if (!args.platform) { + return 'Missing required argument: --platform (-p)'; + } + + const validPlatforms = transformers.map((t) => t.key); + if (!validPlatforms.includes(args.platform)) { + return `Invalid platform: ${args.platform}. Valid options: ${validPlatforms.join(', ')}`; + } + + if (!args.file) { + return 'Missing required argument: --file (-f)'; + } + + if (!checkIfFileExists(args.file)) { + return `File not found: ${args.file}`; + } + + const fileType = getFileType(args.file); + if (fileType !== 'text/csv' && fileType !== 'application/json') { + return 'Invalid file type. Please supply a valid JSON or CSV file'; + } + + // Firebase-specific validation + if (args.platform === 'firebase') { + const hasAnyFirebaseArg = + args.firebaseSignerKey || + args.firebaseSaltSeparator || + args.firebaseRounds || + args.firebaseMemCost; + + const hasAllFirebaseArgs = + args.firebaseSignerKey && + args.firebaseSaltSeparator && + args.firebaseRounds && + args.firebaseMemCost; + + // Check if config is already set in transformer or settings + if (!isFirebaseHashConfigComplete() && !hasAllFirebaseArgs) { + const savedSettings = loadSettings(); + const savedConfig = savedSettings.firebaseHashConfig; + const hasSettingsConfig = + savedConfig && + savedConfig.base64_signer_key && + savedConfig.base64_salt_separator && + savedConfig.rounds && + savedConfig.mem_cost; + + if (!hasSettingsConfig) { + if (hasAnyFirebaseArg) { + return 'Firebase migration requires all hash config options: --firebase-signer-key, --firebase-salt-separator, --firebase-rounds, --firebase-mem-cost'; + } + return 'Firebase migration requires hash configuration. Provide all --firebase-* options or run in interactive mode to configure.'; + } + } + } + + return null; +} + +/** + * Runs the migration in non-interactive mode using CLI arguments + * + * @param args - Parsed CLI arguments + * @returns Configuration object for the migration + */ +/* eslint-disable no-console */ +export async function runNonInteractive(args: CLIArgs): Promise<{ + key: string; + file: string; + resumeAfter: string; + instance: 'dev' | 'prod'; + begin: boolean; + skipPasswordRequirement: boolean; +}> { + // Handle help flag + if (args.help) { + showHelp(); + process.exit(0); + } + + // Ensure CLERK_SECRET_KEY is set (via CLI flag, env var, or .env file) + const hasKey = await ensureClerkSecretKey(true, args.clerkSecretKey); + if (!hasKey) { + process.exit(1); + } + + // Validate arguments + const validationError = validateNonInteractiveArgs(args); + if (validationError) { + console.error(`Error: ${validationError}`); + console.error('Run "bun migrate --help" for usage information.'); + process.exit(1); + } + + // These are guaranteed to be defined after validation + const platform = args.platform as string; + const file = args.file as string; + + console.log(`\nClerk User Migration Utility (non-interactive mode)\n`); + console.log(`Platform: ${platform}`); + console.log(`File: ${file}`); + if (args.resumeAfter) { + console.log(`Resume after: ${args.resumeAfter}`); + } + + // Handle Firebase hash configuration + if (platform === 'firebase') { + if ( + args.firebaseSignerKey && + args.firebaseSaltSeparator && + args.firebaseRounds && + args.firebaseMemCost + ) { + // Use CLI-provided config + const firebaseConfig: FirebaseHashConfig = { + base64_signer_key: args.firebaseSignerKey, + base64_salt_separator: args.firebaseSaltSeparator, + rounds: args.firebaseRounds, + mem_cost: args.firebaseMemCost, + }; + setFirebaseHashConfig(firebaseConfig); + console.log('Firebase hash configuration: provided via CLI'); + } else if (!isFirebaseHashConfigComplete()) { + // Use saved settings + const savedSettings = loadSettings(); + if (savedSettings.firebaseHashConfig) { + setFirebaseHashConfig(savedSettings.firebaseHashConfig); + console.log('Firebase hash configuration: loaded from .settings'); + } + } else { + console.log('Firebase hash configuration: found in transformer'); + } + } + + // Load and analyze users + console.log('\nAnalyzing import file...'); + + const [users, error] = await tryCatch(loadRawUsers(file, platform)); + + if (error) { + console.error( + 'Failed to analyze import file. Please check the file format.' + ); + process.exit(1); + } + + // Filter users if resuming + let filteredUsers = users; + if (args.resumeAfter) { + const resumeIndex = users.findIndex((u) => u.userId === args.resumeAfter); + if (resumeIndex === -1) { + console.error( + `Could not find user ID "${args.resumeAfter}" in the import file.` + ); + process.exit(1); + } + filteredUsers = users.slice(resumeIndex + 1); + console.log( + `Resuming after user ID: ${args.resumeAfter} (skipping ${resumeIndex + 1} users)` + ); + } + + const userCount = filteredUsers.length; + console.log(`Found ${userCount} users to migrate`); + + // Check instance type + const instanceType = detectInstanceType(); + console.log(`Instance type: ${instanceType}`); + + if (instanceType === 'dev' && userCount > DEV_USER_LIMIT) { + console.error( + `Cannot import ${userCount} users to a development instance. ` + + `Development instances are limited to ${DEV_USER_LIMIT} users.` + ); + process.exit(1); + } + + // Analyze fields for validation feedback + const analysis = analyzeFields(filteredUsers); + + if (analysis.identifiers.hasAnyIdentifier === 0) { + console.error( + 'No users can be imported. All users are missing an identifier (verified email, verified phone, or username).' + ); + process.exit(1); + } + + // Check for users without identifiers + const usersWithoutIdentifier = + analysis.totalUsers - analysis.identifiers.hasAnyIdentifier; + if (usersWithoutIdentifier > 0) { + console.warn( + `Warning: ${usersWithoutIdentifier} user(s) will be skipped (missing identifier)` + ); + } + + // Determine skipPasswordRequirement + let skipPasswordRequirement = args.skipPasswordRequirement; + const usersWithPasswords = analysis.fieldCounts.password || 0; + if (usersWithPasswords === 0) { + skipPasswordRequirement = true; + } else if (usersWithPasswords < userCount && !args.skipPasswordRequirement) { + console.log( + `Note: ${userCount - usersWithPasswords} user(s) don't have passwords. ` + + `Use --skip-password-requirement to migrate them anyway.` + ); + } + + console.log('\nStarting migration...\n'); + + // Save settings for future runs + saveSettings({ + key: platform, + file, + }); + + return { + key: platform, + file, + resumeAfter: args.resumeAfter || '', + instance: instanceType, + begin: true, + skipPasswordRequirement, + }; +} +/* eslint-enable no-console */ + const DASHBOARD_CONFIGURATION = color.bold( color.whiteBright('Dashboard Configuration:\n') ); @@ -712,13 +1242,27 @@ export async function handleFirebaseHashConfig( * * Saves settings for future runs and returns all configuration options. * + * @param cliArgs - Optional CLI arguments to pre-populate values * @returns Configuration object with transformer key, file path, resumeAfter, instance type, * and skipPasswordRequirement flag * @throws Exits the process if migration is cancelled or validation fails */ -export async function runCLI() { +export async function runCLI(cliArgs?: CLIArgs) { + // Handle help flag in interactive mode + if (cliArgs?.help) { + showHelp(); + process.exit(0); + } + p.intro(`${color.bgCyan(color.black('Clerk User Migration Utility'))}`); + // Ensure CLERK_SECRET_KEY is set (via CLI flag, env var, or prompts user if missing) + const hasKey = await ensureClerkSecretKey(false, cliArgs?.clerkSecretKey); + if (!hasKey) { + p.cancel('Could not validate CLERK_SECRET_KEY.'); + process.exit(1); + } + // Load previous settings to use as defaults const savedSettings = loadSettings(); @@ -734,20 +1278,26 @@ export async function runCLI() { // Map transformers to include 'value' property for p.select (uses key as value) const selectOptions = transformers.map((t) => ({ ...t, value: t.key })); + // Use CLI args as initial values if provided + const initialPlatform = + cliArgs?.platform || savedSettings.key || transformers[0].key; + const initialFile = cliArgs?.file || savedSettings.file || 'users.json'; + const initialResumeAfter = cliArgs?.resumeAfter || ''; + const initialArgs = await p.group( { key: () => p.select({ message: 'What platform are you migrating your users from?', - initialValue: savedSettings.key || transformers[0].key, + initialValue: initialPlatform, maxItems: 1, options: selectOptions, }), file: () => p.text({ message: 'Specify the file to use for importing your users', - initialValue: savedSettings.file || 'users.json', - placeholder: savedSettings.file || 'users.json', + initialValue: initialFile, + placeholder: initialFile, validate: (value) => { if (!value) { return 'Please provide a file path'; @@ -764,7 +1314,7 @@ export async function runCLI() { resumeAfter: () => p.text({ message: 'Resume after user ID (leave empty to start from beginning)', - initialValue: '', + initialValue: initialResumeAfter, defaultValue: '', placeholder: 'user_xxx or leave empty', }), diff --git a/src/migrate/index.ts b/src/migrate/index.ts index db462c8..3a1dc36 100644 --- a/src/migrate/index.ts +++ b/src/migrate/index.ts @@ -1,6 +1,6 @@ import 'dotenv/config'; -import { runCLI } from './cli'; +import { parseArgs, runCLI, runNonInteractive } from './cli'; import { loadUsersFromFile } from './functions'; import { getLastProcessedUserId, importUsers } from './import-users'; import * as p from '@clack/prompts'; @@ -10,15 +10,21 @@ import color from 'picocolors'; * Main entry point for the user migration script * * Workflow: - * 1. Runs the CLI to gather migration parameters - * 2. Loads and transforms users from the source file - * 3. Filters users if resuming after a specific user ID - * 4. Imports users to Clerk + * 1. Parses CLI arguments to determine mode (interactive vs non-interactive) + * 2. Runs the appropriate CLI mode to gather migration parameters + * 3. Loads and transforms users from the source file + * 4. Filters users if resuming after a specific user ID + * 5. Imports users to Clerk * * @returns A promise that resolves when migration is complete */ async function main() { - const args = await runCLI(); + const cliArgs = parseArgs(process.argv.slice(2)); + + // Run in non-interactive mode if --yes flag is provided with required args + const args = cliArgs.nonInteractive + ? await runNonInteractive(cliArgs) + : await runCLI(cliArgs); // Load all users from file const { users, validationFailed } = await loadUsersFromFile( From 3f56938d2d67783498095186108df6f8ffb95b07 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 6 Feb 2026 16:05:15 -0500 Subject: [PATCH 7/8] feat: Added prompt for migrating, added skills, improved AI related docs/prompts --- .claude/skills/migrate/SKILL.md | 7 + .claude/skills/transformer/SKILL.md | 7 + .cursor/rules | 1 + .cursorrules | 2 +- .github/copilot-instructions.md | 2 +- .windsurfrules | 2 +- AGENTS.md | 3 +- CLAUDE.md | 2 +- CODEX.md | 1 + README.md | 33 ++-- prompts/migration-prompt.md | 218 ++++++++++++++++++++++++ {docs => prompts}/transformer-prompt.md | 29 ++-- src/migrate/cli.ts | 64 +++---- 13 files changed, 302 insertions(+), 69 deletions(-) create mode 100644 .claude/skills/migrate/SKILL.md create mode 100644 .claude/skills/transformer/SKILL.md create mode 120000 .cursor/rules mode change 100644 => 120000 .cursorrules mode change 100644 => 120000 .github/copilot-instructions.md mode change 100644 => 120000 .windsurfrules mode change 100644 => 120000 CLAUDE.md create mode 120000 CODEX.md create mode 100644 prompts/migration-prompt.md rename {docs => prompts}/transformer-prompt.md (86%) diff --git a/.claude/skills/migrate/SKILL.md b/.claude/skills/migrate/SKILL.md new file mode 100644 index 0000000..e717568 --- /dev/null +++ b/.claude/skills/migrate/SKILL.md @@ -0,0 +1,7 @@ +--- +name: migrate +description: Run user migration to Clerk from various authentication platforms (Auth0, Supabase, Firebase, AuthJS, Clerk). Use when user wants to import, migrate, or load users from a data file (JSON/CSV). +user-invocable: true +--- + +!`cat prompts/migration-prompt.md` diff --git a/.claude/skills/transformer/SKILL.md b/.claude/skills/transformer/SKILL.md new file mode 100644 index 0000000..6eecc29 --- /dev/null +++ b/.claude/skills/transformer/SKILL.md @@ -0,0 +1,7 @@ +--- +name: transformer +description: Generate custom Clerk user transformers from sample data. Use when user needs to create a new transformer for an unsupported platform or custom data format. +user-invocable: true +--- + +!`cat prompts/transformer-prompt.md` diff --git a/.cursor/rules b/.cursor/rules new file mode 120000 index 0000000..be77ac8 --- /dev/null +++ b/.cursor/rules @@ -0,0 +1 @@ +../AGENTS.md \ No newline at end of file diff --git a/.cursorrules b/.cursorrules deleted file mode 100644 index 43c994c..0000000 --- a/.cursorrules +++ /dev/null @@ -1 +0,0 @@ -@AGENTS.md diff --git a/.cursorrules b/.cursorrules new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/.cursorrules @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md deleted file mode 100644 index 43c994c..0000000 --- a/.github/copilot-instructions.md +++ /dev/null @@ -1 +0,0 @@ -@AGENTS.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 120000 index 0000000..be77ac8 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1 @@ +../AGENTS.md \ No newline at end of file diff --git a/.windsurfrules b/.windsurfrules deleted file mode 100644 index 43c994c..0000000 --- a/.windsurfrules +++ /dev/null @@ -1 +0,0 @@ -@AGENTS.md diff --git a/.windsurfrules b/.windsurfrules new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/.windsurfrules @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 770935a..d71a235 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -216,4 +216,5 @@ The script auto-detects instance type from `CLERK_SECRET_KEY`: - [docs/schema-fields.md](docs/schema-fields.md) - Complete field reference - [docs/creating-transformers.md](docs/creating-transformers.md) - Transformer development guide -- [docs/transformer-prompt.md](docs/transformer-prompt.md) - AI prompt for generating transformers +- [prompts/migration-prompt.md](prompts/migration-prompt.md) - AI prompt for running migrations +- [prompts/transformer-prompt.md](prompts/transformer-prompt.md) - AI prompt for generating transformers diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 43c994c..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -@AGENTS.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/CODEX.md b/CODEX.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CODEX.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/README.md b/README.md index 02e4323..d88e596 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,8 @@ This repository contains a script that takes a JSON file as input, containing a - [Schema Fields Reference](docs/schema-fields.md) - [Creating Custom Transformers](docs/creating-transformers.md) -- [AI Transformer Generation Prompt](docs/transformer-prompt.md) +- [AI Migration Prompt](prompts/migration-prompt.md) +- [AI Transformer Generation Prompt](prompts/transformer-prompt.md) ## Getting Started @@ -102,14 +103,14 @@ bun migrate [OPTIONS] ### Options -| Option | Description | -| ----------------------------- | ---------------------------------------------------------- | -| `-p, --platform ` | Source platform (clerk, auth0, authjs, firebase, supabase) | -| `-f, --file ` | Path to the user data file (JSON or CSV) | -| `-r, --resume-after ` | Resume migration after this user ID | -| `--skip-password-requirement` | Migrate users even if they don't have passwords | -| `-y, --yes` | Non-interactive mode (skip all confirmations) | -| `-h, --help` | Show help message | +| Option | Description | +| --------------------------------- | ------------------------------------------------------------- | +| `-t, --transformer ` | Source transformer (clerk, auth0, authjs, firebase, supabase) | +| `-f, --file ` | Path to the user data file (JSON or CSV) | +| `-r, --resume-after ` | Resume migration after this user ID | +| `--skip-password-requirement` | Migrate users even if they don't have passwords | +| `-y, --yes` | Non-interactive mode (skip all confirmations) | +| `-h, --help` | Show help message | ### Authentication Options @@ -119,7 +120,7 @@ bun migrate [OPTIONS] ### Firebase Options -Required when `--platform` is `firebase`: +Required when `--transformer` is `firebase`: | Option | Description | | --------------------------------- | --------------------------------- | @@ -135,16 +136,16 @@ Required when `--platform` is `firebase`: bun migrate # Non-interactive mode with required options -bun migrate -y -p auth0 -f users.json +bun migrate -y -t auth0 -f users.json # Non-interactive with secret key (no .env needed) -bun migrate -y -p clerk -f users.json --clerk-secret-key sk_test_xxx +bun migrate -y -t clerk -f users.json --clerk-secret-key sk_test_xxx # Resume a failed migration -bun migrate -y -p clerk -f users.json -r user_abc123 +bun migrate -y -t clerk -f users.json -r user_abc123 # Firebase migration with hash config -bun migrate -y -p firebase -f users.csv \ +bun migrate -y -t firebase -f users.csv \ --firebase-signer-key "abc123..." \ --firebase-salt-separator "Bw==" \ --firebase-rounds 8 \ @@ -157,14 +158,14 @@ For automation and AI agent usage, use the `-y` flag with required options: ```bash bun migrate -y \ - --platform clerk \ + --transformer clerk \ --file users.json \ --clerk-secret-key sk_test_xxx ``` **Required in non-interactive mode:** -- `--platform` (or `-p`) +- `--transformer` (or `-t`) - `--file` (or `-f`) - `CLERK_SECRET_KEY` (via `--clerk-secret-key`, environment variable, or `.env` file) diff --git a/prompts/migration-prompt.md b/prompts/migration-prompt.md new file mode 100644 index 0000000..00bef68 --- /dev/null +++ b/prompts/migration-prompt.md @@ -0,0 +1,218 @@ +# AI Prompt for Running Migrations + +Use this prompt with an AI assistant to analyze your user data file and run the migration to Clerk. + +--- + +## Prompt Template + +Copy and paste the following prompt, replacing `[YOUR FILE PATH]` with the path to your user data file: + +```` +I want to migrate users to Clerk using the migration script. Please help me import the following file: + +[YOUR FILE PATH] + +## Instructions + +1. **Analyze the file** - Read a sample of the file to understand its structure and identify the source platform. + +2. **Match to an existing transformer** - Check if the data matches one of these platforms by looking for their signature fields: + + | Platform | Signature Fields | + |----------|-----------------| + | **Supabase** | `encrypted_password`, `email_confirmed_at`, `raw_user_meta_data`, `instance_id`, `aud`, `is_sso_user` | + | **Auth0** | `user_id` (format: "provider\|id"), `email_verified` (boolean), `phone_number`, `phone_verified`, `user_metadata`, `app_metadata`, `given_name`, `family_name` | + | **Firebase** | `localId`, `passwordHash`, `passwordSalt`, `displayName`, `phoneNumber`, `disabled` | + | **Clerk** | `primary_email_address`, `verified_email_addresses`, `password_digest`, `password_hasher`, `primary_phone_number` | + | **AuthJS** | `email_verified`, `name`, `id`, `email` (minimal - may need customization) | + +3. **If a transformer matches:** + - Check if `.env` exists with `CLERK_SECRET_KEY` + - If not, ask me for the key (found in Clerk Dashboard → API Keys → Secret keys) + - Create/update the `.env` file with the key + - Tell me which transformer will be used and summarize what fields will be mapped + - Ask if I want to proceed with the migration + +4. **If no transformer matches:** + - Tell me the data doesn't match any existing transformer + - Point me to the transformer creation prompt at `prompts/transformer-prompt.md` + - List the fields found in my data so I can use them with the transformer prompt + +5. **Run the migration** (if I confirm): + ```bash + bun migrate -y --transformer [transformer-key] --file [file-path] + ``` + +## Important Notes + +- **Development instances** (`sk_test_*`): Limited to 10 requests/second +- **Production instances** (`sk_live_*`): 100 requests/second +- All operations are logged to `./logs/` with timestamps +- Failed validations are logged but don't stop the migration +- The script handles rate limiting and retries automatically +```` + +--- + +## Transformer Field Mapping Reference + +### Supabase + +``` +id → userId +email → email (routed by email_confirmed_at) +encrypted_password → password +phone → phone (routed by phone_confirmed_at) +raw_user_meta_data → publicMetadata +created_at → createdAt +``` + +Default: `passwordHasher: "bcrypt"` + +### Auth0 + +``` +user_id → userId +email → email (routed by email_verified) +username → username +given_name → firstName +family_name → lastName +phone_number → phone (routed by phone_verified) +passwordHash → password +user_metadata → publicMetadata +app_metadata → privateMetadata +created_at → createdAt +``` + +Default: `passwordHasher: "bcrypt"` + +### Firebase + +``` +localId → userId +email → email (routed by emailVerified) +passwordHash → password (combined with salt and config) +passwordSalt → (combined into password) +phoneNumber → phone +displayName → firstName + lastName (split) +``` + +Default: `passwordHasher: "scrypt_firebase"` + +**Note:** Firebase migrations require hash configuration values (signer key, salt separator, rounds, mem_cost). + +### Clerk (instance-to-instance) + +``` +id → userId +primary_email_address → email +verified_email_addresses → emailAddresses +unverified_email_addresses → unverifiedEmailAddresses +first_name → firstName +last_name → lastName +password_digest → password +password_hasher → passwordHasher +primary_phone_number → phone +verified_phone_numbers → phoneNumbers +unverified_phone_numbers → unverifiedPhoneNumbers +username → username +totp_secret → totpSecret +backup_codes_enabled → backupCodesEnabled +backup_codes → backupCodes +public_metadata → publicMetadata +unsafe_metadata → unsafeMetadata +private_metadata → privateMetadata +``` + +### AuthJS (Next-Auth) + +``` +id → userId +email → email (routed by email_verified) +name → firstName + lastName (split) +created_at → createdAt +updated_at → updatedAt +``` + +**Note:** AuthJS has no standard export format. The transformer assumes a basic SQL export. Customize as needed. + +--- + +## Example Conversations + +### Example 1: Supabase Data (Transformer Matches) + +**User:** I want to migrate `exports/users.json` + +**AI analyzes file, finds Supabase fields:** + +> Your file contains Supabase Auth data. I can see these signature fields: +> +> - `encrypted_password` (bcrypt hash) +> - `email_confirmed_at` +> - `raw_user_meta_data` +> - `instance_id` +> +> The **Supabase transformer** will map: +> +> - `id` → `userId` +> - `email` → verified/unverified based on `email_confirmed_at` +> - `encrypted_password` → `password` (with bcrypt hasher) +> - `raw_user_meta_data` → `publicMetadata` +> +> I don't see a `.env` file with `CLERK_SECRET_KEY`. Please provide your Clerk secret key. + +**User:** sk_test_abc123... + +**AI creates .env and asks:** + +> Created `.env` with your secret key. Ready to migrate with: +> +> ```bash +> bun migrate --transformer supabase --file exports/users.json +> ``` +> +> Would you like me to run the migration? + +--- + +### Example 2: Unknown Data Format (No Match) + +**User:** I want to migrate `data/custom-export.json` + +**AI analyzes file, no match:** + +> I analyzed your file and found these fields: +> +> - `user_uuid` +> - `contact_email` +> - `hashed_pw` +> - `is_email_verified` +> - `profile_data` +> +> This doesn't match any existing transformer. You'll need to create a custom one. +> +> Use the prompt in `prompts/transformer-prompt.md` with a sample of your data to generate a transformer. The AI will ask you about: +> +> 1. Email verification handling +> 2. Password hash algorithm +> 3. Metadata mapping +> +> Once created, add it to `src/transformers/index.ts` and run the migration. + +--- + +## Post-Migration + +After migration completes: + +1. **Check logs** in `./logs/` for any errors +2. **Convert logs** for easier reading: `bun convert-logs` +3. **Verify users** in your Clerk Dashboard + +To delete migrated users (uses `externalId` to identify): + +```bash +bun delete +``` diff --git a/docs/transformer-prompt.md b/prompts/transformer-prompt.md similarity index 86% rename from docs/transformer-prompt.md rename to prompts/transformer-prompt.md index 61213ba..4ef0dbc 100644 --- a/docs/transformer-prompt.md +++ b/prompts/transformer-prompt.md @@ -13,11 +13,12 @@ I need to create a custom transformer for the Clerk user migration script. Pleas ## Environment Setup -Before we begin, please check: -1. Does a `.env` file exist in the project root? -2. If not, would you like me to help you create one with your CLERK_SECRET_KEY? +Before generating the transformer, check if a `.env` file exists with `CLERK_SECRET_KEY`. If not: +1. Ask the user to provide their CLERK_SECRET_KEY (found in Clerk Dashboard → API Keys → Secret keys) +2. Create the `.env` file with the provided key +3. Continue with the transformer generation without stopping -You can find your secret key in the Clerk Dashboard under API Keys → Secret keys. +Do not ask "would you like me to create one?" - just ask for the key directly and create the file. ## Sample User Data @@ -83,18 +84,14 @@ After I answer these questions, generate the complete transformer file with: ## Environment Check -Before generating the transformer, also verify: - -1. Check if a `.env` file exists in the project root -2. If it doesn't exist, ask if I want to create one -3. If yes, ask for my CLERK_SECRET_KEY and create the `.env` file with: - ``` - CLERK_SECRET_KEY= - ``` -4. If no, remind me that I'll need to either: - - Create a `.env` file manually, or - - Pass `--clerk-secret-key` when running the migration, or - - Set the CLERK_SECRET_KEY environment variable +Before generating the transformer: + +1. Check if a `.env` file exists in the project root with `CLERK_SECRET_KEY` +2. If it doesn't exist or is missing the key, immediately ask for the CLERK_SECRET_KEY +3. Create/update the `.env` file with the provided key +4. Continue with the transformer generation + +Do not stop and wait for confirmation - just ask for the key, create the file, and proceed. ```` diff --git a/src/migrate/cli.ts b/src/migrate/cli.ts index f567ca0..ddf73ca 100644 --- a/src/migrate/cli.ts +++ b/src/migrate/cli.ts @@ -33,7 +33,7 @@ import type { * Parsed command-line arguments for the migration script */ export type CLIArgs = { - platform?: string; + transformer?: string; file?: string; resumeAfter?: string; skipPasswordRequirement: boolean; @@ -66,18 +66,18 @@ USAGE: bun migrate [OPTIONS] OPTIONS: - -p, --platform Source platform (${validPlatforms}) - -f, --file Path to the user data file (JSON or CSV) - -r, --resume-after Resume migration after this user ID - --skip-password-requirement Migrate users even if they don't have passwords - -y, --yes Non-interactive mode (skip all confirmations) - -h, --help Show this help message + -t, --transformer Source transformer (${validPlatforms}) + -f, --file Path to the user data file (JSON or CSV) + -r, --resume-after Resume migration after this user ID + --skip-password-requirement Migrate users even if they don't have passwords + -y, --yes Non-interactive mode (skip all confirmations) + -h, --help Show this help message AUTHENTICATION: --clerk-secret-key Clerk secret key (alternative to .env file) Can also be set via CLERK_SECRET_KEY env var -FIREBASE OPTIONS (required when platform is 'firebase'): +FIREBASE OPTIONS (required when transformer is 'firebase'): --firebase-signer-key Firebase hash signer key (base64) --firebase-salt-separator Firebase salt separator (base64) --firebase-rounds Firebase hash rounds @@ -88,16 +88,16 @@ EXAMPLES: bun migrate # Non-interactive mode with all options - bun migrate -y -p auth0 -f users.json + bun migrate -y -t auth0 -f users.json # Non-interactive with secret key (no .env needed) - bun migrate -y -p clerk -f users.json --clerk-secret-key sk_test_xxx + bun migrate -y -t clerk -f users.json --clerk-secret-key sk_test_xxx # Resume a failed migration - bun migrate -y -p clerk -f users.json -r user_abc123 + bun migrate -y -t clerk -f users.json -r user_abc123 # Firebase migration with hash config - bun migrate -y -p firebase -f users.csv \\ + bun migrate -y -t firebase -f users.csv \\ --firebase-signer-key "abc123..." \\ --firebase-salt-separator "Bw==" \\ --firebase-rounds 8 \\ @@ -109,7 +109,7 @@ ENVIRONMENT VARIABLES: CONCURRENCY_LIMIT Override concurrent requests (default: ~9 prod, ~1 dev) NOTES: - - In non-interactive mode (-y), --platform and --file are required + - In non-interactive mode (-y), --transformer and --file are required - Firebase migrations require all four --firebase-* options - The script auto-detects dev/prod instance from CLERK_SECRET_KEY `); @@ -280,9 +280,9 @@ export function parseArgs(argv: string[]): CLIArgs { case '--yes': args.nonInteractive = true; break; - case '-p': - case '--platform': - args.platform = nextArg; + case '-t': + case '--transformer': + args.transformer = nextArg; i++; break; case '-f': @@ -331,13 +331,13 @@ export function parseArgs(argv: string[]): CLIArgs { * @returns Error message if validation fails, null if valid */ function validateNonInteractiveArgs(args: CLIArgs): string | null { - if (!args.platform) { - return 'Missing required argument: --platform (-p)'; + if (!args.transformer) { + return 'Missing required argument: --transformer (-t)'; } - const validPlatforms = transformers.map((t) => t.key); - if (!validPlatforms.includes(args.platform)) { - return `Invalid platform: ${args.platform}. Valid options: ${validPlatforms.join(', ')}`; + const validTransformers = transformers.map((t) => t.key); + if (!validTransformers.includes(args.transformer)) { + return `Invalid transformer: ${args.transformer}. Valid options: ${validTransformers.join(', ')}`; } if (!args.file) { @@ -354,7 +354,7 @@ function validateNonInteractiveArgs(args: CLIArgs): string | null { } // Firebase-specific validation - if (args.platform === 'firebase') { + if (args.transformer === 'firebase') { const hasAnyFirebaseArg = args.firebaseSignerKey || args.firebaseSaltSeparator || @@ -426,18 +426,18 @@ export async function runNonInteractive(args: CLIArgs): Promise<{ } // These are guaranteed to be defined after validation - const platform = args.platform as string; + const transformer = args.transformer as string; const file = args.file as string; console.log(`\nClerk User Migration Utility (non-interactive mode)\n`); - console.log(`Platform: ${platform}`); + console.log(`Transformer: ${transformer}`); console.log(`File: ${file}`); if (args.resumeAfter) { console.log(`Resume after: ${args.resumeAfter}`); } // Handle Firebase hash configuration - if (platform === 'firebase') { + if (transformer === 'firebase') { if ( args.firebaseSignerKey && args.firebaseSaltSeparator && @@ -468,7 +468,7 @@ export async function runNonInteractive(args: CLIArgs): Promise<{ // Load and analyze users console.log('\nAnalyzing import file...'); - const [users, error] = await tryCatch(loadRawUsers(file, platform)); + const [users, error] = await tryCatch(loadRawUsers(file, transformer)); if (error) { console.error( @@ -543,12 +543,12 @@ export async function runNonInteractive(args: CLIArgs): Promise<{ // Save settings for future runs saveSettings({ - key: platform, + key: transformer, file, }); return { - key: platform, + key: transformer, file, resumeAfter: args.resumeAfter || '', instance: instanceType, @@ -1279,8 +1279,8 @@ export async function runCLI(cliArgs?: CLIArgs) { const selectOptions = transformers.map((t) => ({ ...t, value: t.key })); // Use CLI args as initial values if provided - const initialPlatform = - cliArgs?.platform || savedSettings.key || transformers[0].key; + const initialTransformer = + cliArgs?.transformer || savedSettings.key || transformers[0].key; const initialFile = cliArgs?.file || savedSettings.file || 'users.json'; const initialResumeAfter = cliArgs?.resumeAfter || ''; @@ -1288,8 +1288,8 @@ export async function runCLI(cliArgs?: CLIArgs) { { key: () => p.select({ - message: 'What platform are you migrating your users from?', - initialValue: initialPlatform, + message: 'Which transformer should be used for your user data?', + initialValue: initialTransformer, maxItems: 1, options: selectOptions, }), From 4c84741a5c64a46b06bcbb641b16c454ceebd049 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 6 Feb 2026 17:40:20 -0500 Subject: [PATCH 8/8] chore: Improve prompts, CLI, tests --- README.md | 16 ++-- prompts/migration-prompt.md | 88 +++++++++++------ prompts/transformer-prompt.md | 57 +++++++++-- src/delete/index.ts | 8 ++ src/migrate/cli.ts | 14 ++- src/migrate/import-users.ts | 4 +- tests/delete.test.ts | 26 ++++- tests/transformers.test.ts | 174 ++++++++++++++++++++++++++++++++++ 8 files changed, 329 insertions(+), 58 deletions(-) create mode 100644 tests/transformers.test.ts diff --git a/README.md b/README.md index d88e596..c9a2052 100644 --- a/README.md +++ b/README.md @@ -103,14 +103,14 @@ bun migrate [OPTIONS] ### Options -| Option | Description | -| --------------------------------- | ------------------------------------------------------------- | -| `-t, --transformer ` | Source transformer (clerk, auth0, authjs, firebase, supabase) | -| `-f, --file ` | Path to the user data file (JSON or CSV) | -| `-r, --resume-after ` | Resume migration after this user ID | -| `--skip-password-requirement` | Migrate users even if they don't have passwords | -| `-y, --yes` | Non-interactive mode (skip all confirmations) | -| `-h, --help` | Show help message | +| Option | Description | +| --------------------------------- | ---------------------------------------------------------------------------------------- | +| `-t, --transformer ` | Source transformer (clerk, auth0, authjs, firebase, supabase) | +| `-f, --file ` | Path to the user data file (JSON or CSV) | +| `-r, --resume-after ` | Resume migration after this user ID | +| `--require-password` | Only migrate users who have passwords (by default, users without passwords are migrated) | +| `-y, --yes` | Non-interactive mode (skip all confirmations) | +| `-h, --help` | Show help message | ### Authentication Options diff --git a/prompts/migration-prompt.md b/prompts/migration-prompt.md index 00bef68..c893310 100644 --- a/prompts/migration-prompt.md +++ b/prompts/migration-prompt.md @@ -15,42 +15,70 @@ I want to migrate users to Clerk using the migration script. Please help me impo ## Instructions -1. **Analyze the file** - Read a sample of the file to understand its structure and identify the source platform. - -2. **Match to an existing transformer** - Check if the data matches one of these platforms by looking for their signature fields: - - | Platform | Signature Fields | - |----------|-----------------| - | **Supabase** | `encrypted_password`, `email_confirmed_at`, `raw_user_meta_data`, `instance_id`, `aud`, `is_sso_user` | - | **Auth0** | `user_id` (format: "provider\|id"), `email_verified` (boolean), `phone_number`, `phone_verified`, `user_metadata`, `app_metadata`, `given_name`, `family_name` | - | **Firebase** | `localId`, `passwordHash`, `passwordSalt`, `displayName`, `phoneNumber`, `disabled` | - | **Clerk** | `primary_email_address`, `verified_email_addresses`, `password_digest`, `password_hasher`, `primary_phone_number` | - | **AuthJS** | `email_verified`, `name`, `id`, `email` (minimal - may need customization) | - -3. **If a transformer matches:** - - Check if `.env` exists with `CLERK_SECRET_KEY` - - If not, ask me for the key (found in Clerk Dashboard → API Keys → Secret keys) - - Create/update the `.env` file with the key - - Tell me which transformer will be used and summarize what fields will be mapped - - Ask if I want to proceed with the migration - -4. **If no transformer matches:** - - Tell me the data doesn't match any existing transformer - - Point me to the transformer creation prompt at `prompts/transformer-prompt.md` - - List the fields found in my data so I can use them with the transformer prompt - -5. **Run the migration** (if I confirm): +Follow these steps EXACTLY in order. Do NOT skip any steps. + +### Step 1: Verify Environment + +Before doing ANYTHING else: +1. Check if `.env` file exists with `CLERK_SECRET_KEY` +2. If missing, IMMEDIATELY ask for the key (Clerk Dashboard → API Keys → Secret keys, or https://dashboard.clerk.com/~/api-keys) +3. Create/update the `.env` file with the provided key +4. Do NOT proceed until the key is configured + +### Step 2: Analyze the Data File + +Read a sample of the file to understand its structure. Look for signature fields that identify the source platform: + +| Platform | Signature Fields | +|----------|-----------------| +| **Supabase** | `encrypted_password`, `email_confirmed_at`, `raw_user_meta_data`, `instance_id`, `aud`, `is_sso_user` | +| **Auth0** | `user_id` (format: "provider\|id"), `email_verified` (boolean), `phone_number`, `phone_verified`, `user_metadata`, `app_metadata`, `given_name`, `family_name` | +| **Firebase** | `localId`, `passwordHash`, `passwordSalt`, `displayName`, `phoneNumber`, `disabled` | +| **Clerk** | `primary_email_address`, `verified_email_addresses`, `password_digest`, `password_hasher`, `primary_phone_number` | +| **AuthJS** | `email_verified`, `name`, `id`, `email` (minimal - may need customization) | + +### Step 3A: If a Transformer Matches + +1. Tell me which transformer will be used +2. Summarize the field mappings that will be applied +3. Ask if I want to proceed with the migration +4. If confirmed, run: ```bash bun migrate -y --transformer [transformer-key] --file [file-path] ``` -## Important Notes +### Step 3B: If NO Transformer Matches - CRITICAL STEPS + +If the data doesn't match any existing transformer, you MUST: -- **Development instances** (`sk_test_*`): Limited to 10 requests/second +1. **Inform the user**: Explain that no existing transformer matches their data format +2. **List the fields found**: Show all fields discovered in their data file +3. **Create a custom transformer**: Generate a transformer file at `src/transformers/[platform-name].ts` using the 'transformer' skill or the `prompts/transformer-prompt.md` +4. **MANDATORY - Register the transformer**: + - Add an import to `src/transformers/index.ts` + - Add the transformer to the `transformers` array + + **THIS STEP IS NOT OPTIONAL.** If you skip registration: + - The transformer will NOT appear in the CLI's platform selection + - The `bun delete` command will NOT find migrated users + - Users will see "Found 0 migrated users to delete" after migration + +5. **Run tests**: Execute `bun run test` to verify the transformer is properly registered +6. **Run the migration**: After tests pass, run the migration command + +### Step 4: Post-Migration Verification + +After migration completes: +1. Report the number of users successfully migrated +2. Report any failures or validation errors from the logs +3. Remind the user they can run `bun delete` to remove migrated users if needed + +## Rate Limits + +- **Development instances** (`sk_test_*`): 10 requests/second - **Production instances** (`sk_live_*`): 100 requests/second -- All operations are logged to `./logs/` with timestamps -- Failed validations are logged but don't stop the migration -- The script handles rate limiting and retries automatically + +The script handles rate limiting and retries automatically. All operations are logged to `./logs/`. ```` --- diff --git a/prompts/transformer-prompt.md b/prompts/transformer-prompt.md index 4ef0dbc..f5d34bb 100644 --- a/prompts/transformer-prompt.md +++ b/prompts/transformer-prompt.md @@ -26,7 +26,12 @@ Do not ask "would you like me to create one?" - just ask for the key directly an ## Requirements -1. Analyze the JSON/CSV structure to identify: +1. Before beginning work, check if CLERK_SECRET_KEY is present. + - Check if `.env` exists with `CLERK_SECRET_KEY` + - If not, ask me for the key (found in Clerk Dashboard → API Keys → Secret keys, or https://dashboard.clerk.com/~/api-keys) + - Create/update the `.env` file with the key + +2. Analyze the JSON/CSV structure to identify: - User ID field (maps to `userId`) - Email field(s) and verification status - Phone field(s) and verification status @@ -34,7 +39,7 @@ Do not ask "would you like me to create one?" - just ask for the key directly an - Password field and hash algorithm - Any metadata fields -2. Generate a complete transformer file following this structure: +3. Generate a complete transformer file following this structure: ```typescript // src/transformers/[platform-name].ts @@ -54,6 +59,27 @@ const [platformName]Transformer = { }; export default [platformName]Transformer; +``` + +4. **CRITICAL - Register the transformer**: After creating the transformer file, you MUST register it in `src/transformers/index.ts`. This is NOT optional. The migration and delete commands will fail silently if the transformer is not registered. + + Add both an import and include it in the exports array: + + ```typescript + // Add import at the top + import customTransformer from './custom'; + + // Add to the transformers array + export const transformers = [ + // ... existing transformers + customTransformer, // ADD YOUR TRANSFORMER HERE + ]; + ``` + + **WARNING**: If you skip this step: + - The transformer will NOT appear in the CLI's platform selection + - The `bun delete` command will NOT be able to find migrated users + - Users will see "Found 0 migrated users to delete" even after successful migration ```` ## Questions to Answer @@ -162,19 +188,32 @@ See [Schema Fields Reference](schema-fields.md) for the complete list. --- -## Testing Commands +## Post-Generation Checklist -After generating your transformer: +After generating your transformer, verify these steps were completed: -```bash -# Add the transformer to src/transformers/index.ts +### 1. Transformer File Created -# Test with the CLI -bun migrate +- [ ] File exists at `src/transformers/[platform-name].ts` +- [ ] Has a default export with `key`, `value`, `label`, `description`, and `transformer` fields +- [ ] The `transformer` object maps source fields to Clerk fields (including a field that maps to `userId`) + +### 2. Transformer Registered (CRITICAL) + +- [ ] Import added to `src/transformers/index.ts` +- [ ] Transformer added to the `transformers` array export + +**If you skip registration, the delete command will fail to find migrated users!** -# Run validation tests +### 3. Validate the Setup + +```bash +# Run tests to verify transformer is properly registered bun run test # Check for lint errors bun lint + +# Test the migration CLI (should show your new platform in the list) +bun migrate ``` diff --git a/src/delete/index.ts b/src/delete/index.ts index 73cbda6..531d1af 100644 --- a/src/delete/index.ts +++ b/src/delete/index.ts @@ -171,6 +171,10 @@ export const readMigrationFile = async ( if (sourceUserIdField && data[sourceUserIdField]) { userIds.add(data[sourceUserIdField]); } + // Common field name for user IDs (custom transformers) + else if (data.user_id) { + userIds.add(data.user_id); + } // Firebase uses 'localId' for user IDs else if (data.localId) { userIds.add(data.localId); @@ -219,6 +223,10 @@ export const readMigrationFile = async ( else if (typeof user.userId === 'string' && user.userId) { userIds.add(user.userId); } + // Common field name for user IDs (custom transformers) + else if (typeof user.user_id === 'string' && user.user_id) { + userIds.add(user.user_id); + } // Firebase uses 'localId' for user IDs else if (typeof user.localId === 'string' && user.localId) { userIds.add(user.localId); diff --git a/src/migrate/cli.ts b/src/migrate/cli.ts index ddf73ca..4d6b237 100644 --- a/src/migrate/cli.ts +++ b/src/migrate/cli.ts @@ -69,7 +69,7 @@ OPTIONS: -t, --transformer Source transformer (${validPlatforms}) -f, --file Path to the user data file (JSON or CSV) -r, --resume-after Resume migration after this user ID - --skip-password-requirement Migrate users even if they don't have passwords + --require-password Only migrate users who have passwords (default: false) -y, --yes Non-interactive mode (skip all confirmations) -h, --help Show this help message @@ -262,7 +262,7 @@ async function ensureClerkSecretKey( */ export function parseArgs(argv: string[]): CLIArgs { const args: CLIArgs = { - skipPasswordRequirement: false, + skipPasswordRequirement: true, nonInteractive: false, help: false, }; @@ -296,8 +296,12 @@ export function parseArgs(argv: string[]): CLIArgs { i++; break; case '--skip-password-requirement': + // Legacy flag, kept for backwards compatibility (now default) args.skipPasswordRequirement = true; break; + case '--require-password': + args.skipPasswordRequirement = false; + break; case '--clerk-secret-key': args.clerkSecretKey = nextArg; i++; @@ -532,10 +536,10 @@ export async function runNonInteractive(args: CLIArgs): Promise<{ const usersWithPasswords = analysis.fieldCounts.password || 0; if (usersWithPasswords === 0) { skipPasswordRequirement = true; - } else if (usersWithPasswords < userCount && !args.skipPasswordRequirement) { + } else if (usersWithPasswords < userCount && skipPasswordRequirement) { console.log( - `Note: ${userCount - usersWithPasswords} user(s) don't have passwords. ` + - `Use --skip-password-requirement to migrate them anyway.` + `Note: ${userCount - usersWithPasswords} user(s) don't have passwords and will be migrated. ` + + `Use --require-password to skip them.` ); } diff --git a/src/migrate/import-users.ts b/src/migrate/import-users.ts index d809a22..d296979 100644 --- a/src/migrate/import-users.ts +++ b/src/migrate/import-users.ts @@ -436,13 +436,13 @@ function displaySummary(summary: ImportSummary) { * Logs all results to timestamped log files. * * @param users - Array of validated users to import - * @param skipPasswordRequirement - Whether to allow users without passwords (default: false) + * @param skipPasswordRequirement - Whether to allow users without passwords (default: true) * @param validationFailed - Number of users that failed validation (default: 0) * @returns A promise that resolves when all users are processed */ export async function importUsers( users: User[], - skipPasswordRequirement: boolean = false, + skipPasswordRequirement: boolean = true, validationFailed: number = 0 ) { const dateTime = getDateTimeStamp(); diff --git a/tests/delete.test.ts b/tests/delete.test.ts index fe5ec3f..5def214 100644 --- a/tests/delete.test.ts +++ b/tests/delete.test.ts @@ -561,10 +561,7 @@ describe('delete-users', () => { }); test('falls back to userId/id when transformer key is not provided', async () => { - const mockUsers = [ - { user_id: 'auth0|abc123', userId: 'fallback_1' }, - { id: 'fallback_2' }, - ]; + const mockUsers = [{ userId: 'fallback_1' }, { id: 'fallback_2' }]; mockExistsSync.mockReturnValue(true); mockReadFileSync.mockReturnValue(JSON.stringify(mockUsers)); @@ -575,6 +572,27 @@ describe('delete-users', () => { expect(result.has('fallback_1')).toBe(true); expect(result.has('fallback_2')).toBe(true); }); + + test('user_id takes precedence over id but not userId in fallback chain', async () => { + const mockUsers = [ + { userId: 'primary', user_id: 'secondary', id: 'tertiary' }, + { user_id: 'secondary_only', id: 'tertiary' }, + { id: 'tertiary_only' }, + ]; + + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockReturnValue(JSON.stringify(mockUsers)); + + const result = await readMigrationFile('samples/users.json'); + + expect(result.size).toBe(3); + // userId takes precedence + expect(result.has('primary')).toBe(true); + // user_id is used when no userId + expect(result.has('secondary_only')).toBe(true); + // id is used as last resort + expect(result.has('tertiary_only')).toBe(true); + }); }); describe('getSourceUserIdField', () => { diff --git a/tests/transformers.test.ts b/tests/transformers.test.ts new file mode 100644 index 0000000..e519f63 --- /dev/null +++ b/tests/transformers.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, test } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; + +/** + * Tests to ensure all transformers are properly registered in index.ts + * + * This prevents a common issue where a custom transformer is created but not + * registered, causing the delete command to fail to find migrated users. + */ + +const TRANSFORMERS_DIR = path.join(process.cwd(), 'src/transformers'); +const INDEX_FILE = path.join(TRANSFORMERS_DIR, 'index.ts'); + +/** + * Gets all transformer files (excluding index.ts) + */ +function getTransformerFiles(): string[] { + const files = fs.readdirSync(TRANSFORMERS_DIR); + return files + .filter((file) => file.endsWith('.ts') && file !== 'index.ts') + .map((file) => file.replace('.ts', '')); +} + +describe('transformer registration', () => { + const transformerFiles = getTransformerFiles(); + const indexContent = fs.readFileSync(INDEX_FILE, 'utf-8'); + + test('should have at least one transformer file', () => { + expect(transformerFiles.length).toBeGreaterThan(0); + }); + + describe('each transformer file has a default export', () => { + test.each(transformerFiles)('%s has a default export', (fileName) => { + const filePath = path.join(TRANSFORMERS_DIR, `${fileName}.ts`); + const content = fs.readFileSync(filePath, 'utf-8'); + + // Check for default export pattern + const hasDefaultExport = + /export\s+default\s+\w+/.test(content) || + /export\s*\{\s*\w+\s+as\s+default\s*\}/.test(content); + + expect( + hasDefaultExport, + `${fileName}.ts must have a default export` + ).toBe(true); + }); + }); + + describe('each transformer file is imported in index.ts', () => { + test.each(transformerFiles)('%s is imported in index.ts', (fileName) => { + // Check for import statement - handles various import patterns + // e.g., import auth0Transformer from './auth0'; + // e.g., import customTransformer from './custom'; + const importPattern = new RegExp( + `import\\s+\\w+\\s+from\\s+['"]\\.\\/${fileName}['"]` + ); + + expect( + importPattern.test(indexContent), + `${fileName} must be imported in index.ts. Add: import ${fileName}Transformer from './${fileName}';` + ).toBe(true); + }); + }); + + describe('each transformer is exported in the transformers array', () => { + test.each(transformerFiles)( + '%s transformer is in the exports array', + (fileName) => { + // Extract the transformer key from the file content + const filePath = path.join(TRANSFORMERS_DIR, `${fileName}.ts`); + const content = fs.readFileSync(filePath, 'utf-8'); + + // Extract the key value from the transformer definition + const keyMatch = content.match(/key:\s*['"]([^'"]+)['"]/); + expect( + keyMatch, + `${fileName}.ts must have a key property` + ).not.toBeNull(); + + // After the expect, we know keyMatch is not null + const transformerKey = keyMatch ? keyMatch[1] : ''; + + // Check that the variable name used in the file is exported in index.ts + // The transformer should be imported and added to the array + const variableMatch = content.match( + /const\s+(\w+)\s*=\s*\{[\s\S]*?key:\s*['"]/ + ); + expect( + variableMatch, + `${fileName}.ts must define a transformer constant` + ).not.toBeNull(); + + // After the expect, we know variableMatch is not null + const variableName = variableMatch ? variableMatch[1] : ''; + + // Check the variable is in the transformers array in index.ts + const arrayPattern = new RegExp(`\\b${variableName}\\b`); + expect( + arrayPattern.test(indexContent), + `Transformer "${variableName}" with key "${transformerKey}" from ${fileName}.ts must be added to the transformers array in index.ts` + ).toBe(true); + } + ); + }); + + test('transformers array has correct number of entries', () => { + // Count the number of imports in index.ts (excluding type imports) + const importMatches = indexContent.match( + /import\s+\w+\s+from\s+['"]\.\/\w+['"]/g + ); + const importCount = importMatches ? importMatches.length : 0; + + expect( + importCount, + `Expected ${transformerFiles.length} transformer imports but found ${importCount}. ` + + `Make sure all transformer files are imported in index.ts` + ).toBe(transformerFiles.length); + }); + + describe('each transformer has required properties', () => { + test.each(transformerFiles)( + '%s has key, label, description, and transformer fields', + (fileName) => { + const filePath = path.join(TRANSFORMERS_DIR, `${fileName}.ts`); + const content = fs.readFileSync(filePath, 'utf-8'); + + // Check for key property + expect( + /key:\s*['"][^'"]+['"]/.test(content), + `${fileName}.ts must have a key property` + ).toBe(true); + + // Check for label property + expect( + /label:\s*['"][^'"]+['"]/.test(content), + `${fileName}.ts must have a label property` + ).toBe(true); + + // Check for description property + expect( + /description:\s*/.test(content), + `${fileName}.ts must have a description property` + ).toBe(true); + + // Check for transformer property (object with field mappings) + expect( + /transformer:\s*\{/.test(content), + `${fileName}.ts must have a transformer property with field mappings` + ).toBe(true); + } + ); + }); + + describe('each transformer maps a field to userId', () => { + test.each(transformerFiles)( + '%s transformer maps a source field to userId', + (fileName) => { + const filePath = path.join(TRANSFORMERS_DIR, `${fileName}.ts`); + const content = fs.readFileSync(filePath, 'utf-8'); + + // Check that a field maps to 'userId' + // Patterns: 'field': 'userId' or field: 'userId' + const mapsToUserId = /['"]?\w+['"]?\s*:\s*['"]userId['"]/.test(content); + + expect( + mapsToUserId, + `Transformer ${fileName}.ts must map a source field to 'userId'. ` + + `This is required for the delete command to identify migrated users.` + ).toBe(true); + } + ); + }); +});