Skip to content

Commit 327df56

Browse files
mainquegmainqueg
committed
🚧 server: kyc account migration
1 parent 907ade0 commit 327df56

5 files changed

Lines changed: 519 additions & 23 deletions

File tree

‎server/script/kyc-migration.ts‎

Lines changed: 304 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,304 @@
1+
import createDebug from "debug";
2+
import { inspect } from "node:util";
3+
import { safeParse, ValiError, type InferOutput } from "valibot";
4+
5+
import * as persona from "../utils/persona";
6+
import { buildIssueMessages } from "../utils/validatorHook";
7+
8+
const BATCH_SIZE = 10;
9+
10+
const debug = createDebug("migration:debug");
11+
const log = createDebug("migration:log");
12+
const warn = createDebug("migration:warn");
13+
const unexpected = createDebug("migration:unexpected");
14+
15+
let reference: string | undefined;
16+
let all = false;
17+
let onlyLogs = false;
18+
let initialNext: string | undefined;
19+
let onlyPandaTemplates = false;
20+
21+
const options = process.argv.slice(2);
22+
for (const option of options) {
23+
switch (true) {
24+
case option.startsWith("--reference-id="):
25+
reference = option.split("=")[1];
26+
break;
27+
case option.startsWith("--all"):
28+
all = true;
29+
break;
30+
case option.startsWith("--only-logs"):
31+
log("Running in only logs mode");
32+
onlyLogs = true;
33+
break;
34+
case option.startsWith("--next="):
35+
initialNext = option.split("=")[1];
36+
break;
37+
case option.startsWith("--only-panda-templates"):
38+
onlyPandaTemplates = true;
39+
break;
40+
default:
41+
unexpected(`❌ unknown option: ${option}`);
42+
throw new Error(`unknown option: ${option}`);
43+
}
44+
}
45+
46+
main().catch((error: unknown) => {
47+
unexpected("❌ migration failed", inspect(error, { depth: null, colors: true }));
48+
});
49+
50+
let migratedAccounts = 0;
51+
let redactedAccounts = 0;
52+
let redactedInquiries = 0;
53+
let failedToRedactAccounts = 0;
54+
let noApprovedInquiryAccounts = 0;
55+
let unknownTemplates = 0;
56+
let cryptomateTemplates = 0;
57+
let pandaTemplates = 0;
58+
let schemaErrors = 0;
59+
let failedAccounts = 0;
60+
let inquirySchemaErrors = 0;
61+
let noReferenceIdAccounts = 0;
62+
let totalAccounts = 0;
63+
64+
async function main() {
65+
if (all) {
66+
log("🔍 Processing all accounts");
67+
} else if (reference) {
68+
log(`🔍 Processing accounts with reference ID: ${reference}`);
69+
} else {
70+
unexpected("❌ please provide --reference-id=<id> or --all is required");
71+
throw new Error("missing --reference-id=<id> or --all");
72+
}
73+
74+
let next = initialNext;
75+
let batch = 0;
76+
let retries = 0;
77+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
78+
while (true) {
79+
try {
80+
log(
81+
`\n ----- Processing batch ${batch++} (Batch size: ${BATCH_SIZE}, next: ${next ?? "undefined"}) ${retries > 0 ? `(Retry ${retries})` : ""} -----`,
82+
);
83+
84+
const accounts = await getAccounts(BATCH_SIZE, next ?? undefined, reference).catch((error: unknown) => {
85+
if (error instanceof ValiError) {
86+
unexpected(`❌ Failed process batch ${batch} due to schema errors. Aborting...`);
87+
unexpected("❌ Schema errors:", buildIssueMessages(error.issues));
88+
return { data: [], links: { next: null } };
89+
}
90+
throw error;
91+
});
92+
93+
totalAccounts += accounts.data.length;
94+
log(`🔍 Found ${accounts.data.length} accounts`);
95+
96+
for (const account of accounts.data) {
97+
try {
98+
if (!account.attributes["reference-id"]) {
99+
noReferenceIdAccounts++;
100+
warn(`Account ${account.id} has no reference id`);
101+
continue;
102+
}
103+
await processAccount(account.id, account.attributes["reference-id"]);
104+
} catch (error: unknown) {
105+
unexpected(
106+
`❌ Failed to process batch ${batch}, next: ${next ?? "undefined"}, account: ${account.id}/${account.attributes["reference-id"]} due to: ${inspect(error, { depth: null, colors: true })}`,
107+
);
108+
failedAccounts++;
109+
await new Promise((resolve) => setTimeout(resolve, 1000));
110+
continue;
111+
}
112+
}
113+
114+
next = accounts.data.at(-1)?.id;
115+
if (!next) break;
116+
retries = 0;
117+
} catch (error: unknown) {
118+
unexpected(`❌ Failed to process batch ${batch} due to: ${inspect(error, { depth: null, colors: true })}`);
119+
await new Promise((resolve) => setTimeout(resolve, 1000));
120+
retries++;
121+
if (retries >= 3) {
122+
unexpected(`❌ Failed to process batch ${batch} after 3 retries. Aborting...`);
123+
break;
124+
}
125+
}
126+
}
127+
128+
log(`\n ----- Migration summary -----`);
129+
log(`🔍 Total accounts processed: ${totalAccounts}`);
130+
log(`🔍 Redacted inquiries: ${redactedInquiries}`);
131+
log(`🔍 No approved inquiry accounts, redaction needed: ${noApprovedInquiryAccounts}`);
132+
log(` ---------------------------------`);
133+
log(`✅ Migrated approved accounts: ${migratedAccounts}`);
134+
log(`♻️ Redacted accounts: ${redactedAccounts}`);
135+
log(`❌ Accounts failed to redact: ${failedToRedactAccounts}`);
136+
log(`❌ Accounts failed to process: ${failedAccounts}`);
137+
log(` ---------------------------------`);
138+
139+
log(`\n ----- Approved accounts summary -----`);
140+
log(`🔍 Panda templates: ${pandaTemplates}`);
141+
log(`🚨 Schema errors: ${schemaErrors}`);
142+
log(`🚨 Inquiry schema errors: ${inquirySchemaErrors}`);
143+
log(` ---------------------------------`);
144+
145+
log(`\n ----- Inquiry Statistics summary -----`);
146+
log(`🚨 Unknown templates: ${unknownTemplates}`);
147+
log(`⚰️ Cryptomate templates: ${cryptomateTemplates}`);
148+
log(`⚠️ No reference id accounts: ${noReferenceIdAccounts}`);
149+
log(` ----- Statistics summary -----`);
150+
}
151+
152+
function getAccounts(limit: number, after?: string, referenceId?: string) {
153+
return persona.getUnknownAccounts(limit, after, referenceId);
154+
}
155+
156+
function updateAccountFromInquiry(accountId: string, inquiry: InferOutput<typeof persona.PandaInquiryApproved>) {
157+
const annualSalary =
158+
inquiry.attributes.fields["annual-salary-ranges-us-150-000"]?.value ??
159+
inquiry.attributes.fields["annual-salary"]?.value;
160+
const expectedMonthlyVolume =
161+
inquiry.attributes.fields["monthly-purchases-range"]?.value ??
162+
inquiry.attributes.fields["expected-monthly-volume"]?.value;
163+
if (!annualSalary) throw new Error("annual salary is required");
164+
if (!expectedMonthlyVolume) throw new Error("expected monthly volume is required");
165+
166+
const exaCardTc =
167+
inquiry.attributes.fields["new-screen-2-2-input-checkbox"]?.value ??
168+
inquiry.attributes.fields["new-screen-input-checkbox-2"]?.value;
169+
if (exaCardTc !== true) throw new Error("exa card tc is required");
170+
171+
return persona.updateAccount(accountId, {
172+
rain_e_sign_consent: inquiry.attributes.fields["input-checkbox"].value,
173+
exa_card_tc: exaCardTc,
174+
privacy__policy: inquiry.attributes.fields["new-screen-input-checkbox"].value,
175+
account_opening_disclosure: inquiry.attributes.fields["new-screen-input-checkbox-4"]?.value ?? null,
176+
177+
economic_activity: inquiry.attributes.fields["input-select"].value,
178+
annual_salary: annualSalary,
179+
expected_monthly_volume: expectedMonthlyVolume,
180+
accurate_info_confirmation: inquiry.attributes.fields["new-screen-input-checkbox-1"].value,
181+
non_unauthorized_solicitation: inquiry.attributes.fields["new-screen-input-checkbox-3"].value,
182+
non_illegal_activities_2: inquiry.attributes.fields["illegal-activites"].value, // cspell:ignore illegal-activites
183+
});
184+
}
185+
186+
async function processAccount(accountId: string, referenceId: string) {
187+
const unknownInquiry = await persona
188+
.getUnknownApprovedInquiry(referenceId, onlyPandaTemplates ? persona.PANDA_TEMPLATE : undefined)
189+
.catch((error: unknown) => {
190+
if (error instanceof ValiError) {
191+
unexpected(
192+
`❌ Failed to get unknown approved inquiry for account ${referenceId}/${accountId} due to schema errors`,
193+
buildIssueMessages(error.issues),
194+
);
195+
inquirySchemaErrors++;
196+
throw error;
197+
}
198+
throw error;
199+
});
200+
201+
if (!unknownInquiry) {
202+
noApprovedInquiryAccounts++;
203+
log(`Account ${referenceId}/${accountId} has no approved inquiry. Redacting account...`);
204+
if (onlyLogs) return;
205+
await persona
206+
.redactAccount(accountId)
207+
.then(() => {
208+
log(`♻️ Account ${referenceId}/${accountId} redacted successfully`);
209+
redactedAccounts++;
210+
})
211+
.catch((error: unknown) => {
212+
unexpected(
213+
`❌ Account ${referenceId}/${accountId} redacting failed`,
214+
inspect(error, { depth: null, colors: true }),
215+
);
216+
failedToRedactAccounts++;
217+
});
218+
return;
219+
}
220+
221+
if (unknownInquiry.attributes["redacted-at"]) {
222+
redactedInquiries++;
223+
log(`Inquiry ${referenceId}/${accountId} is redacted. Redacting account...`);
224+
if (onlyLogs) return;
225+
await persona
226+
.redactAccount(accountId)
227+
.then(() => {
228+
log(`♻️ Account ${referenceId}/${accountId} redacted successfully`);
229+
redactedAccounts++;
230+
})
231+
.catch((error: unknown) => {
232+
unexpected(
233+
`❌ Account ${referenceId}/${accountId} redacting failed`,
234+
inspect(error, { depth: null, colors: true }),
235+
);
236+
failedToRedactAccounts++;
237+
});
238+
return;
239+
}
240+
241+
const isPandaTemplate = unknownInquiry.relationships["inquiry-template"]?.data.id === persona.PANDA_TEMPLATE;
242+
const isCryptomateTemplate =
243+
unknownInquiry.relationships["inquiry-template"]?.data.id === persona.CRYPTOMATE_TEMPLATE;
244+
245+
if (isPandaTemplate) {
246+
pandaTemplates++;
247+
const pandaInquiry = safeParse(persona.PandaInquiryApproved, unknownInquiry);
248+
if (!pandaInquiry.success) {
249+
inquirySchemaErrors++;
250+
unexpected(
251+
`❌ Account ${referenceId}/${accountId} failed to parse panda inquiry`,
252+
buildIssueMessages(pandaInquiry.issues),
253+
);
254+
return;
255+
}
256+
debug(`✅ PANDA TEMPLATE: Account ${referenceId}/${accountId} has approved inquiry`);
257+
if (onlyLogs) return;
258+
await updateAccountFromInquiry(accountId, pandaInquiry.output);
259+
await persona.addDocument(pandaInquiry.output.attributes["reference-id"], {
260+
id_class: { value: pandaInquiry.output.attributes.fields["identification-class"].value },
261+
id_number: { value: pandaInquiry.output.attributes.fields["identification-number"].value },
262+
id_issuing_country: { value: pandaInquiry.output.attributes.fields["selected-country-code"].value },
263+
id_document_id: { value: pandaInquiry.output.attributes.fields["current-government-id"].value.id },
264+
});
265+
266+
// validate basic scope
267+
const basicAccount = await persona.getAccount(referenceId, "basic").catch((error: unknown) => {
268+
if (error instanceof ValiError) {
269+
schemaErrors++;
270+
unexpected(
271+
`❌ Account ${referenceId}/${accountId} failed to get basic scope due to schema errors`,
272+
buildIssueMessages(error.issues),
273+
);
274+
} else {
275+
failedAccounts++;
276+
unexpected(
277+
`❌ Account ${referenceId}/${accountId} getting basic scope failed`,
278+
inspect(error, { depth: null, colors: true }),
279+
);
280+
}
281+
});
282+
283+
if (!basicAccount) {
284+
unexpected(`❌ Account ${referenceId}/${accountId} failed to get basic scope`);
285+
return;
286+
}
287+
log(`🎉 PANDA TEMPLATE: Account ${referenceId}/${basicAccount.id} has been migrated and has a valid basic scope`);
288+
migratedAccounts++;
289+
return;
290+
}
291+
292+
if (isCryptomateTemplate) {
293+
cryptomateTemplates++;
294+
warn(
295+
`⚰️ CRYPTOMATE TEMPLATE: Account ${referenceId} has approved inquiry of template ${unknownInquiry.relationships["inquiry-template"]?.data.id}`,
296+
);
297+
return;
298+
}
299+
300+
unknownTemplates++;
301+
warn(
302+
`🚨 UNKNOWN TEMPLATE: Account ${referenceId} has an approved inquiry of template ${unknownInquiry.relationships["inquiry-template"]?.data.id}`,
303+
);
304+
}

‎server/test/api/kyc.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1057,6 +1057,7 @@ const personaTemplate = {
10571057
attributes: {
10581058
status: "approved" as const,
10591059
"reference-id": "ref-123",
1060+
"redacted-at": null,
10601061
"name-first": "John",
10611062
"name-middle": null,
10621063
"name-last": "Doe",
@@ -1068,6 +1069,7 @@ const personaTemplate = {
10681069
relationships: {
10691070
documents: { data: [{ type: "document", id: "1234567890" }] },
10701071
account: { data: { id: "1234567890", type: "account" } } as const,
1072+
"inquiry-template": { data: { id: "template-id" } },
10711073
},
10721074
};
10731075

0 commit comments

Comments
 (0)