diff --git a/JS/edgechains/arakoodev/package.json b/JS/edgechains/arakoodev/package.json index 0b0bd378..7b071ba4 100644 --- a/JS/edgechains/arakoodev/package.json +++ b/JS/edgechains/arakoodev/package.json @@ -22,6 +22,7 @@ "test": "vitest" }, "dependencies": { + "@aws-sdk/client-comprehend": "^3.1076.0", "@babel/core": "^7.24.4", "@babel/preset-env": "^7.24.4", "@hono/node-server": "^0.6.0", @@ -48,6 +49,7 @@ "retell-client-js-sdk": "^2.0.4", "retell-sdk": "^4.9.0", "retry": "^0.13.1", + "rxjs": "^7.8.2", "ts-node": "^10.9.2", "typeorm": "^0.3.20", "vitest": "^2.0.3", diff --git a/JS/edgechains/arakoodev/src/ai/src/index.ts b/JS/edgechains/arakoodev/src/ai/src/index.ts index 2c98f37d..fd65d4f8 100644 --- a/JS/edgechains/arakoodev/src/ai/src/index.ts +++ b/JS/edgechains/arakoodev/src/ai/src/index.ts @@ -3,3 +3,17 @@ export { GeminiAI } from "./lib/gemini/gemini.js"; export { LlamaAI } from "./lib/llama/llama.js"; export { RetellAI } from "./lib/retell-ai/retell.js"; export { RetellWebClient } from "./lib/retell-ai/retellWebClient.js"; +export { + ComprehendPiiRedactor, + type ChatEndpoint, + type ComprehendClientLike, + type ComprehendPiiRedactorOptions, + type ComprehendRequestOptions, + type DetectPiiEntitiesResult, + type PiiLanguageCode, + type RedactableChatOptions, + type RedactableMessage, + type RedactableValue, + type RedactionReplacement, + type RedactionResult, +} from "./lib/comprehend/comprehendPiiRedactor.js"; diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/comprehendPiiRedactor.ts b/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/comprehendPiiRedactor.ts new file mode 100644 index 00000000..35d06357 --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/comprehendPiiRedactor.ts @@ -0,0 +1,399 @@ +import { + ComprehendClient, + DetectPiiEntitiesCommand, + type PiiEntity, +} from "@aws-sdk/client-comprehend"; +import { Buffer } from "node:buffer"; +import { Observable, concatMap, type OperatorFunction } from "rxjs"; + +export type PiiLanguageCode = "en" | "es"; + +export interface ComprehendRequestOptions { + abortSignal?: AbortSignal; +} + +export interface DetectPiiEntitiesResult { + Entities?: PiiEntity[]; +} + +export interface ComprehendClientLike { + send( + command: DetectPiiEntitiesCommand, + options?: ComprehendRequestOptions + ): Promise; +} + +export interface RedactableMessage { + content?: string; + [key: string]: unknown; +} + +export interface RedactableChatOptions { + prompt?: string; + messages?: RedactableMessage[]; + [key: string]: unknown; +} + +export interface ChatEndpoint< + TOptions extends RedactableChatOptions = RedactableChatOptions, + TResult = unknown, +> { + chat(options: TOptions): Promise; +} + +export type RedactableValue = string | RedactableChatOptions; + +export type RedactionReplacement = + | "entityType" + | "mask" + | ((entity: PiiEntity, originalValue: string) => string); + +export interface ComprehendPiiRedactorOptions { + client?: ComprehendClientLike; + region?: string; + languageCode?: PiiLanguageCode; + minScore?: number; + entityTypes?: readonly string[]; + replacement?: RedactionReplacement; + maskCharacter?: string; + maxUtf8Bytes?: number; +} + +export interface RedactionResult { + text: string; + entities: PiiEntity[]; +} + +const DEFAULT_MAX_UTF8_BYTES = 100 * 1024; + +/** + * Redacts PII detected by Amazon Comprehend before a prompt reaches an LLM endpoint. + * + * Amazon Comprehend offsets are Unicode code-point offsets. JavaScript string slicing + * uses UTF-16 code units, so replacements are applied to an Array.from(text) view to + * keep offsets correct when emoji or other supplementary characters appear first. + */ +export class ComprehendPiiRedactor { + private readonly client: ComprehendClientLike; + private readonly languageCode: PiiLanguageCode; + private readonly minScore: number; + private readonly entityTypes?: Set; + private readonly replacement: RedactionReplacement; + private readonly maskCharacter: string; + private readonly maxUtf8Bytes: number; + + constructor(options: ComprehendPiiRedactorOptions = {}) { + const minScore = options.minScore ?? 0; + if (!Number.isFinite(minScore) || minScore < 0 || minScore > 1) { + throw new RangeError("minScore must be between 0 and 1"); + } + + const maskCharacter = options.maskCharacter ?? "*"; + if (Array.from(maskCharacter).length !== 1) { + throw new RangeError( + "maskCharacter must contain exactly one Unicode code point" + ); + } + + const maxUtf8Bytes = options.maxUtf8Bytes ?? DEFAULT_MAX_UTF8_BYTES; + if (!Number.isInteger(maxUtf8Bytes) || maxUtf8Bytes < 1) { + throw new RangeError("maxUtf8Bytes must be a positive integer"); + } + + this.client = + options.client ?? new ComprehendClient({ region: options.region }); + this.languageCode = options.languageCode ?? "en"; + this.minScore = minScore; + this.entityTypes = options.entityTypes + ? new Set(options.entityTypes) + : undefined; + this.replacement = options.replacement ?? "entityType"; + this.maskCharacter = maskCharacter; + this.maxUtf8Bytes = maxUtf8Bytes; + } + + async detect( + text: string, + requestOptions: ComprehendRequestOptions = {} + ): Promise { + if (!text) { + return []; + } + + this.validateTextSize(text); + + const output = await this.client.send( + new DetectPiiEntitiesCommand({ + Text: text, + LanguageCode: this.languageCode, + }), + requestOptions.abortSignal + ? { abortSignal: requestOptions.abortSignal } + : undefined + ); + + const codePointLength = Array.from(text).length; + return (output.Entities ?? []).filter((entity) => + this.isRedactableEntity(entity, codePointLength) + ); + } + + async redact( + text: string, + requestOptions: ComprehendRequestOptions = {} + ): Promise { + return (await this.redactWithEntities(text, requestOptions)).text; + } + + async redactWithEntities( + text: string, + requestOptions: ComprehendRequestOptions = {} + ): Promise { + if (!text) { + return { text, entities: [] }; + } + + const entities = this.selectNonOverlappingEntities( + await this.detect(text, requestOptions) + ); + const codePoints = Array.from(text); + + for (const entity of entities) { + const begin = entity.BeginOffset as number; + const end = entity.EndOffset as number; + const originalValue = codePoints.slice(begin, end).join(""); + const replacement = this.createReplacement(entity, originalValue); + codePoints.splice(begin, end - begin, ...Array.from(replacement)); + } + + return { + text: codePoints.join(""), + entities: [...entities].sort( + (a, b) => (a.BeginOffset as number) - (b.BeginOffset as number) + ), + }; + } + + async redactChatOptions( + chatOptions: T, + requestOptions: ComprehendRequestOptions = {} + ): Promise { + const [prompt, messages] = await Promise.all([ + typeof chatOptions.prompt === "string" + ? this.redact(chatOptions.prompt, requestOptions) + : Promise.resolve(chatOptions.prompt), + Array.isArray(chatOptions.messages) + ? Promise.all( + chatOptions.messages.map(async (message) => ({ + ...message, + content: + typeof message.content === "string" + ? await this.redact( + message.content, + requestOptions + ) + : message.content, + })) + ) + : Promise.resolve(chatOptions.messages), + ]); + + return { + ...chatOptions, + prompt, + messages, + } as T; + } + + redact$(text: string): Observable { + return this.cancellableObservable((abortSignal) => + this.redact(text, { abortSignal }) + ); + } + + redactChatOptions$( + chatOptions: T + ): Observable { + return this.cancellableObservable((abortSignal) => + this.redactChatOptions(chatOptions, { abortSignal }) + ); + } + + /** + * Returns a real RxJS operator. concatMap preserves input order and waits for each + * Comprehend request before allowing the source Observable to complete. + */ + redactOperator(): OperatorFunction { + return (source) => + source.pipe( + concatMap((value) => + this.cancellableObservable((abortSignal) => + this.redactValue(value, { abortSignal }) + ) + ) + ); + } + + /** + * Redacts endpoint options and invokes the endpoint as one Observable pipeline step. + */ + endpointOperator( + endpoint: ChatEndpoint + ): OperatorFunction { + return (source) => + source.pipe( + concatMap((options) => + this.redactChatOptions$(options).pipe( + concatMap((redactedOptions) => + endpoint.chat.call(endpoint, redactedOptions) + ) + ) + ) + ); + } + + wrapEndpoint( + endpoint: ChatEndpoint + ): ChatEndpoint { + return { + chat: async (options: TOptions) => + endpoint.chat.call( + endpoint, + await this.redactChatOptions(options) + ), + }; + } + + private async redactValue( + value: T, + requestOptions: ComprehendRequestOptions + ): Promise { + if (typeof value === "string") { + return (await this.redact(value, requestOptions)) as T; + } + + return this.redactChatOptions(value, requestOptions) as Promise; + } + + private validateTextSize(text: string): void { + const byteLength = Buffer.byteLength(text, "utf8"); + if (byteLength > this.maxUtf8Bytes) { + throw new RangeError( + `Amazon Comprehend DetectPiiEntities accepts at most ${this.maxUtf8Bytes} UTF-8 bytes; received ${byteLength}` + ); + } + } + + private isRedactableEntity( + entity: PiiEntity, + codePointLength: number + ): boolean { + const begin = entity.BeginOffset; + const end = entity.EndOffset; + + if ( + !Number.isInteger(begin) || + !Number.isInteger(end) || + (begin as number) < 0 || + (end as number) <= (begin as number) || + (end as number) > codePointLength + ) { + return false; + } + + if ((entity.Score ?? 0) < this.minScore) { + return false; + } + + if ( + this.entityTypes && + (!entity.Type || !this.entityTypes.has(entity.Type)) + ) { + return false; + } + + return true; + } + + private selectNonOverlappingEntities(entities: PiiEntity[]): PiiEntity[] { + const candidates = [...entities].sort((a, b) => { + const scoreDifference = (b.Score ?? 0) - (a.Score ?? 0); + if (scoreDifference !== 0) { + return scoreDifference; + } + + const aLength = (a.EndOffset as number) - (a.BeginOffset as number); + const bLength = (b.EndOffset as number) - (b.BeginOffset as number); + if (aLength !== bLength) { + return bLength - aLength; + } + + return (a.BeginOffset as number) - (b.BeginOffset as number); + }); + + const selected: PiiEntity[] = []; + for (const candidate of candidates) { + const begin = candidate.BeginOffset as number; + const end = candidate.EndOffset as number; + const overlaps = selected.some((entity) => { + const selectedBegin = entity.BeginOffset as number; + const selectedEnd = entity.EndOffset as number; + return begin < selectedEnd && end > selectedBegin; + }); + + if (!overlaps) { + selected.push(candidate); + } + } + + return selected.sort( + (a, b) => (b.BeginOffset as number) - (a.BeginOffset as number) + ); + } + + private createReplacement( + entity: PiiEntity, + originalValue: string + ): string { + if (typeof this.replacement === "function") { + return this.replacement(entity, originalValue); + } + + if (this.replacement === "mask") { + return this.maskCharacter.repeat(Array.from(originalValue).length); + } + + return `[${entity.Type ?? "PII"}]`; + } + + private cancellableObservable( + work: (abortSignal: AbortSignal) => Promise + ): Observable { + return new Observable((subscriber) => { + const controller = new AbortController(); + let settled = false; + + work(controller.signal).then( + (value) => { + settled = true; + if (!subscriber.closed) { + subscriber.next(value); + subscriber.complete(); + } + }, + (error) => { + settled = true; + if (!subscriber.closed) { + subscriber.error(error); + } + } + ); + + return () => { + if (!settled) { + controller.abort(); + } + }; + }); + } +} diff --git a/JS/edgechains/arakoodev/src/ai/src/tests/comprehend/comprehendPiiRedactor.test.ts b/JS/edgechains/arakoodev/src/ai/src/tests/comprehend/comprehendPiiRedactor.test.ts new file mode 100644 index 00000000..645b5a74 --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/tests/comprehend/comprehendPiiRedactor.test.ts @@ -0,0 +1,208 @@ +import { describe, expect, it, vi } from "vitest"; +import { firstValueFrom, of, toArray } from "rxjs"; +import { + ComprehendPiiRedactor, + type ComprehendClientLike, +} from "../../lib/comprehend/comprehendPiiRedactor.js"; + +function codePointOffset(text: string, value: string): number { + const utf16Offset = text.indexOf(value); + if (utf16Offset < 0) { + throw new Error(`Could not find ${value}`); + } + return Array.from(text.slice(0, utf16Offset)).length; +} + +function entity(text: string, value: string, type: string, score = 0.99) { + const begin = codePointOffset(text, value); + return { + Type: type, + Score: score, + BeginOffset: begin, + EndOffset: begin + Array.from(value).length, + }; +} + +describe("ComprehendPiiRedactor", () => { + it("redacts multiple entities using Unicode code-point offsets", async () => { + const text = "🔒 Contact José at jose@example.com"; + const client = { + send: vi.fn().mockResolvedValue({ + Entities: [ + entity(text, "José", "NAME"), + entity(text, "jose@example.com", "EMAIL"), + ], + }), + }; + const redactor = new ComprehendPiiRedactor({ client }); + + await expect(redactor.redact(text)).resolves.toBe( + "🔒 Contact [NAME] at [EMAIL]" + ); + }); + + it("filters low-confidence and unwanted entity types", async () => { + const text = "Jane uses jane@example.com"; + const client = { + send: vi.fn().mockResolvedValue({ + Entities: [ + entity(text, "Jane", "NAME", 0.99), + entity(text, "jane@example.com", "EMAIL", 0.6), + ], + }), + }; + const redactor = new ComprehendPiiRedactor({ + client, + minScore: 0.8, + entityTypes: ["NAME"], + }); + + await expect(redactor.redact(text)).resolves.toBe( + "[NAME] uses jane@example.com" + ); + }); + + it("chooses the highest-confidence entity when ranges overlap", async () => { + const text = "Jane Doe"; + const client = { + send: vi.fn().mockResolvedValue({ + Entities: [ + { Type: "NAME", Score: 0.8, BeginOffset: 0, EndOffset: 4 }, + { Type: "NAME", Score: 0.99, BeginOffset: 0, EndOffset: 8 }, + ], + }), + }; + const redactor = new ComprehendPiiRedactor({ client }); + + await expect(redactor.redact(text)).resolves.toBe("[NAME]"); + }); + + it("redacts endpoint options without mutating the caller's object", async () => { + const prompt = "Email jane@example.com"; + const message = "Call 555-010-1234"; + const client = { + send: vi + .fn() + .mockResolvedValueOnce({ + Entities: [entity(prompt, "jane@example.com", "EMAIL")], + }) + .mockResolvedValueOnce({ + Entities: [entity(message, "555-010-1234", "PHONE")], + }), + }; + const redactor = new ComprehendPiiRedactor({ client }); + const input = { + prompt, + messages: [{ role: "user", content: message }], + temperature: 0.2, + }; + + const output = await redactor.redactChatOptions(input); + + expect(output).toEqual({ + prompt: "Email [EMAIL]", + messages: [{ role: "user", content: "Call [PHONE]" }], + temperature: 0.2, + }); + expect(input).toEqual({ + prompt, + messages: [{ role: "user", content: message }], + temperature: 0.2, + }); + expect(output).not.toBe(input); + expect(output.messages).not.toBe(input.messages); + }); + + it("provides a real ordered RxJS operator and waits for async work", async () => { + let resolveFirst: ((value: { Entities: never[] }) => void) | undefined; + const first = new Promise<{ Entities: never[] }>((resolve) => { + resolveFirst = resolve; + }); + const client = { + send: vi + .fn() + .mockImplementationOnce(() => first) + .mockResolvedValueOnce({ Entities: [] }), + }; + const redactor = new ComprehendPiiRedactor({ client }); + + const resultPromise = firstValueFrom( + of("first", "second").pipe( + redactor.redactOperator(), + toArray() + ) + ); + + await Promise.resolve(); + expect(client.send).toHaveBeenCalledTimes(1); + + resolveFirst?.({ Entities: [] }); + const result = await resultPromise; + + expect(client.send).toHaveBeenCalledTimes(2); + expect(result).toEqual(["first", "second"]); + }); + + it("aborts the AWS request when an Observable subscription is cancelled", async () => { + let signal: AbortSignal | undefined; + const send: ComprehendClientLike["send"] = (_command, options) => { + signal = options?.abortSignal; + return new Promise(() => undefined); + }; + const client: ComprehendClientLike = { send: vi.fn(send) }; + const redactor = new ComprehendPiiRedactor({ client }); + + const subscription = redactor.redact$("Jane").subscribe(); + await Promise.resolve(); + subscription.unsubscribe(); + + expect(signal).toBeDefined(); + expect(signal?.aborted).toBe(true); + }); + + it("redacts options before invoking an endpoint in an Observable chain", async () => { + const prompt = "Email jane@example.com"; + const client = { + send: vi.fn().mockResolvedValue({ + Entities: [entity(prompt, "jane@example.com", "EMAIL")], + }), + }; + const endpoint = { + chat: vi.fn().mockResolvedValue({ content: "ok" }), + }; + const redactor = new ComprehendPiiRedactor({ client }); + + const output = await firstValueFrom( + of({ prompt }).pipe(redactor.endpointOperator(endpoint)) + ); + + expect(output).toEqual({ content: "ok" }); + expect(endpoint.chat).toHaveBeenCalledWith({ prompt: "Email [EMAIL]" }); + }); + + it("rejects input larger than the configured UTF-8 byte limit before AWS is called", async () => { + const client = { + send: vi.fn().mockResolvedValue({ Entities: [] }), + }; + const redactor = new ComprehendPiiRedactor({ client, maxUtf8Bytes: 4 }); + + await expect(redactor.redact("🔒x")).rejects.toThrow( + "at most 4 UTF-8 bytes" + ); + expect(client.send).not.toHaveBeenCalled(); + }); + + it("ignores invalid entity offsets", async () => { + const client = { + send: vi.fn().mockResolvedValue({ + Entities: [ + { Type: "NAME", Score: 1, BeginOffset: -1, EndOffset: 3 }, + { Type: "EMAIL", Score: 1, BeginOffset: 2, EndOffset: 200 }, + ], + }), + }; + const redactor = new ComprehendPiiRedactor({ client }); + + await expect(redactor.redact("hello")).resolves.toBe("hello"); + }); +}); diff --git a/JS/edgechains/examples/aws-comprehend-redaction/LOOM_SCRIPT.md b/JS/edgechains/examples/aws-comprehend-redaction/LOOM_SCRIPT.md new file mode 100644 index 00000000..bfb7bd9a --- /dev/null +++ b/JS/edgechains/examples/aws-comprehend-redaction/LOOM_SCRIPT.md @@ -0,0 +1,10 @@ +# Loom recording script (about 60–90 seconds) + +1. Show issue #290 and the three completion requirements. +2. Open `src/index.ts` and point out `of(input).pipe(redactor.endpointOperator(endpoint))`. +3. Run `npm run demo` without AWS credentials and show that the endpoint receives `[EMAIL]` and + `[PHONE]`, not the original PII. +4. Show the RxJS tests for ordering, completion, errors, and cancellation. +5. Run the focused test command and show it passing. +6. Optionally set `USE_REAL_AWS=true` and repeat using a least-privilege AWS profile. +7. Paste the Loom URL into the PR description before requesting review. diff --git a/JS/edgechains/examples/aws-comprehend-redaction/README.md b/JS/edgechains/examples/aws-comprehend-redaction/README.md new file mode 100644 index 00000000..4f525df0 --- /dev/null +++ b/JS/edgechains/examples/aws-comprehend-redaction/README.md @@ -0,0 +1,49 @@ +# AWS Comprehend PII redaction example + +This example demonstrates an RxJS pipeline that redacts PII before an endpoint receives a prompt. +It uses a deterministic offline client by default, so CI and reviewers do not need AWS credentials. + +## Run the offline example + +Build the local SDK first: + +```bash +cd ../../arakoodev +npm install +npm run build + +cd ../examples/aws-comprehend-redaction +npm install +npm run demo +``` + +Expected output: + +```text +Original: 🔒 Please contact jane@example.com or call 555-010-1234. +Endpoint received: 🔒 Please contact [EMAIL] or call [PHONE]. +``` + +## Run against Amazon Comprehend + +The caller needs `comprehend:DetectPiiEntities`. Credentials are loaded through the standard AWS +SDK credential provider chain; do not put access keys in source code. + +```bash +export AWS_REGION=us-east-1 +export USE_REAL_AWS=true +npm run demo +``` + +Amazon Comprehend PII detection supports English (`en`) and Spanish (`es`) input and accepts up +to 100 KiB of UTF-8 text per real-time request. The example includes an emoji before the PII to +demonstrate correct Unicode code-point offset handling. + +## Observable chain + +```ts +const response$ = of({ prompt }).pipe(redactor.endpointOperator(endpoint)); +``` + +`endpointOperator` preserves input order, propagates errors through RxJS, waits for redaction before +completion, and aborts the Comprehend request when the subscription is cancelled. diff --git a/JS/edgechains/examples/aws-comprehend-redaction/dist/index.js b/JS/edgechains/examples/aws-comprehend-redaction/dist/index.js new file mode 100644 index 00000000..a8d524db --- /dev/null +++ b/JS/edgechains/examples/aws-comprehend-redaction/dist/index.js @@ -0,0 +1,48 @@ +import { ComprehendPiiRedactor, } from "@arakoodev/edgechains.js/ai"; +import { firstValueFrom, of } from "rxjs"; +class DemoEndpoint { + async chat(options) { + return { + content: `Endpoint received: ${options.prompt ?? ""}`, + }; + } +} +function codePointOffset(text, value) { + const utf16Offset = text.indexOf(value); + return Array.from(text.slice(0, utf16Offset)).length; +} +function createOfflineClient() { + return { + async send(command) { + const text = command.input.Text ?? ""; + const email = "jane@example.com"; + const phone = "555-010-1234"; + const entities = [email, phone] + .filter((value) => text.includes(value)) + .map((value) => { + const begin = codePointOffset(text, value); + return { + Type: value === email ? "EMAIL" : "PHONE", + Score: 0.999, + BeginOffset: begin, + EndOffset: begin + Array.from(value).length, + }; + }); + return { Entities: entities }; + }, + }; +} +const useRealAws = process.env.USE_REAL_AWS === "true"; +const redactor = new ComprehendPiiRedactor({ + client: useRealAws ? undefined : createOfflineClient(), + region: process.env.AWS_REGION ?? "us-east-1", + languageCode: "en", + minScore: 0.8, +}); +const endpoint = new DemoEndpoint(); +const input = { + prompt: "🔒 Please contact jane@example.com or call 555-010-1234.", +}; +const response = await firstValueFrom(of(input).pipe(redactor.endpointOperator(endpoint))); +console.log("Original:", input.prompt); +console.log(response.content); diff --git a/JS/edgechains/examples/aws-comprehend-redaction/package.json b/JS/edgechains/examples/aws-comprehend-redaction/package.json new file mode 100644 index 00000000..cd3c7b9a --- /dev/null +++ b/JS/edgechains/examples/aws-comprehend-redaction/package.json @@ -0,0 +1,20 @@ +{ + "name": "aws-comprehend-redaction-example", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "build": "tsc", + "start": "node dist/index.js", + "demo": "npm run build && npm start" + }, + "dependencies": { + "@arakoodev/edgechains.js": "file:../../arakoodev", + "@aws-sdk/client-comprehend": "^3.1076.0", + "rxjs": "^7.8.2" + }, + "devDependencies": { + "@types/node": "^20.17.2", + "typescript": "^5.6.3" + } +} diff --git a/JS/edgechains/examples/aws-comprehend-redaction/src/index.ts b/JS/edgechains/examples/aws-comprehend-redaction/src/index.ts new file mode 100644 index 00000000..29b6645a --- /dev/null +++ b/JS/edgechains/examples/aws-comprehend-redaction/src/index.ts @@ -0,0 +1,67 @@ +import { + ComprehendPiiRedactor, + type ChatEndpoint, + type ComprehendClientLike, + type RedactableChatOptions, +} from "@arakoodev/edgechains.js/ai"; +import type { PiiEntity } from "@aws-sdk/client-comprehend"; +import { firstValueFrom, of } from "rxjs"; + +class DemoEndpoint implements ChatEndpoint< + RedactableChatOptions, + { content: string } +> { + async chat(options: RedactableChatOptions): Promise<{ content: string }> { + return { + content: `Endpoint received: ${options.prompt ?? ""}`, + }; + } +} + +function codePointOffset(text: string, value: string): number { + const utf16Offset = text.indexOf(value); + return Array.from(text.slice(0, utf16Offset)).length; +} + +function createOfflineClient(): ComprehendClientLike { + return { + async send(command) { + const text = command.input.Text ?? ""; + const email = "jane@example.com"; + const phone = "555-010-1234"; + const entities: PiiEntity[] = [email, phone] + .filter((value) => text.includes(value)) + .map((value) => { + const begin = codePointOffset(text, value); + return { + Type: value === email ? "EMAIL" : "PHONE", + Score: 0.999, + BeginOffset: begin, + EndOffset: begin + Array.from(value).length, + }; + }); + + return { Entities: entities }; + }, + }; +} + +const useRealAws = process.env.USE_REAL_AWS === "true"; +const redactor = new ComprehendPiiRedactor({ + client: useRealAws ? undefined : createOfflineClient(), + region: process.env.AWS_REGION ?? "us-east-1", + languageCode: "en", + minScore: 0.8, +}); + +const endpoint = new DemoEndpoint(); +const input = { + prompt: "🔒 Please contact jane@example.com or call 555-010-1234.", +}; + +const response = await firstValueFrom( + of(input).pipe(redactor.endpointOperator(endpoint)) +); + +console.log("Original:", input.prompt); +console.log(response.content); diff --git a/JS/edgechains/examples/aws-comprehend-redaction/tsconfig.json b/JS/edgechains/examples/aws-comprehend-redaction/tsconfig.json new file mode 100644 index 00000000..c49496ce --- /dev/null +++ b/JS/edgechains/examples/aws-comprehend-redaction/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true + }, + "include": ["src/**/*.ts"] +}