diff --git a/JS/edgechains/arakoodev/package.json b/JS/edgechains/arakoodev/package.json index 0b0bd378..1be44121 100644 --- a/JS/edgechains/arakoodev/package.json +++ b/JS/edgechains/arakoodev/package.json @@ -1,79 +1,81 @@ { - "name": "@arakoodev/edgechains.js", - "version": "0.1.23", - "main": "dist/index.js", - "files": [ - "dist" - ], - "exports": { - "./ai": "./dist/ai/src/index.js", - "./vector-db": "./dist/vector-db/src/index.js", - "./document-loader": "./dist/document-loader/src/index.js", - "./splitter": "./dist/splitter/src/index.js", - "./arakooserver": "./dist/arakooserver/src/index.js", - "./db": "./dist/db/src/index.js", - "./scraper": "./dist/scraper/src/index.js", - "./sync-rpc": "./dist/sync-rpc/index.js" - }, - "scripts": { - "build": "rm -rf dist && tsc -b", - "lint": "eslint --ignore-path .eslintignore --ext .js,.ts", - "format": "prettier --ignore-path .gitignore --write \"**/*.+(js|ts|json)\"", - "test": "vitest" - }, - "dependencies": { - "@babel/core": "^7.24.4", - "@babel/preset-env": "^7.24.4", - "@hono/node-server": "^0.6.0", - "@lifeomic/attempt": "^3.1.0", - "@playwright/test": "^1.45.3", - "@supabase/supabase-js": "^2.42.3", - "axios": "^1.7.4", - "axios-retry": "^4.1.0", - "cheerio": "^1.0.0-rc.12", - "cors": "^2.8.5", - "document": "^0.4.7", - "dts-bundle-generator": "^9.3.1", - "esbuild": "^0.20.2", - "hono": "3.9", - "jest-environment-jsdom": "^29.7.0", - "jsdom": "^24.1.0", - "node-html-parser": "^6.1.13", - "pdf-parse": "^1.1.1", - "pg": "^8.11.5", - "playwright": "^1.45.1", - "prettier": "^3.2.5", - "regenerator-runtime": "^0.14.1", - "request": "^2.88.2", - "retell-client-js-sdk": "^2.0.4", - "retell-sdk": "^4.9.0", - "retry": "^0.13.1", - "ts-node": "^10.9.2", - "typeorm": "^0.3.20", - "vitest": "^2.0.3", - "youtube-transcript": "^1.2.1", - "zod": "^3.23.8", - "zod-to-json-schema": "^3.23.0" - }, - "keywords": [], - "author": "", - "license": "ISC", - "devDependencies": { - "@babel/preset-typescript": "^7.24.1", - "@types/cors": "^2.8.17", - "@types/jest": "^29.5.12", - "@types/node": "^20.17.2", - "@types/pdf-parse": "^1.1.4", - "@types/ws": "^8.5.12", - "buffer": "^6.0.3", - "crypto-browserify": "^3.12.1", - "jest": "^29.7.0", - "process": "^0.11.10", - "stream-browserify": "^3.0.0", - "stream-http": "^3.2.0", - "ts-jest": "^29.1.2", - "ts-loader": "^9.5.1", - "typescript": "^5.6.3", - "util": "^0.12.5" - } + "name": "@arakoodev/edgechains.js", + "version": "0.1.23", + "main": "dist/index.js", + "files": [ + "dist" + ], + "exports": { + "./ai": "./dist/ai/src/index.js", + "./vector-db": "./dist/vector-db/src/index.js", + "./document-loader": "./dist/document-loader/src/index.js", + "./splitter": "./dist/splitter/src/index.js", + "./arakooserver": "./dist/arakooserver/src/index.js", + "./db": "./dist/db/src/index.js", + "./scraper": "./dist/scraper/src/index.js", + "./sync-rpc": "./dist/sync-rpc/index.js" + }, + "scripts": { + "build": "rm -rf dist && tsc -b", + "lint": "eslint --ignore-path .eslintignore --ext .js,.ts", + "format": "prettier --ignore-path .gitignore --write \"**/*.+(js|ts|json)\"", + "test": "vitest" + }, + "dependencies": { + "@babel/core": "^7.24.4", + "@babel/preset-env": "^7.24.4", + "@hono/node-server": "^0.6.0", + "@lifeomic/attempt": "^3.1.0", + "@playwright/test": "^1.45.3", + "@supabase/supabase-js": "^2.42.3", + "axios": "^1.7.4", + "axios-retry": "^4.1.0", + "cheerio": "^1.0.0-rc.12", + "cors": "^2.8.5", + "document": "^0.4.7", + "dts-bundle-generator": "^9.3.1", + "esbuild": "^0.20.2", + "hono": "3.9", + "jest-environment-jsdom": "^29.7.0", + "jsdom": "^24.1.0", + "node-html-parser": "^6.1.13", + "pdf-parse": "^1.1.1", + "pg": "^8.11.5", + "playwright": "^1.45.1", + "prettier": "^3.2.5", + "regenerator-runtime": "^0.14.1", + "request": "^2.88.2", + "retell-client-js-sdk": "^2.0.4", + "retell-sdk": "^4.9.0", + "retry": "^0.13.1", + "ts-node": "^10.9.2", + "typeorm": "^0.3.20", + "vitest": "^2.0.3", + "youtube-transcript": "^1.2.1", + "zod": "^3.23.8", + "zod-to-json-schema": "^3.23.0", + "@aws-sdk/client-comprehend": "^3.600.0", + "rxjs": "^7.8.1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@babel/preset-typescript": "^7.24.1", + "@types/cors": "^2.8.17", + "@types/jest": "^29.5.12", + "@types/node": "^20.17.2", + "@types/pdf-parse": "^1.1.4", + "@types/ws": "^8.5.12", + "buffer": "^6.0.3", + "crypto-browserify": "^3.12.1", + "jest": "^29.7.0", + "process": "^0.11.10", + "stream-browserify": "^3.0.0", + "stream-http": "^3.2.0", + "ts-jest": "^29.1.2", + "ts-loader": "^9.5.1", + "typescript": "^5.6.3", + "util": "^0.12.5" + } } diff --git a/JS/edgechains/arakoodev/src/ai/src/index.ts b/JS/edgechains/arakoodev/src/ai/src/index.ts index 2c98f37d..2d89cd00 100644 --- a/JS/edgechains/arakoodev/src/ai/src/index.ts +++ b/JS/edgechains/arakoodev/src/ai/src/index.ts @@ -3,3 +3,4 @@ export { GeminiAI } from "./lib/gemini/gemini.js"; export { LlamaAI } from "./lib/llama/llama.js"; export { RetellAI } from "./lib/retell-ai/retell.js"; export { RetellWebClient } from "./lib/retell-ai/retellWebClient.js"; +export { ComprehendRedactor } from "./lib/comprehend-redactor/index.js"; diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/comprehend-redactor/ComprehendRedactor.ts b/JS/edgechains/arakoodev/src/ai/src/lib/comprehend-redactor/ComprehendRedactor.ts new file mode 100644 index 00000000..5965d14f --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/comprehend-redactor/ComprehendRedactor.ts @@ -0,0 +1,112 @@ +import { Observable } from "rxjs"; + +export type PiiEntityType = + | "ADDRESS" | "AGE" | "AWS_ACCESS_KEY" | "AWS_SECRET_KEY" | "BANK_ACCOUNT_NUMBER" + | "BANK_ROUTING" | "CREDIT_DEBIT_CVV" | "CREDIT_DEBIT_EXPIRY" | "CREDIT_DEBIT_NUMBER" + | "DATE_TIME" | "DRIVER_ID" | "EMAIL" | "INTERNATIONAL_BANK_ACCOUNT_NUMBER" + | "IP_ADDRESS" | "LICENSE_PLATE" | "MAC_ADDRESS" | "NAME" | "PASSPORT_NUMBER" + | "PASSWORD" | "PHONE" | "PIN" | "SSN" | "URL" | "USERNAME" | "VEHICLE_IDENTIFICATION_NUMBER"; + +export interface ComprehendRedactorConfig { + region?: string; + accessKeyId?: string; + secretAccessKey?: string; +} + +export interface RedactResult { + original: string; + redacted: string; + piiEntities: Array<{ type: string; score: number }>; +} + +function maskPii(text: string, piiEntities: Array<{ type: string; score: number; beginOffset: number; endOffset: number }>): string { + let result = text; + const sorted = [...piiEntities].sort((a, b) => b.beginOffset - a.beginOffset); + for (const entity of sorted) { + const mask = `[${entity.type}]`; + result = result.slice(0, entity.beginOffset) + mask + result.slice(entity.endOffset); + } + return result; +} + +export class ComprehendRedactor { + private config: ComprehendRedactorConfig; + + constructor(config: ComprehendRedactorConfig = {}) { + this.config = { + region: config.region || process.env.AWS_REGION || "us-east-1", + accessKeyId: config.accessKeyId || process.env.AWS_ACCESS_KEY_ID || "", + secretAccessKey: config.secretAccessKey || process.env.AWS_SECRET_ACCESS_KEY || "", + }; + } + + async redact(text: string, piiTypes?: PiiEntityType[]): Promise { + const entities = await this.detectPii(text, piiTypes); + const redacted = maskPii(text, entities); + return { + original: text, + redacted, + piiEntities: entities.map(e => ({ type: e.type, score: e.score })), + }; + } + + redact$(text: string, piiTypes?: PiiEntityType[]): Observable { + return new Observable(subscriber => { + this.redact(text, piiTypes) + .then(result => { + subscriber.next(result); + subscriber.complete(); + }) + .catch(err => subscriber.error(err)); + }); + } + + mask$(source: Observable, piiTypes?: PiiEntityType[]): Observable { + return new Observable(subscriber => { + source.subscribe({ + next: async (text) => { + try { + const result = await this.redact(text, piiTypes); + subscriber.next(result.redacted); + } catch (err) { + subscriber.error(err); + } + }, + error: (err) => subscriber.error(err), + complete: () => subscriber.complete(), + }); + }); + } + + private async detectPii( + text: string, + piiTypes?: PiiEntityType[], + ): Promise> { + const { ComprehendClient, DetectPiiEntitiesCommand } = await import("@aws-sdk/client-comprehend"); + + const client = new ComprehendClient({ + region: this.config.region, + credentials: { + accessKeyId: this.config.accessKeyId, + secretAccessKey: this.config.secretAccessKey, + }, + }); + + const command = new DetectPiiEntitiesCommand({ + Text: text, + LanguageCode: "en", + }); + + const response = await client.send(command); + const entities = (response.Entities || []) + .filter(e => !piiTypes || piiTypes.includes(e.Type as PiiEntityType)) + .map(e => ({ + type: e.Type || "UNKNOWN", + score: e.Score || 0, + beginOffset: e.BeginOffset || 0, + endOffset: e.EndOffset || 0, + })); + + return entities; + } +} diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/comprehend-redactor/index.ts b/JS/edgechains/arakoodev/src/ai/src/lib/comprehend-redactor/index.ts new file mode 100644 index 00000000..8b313d2b --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/comprehend-redactor/index.ts @@ -0,0 +1,2 @@ +export { ComprehendRedactor } from "./ComprehendRedactor"; +export type { ComprehendRedactorConfig, RedactResult, PiiEntityType } from "./ComprehendRedactor"; diff --git a/JS/edgechains/arakoodev/src/ai/src/tests/comprehendRedactor.test.ts b/JS/edgechains/arakoodev/src/ai/src/tests/comprehendRedactor.test.ts new file mode 100644 index 00000000..5fb4b4cd --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/tests/comprehendRedactor.test.ts @@ -0,0 +1,101 @@ +import { describe, it, expect, vi } from "vitest"; +import { ComprehendRedactor } from "../lib/comprehend-redactor/ComprehendRedactor"; + +vi.mock("@aws-sdk/client-comprehend", () => { + const mockSend = vi.fn(); + return { + ComprehendClient: vi.fn(() => ({ + send: mockSend, + })), + DetectPiiEntitiesCommand: vi.fn(), + __mockSend: mockSend, + }; +}); + +describe("ComprehendRedactor", () => { + it("should redact PII from text", async () => { + const { __mockSend } = await import("@aws-sdk/client-comprehend"); + + __mockSend.mockResolvedValueOnce({ + Entities: [ + { Type: "EMAIL", Score: 0.99, BeginOffset: 0, EndOffset: 17 }, + { Type: "PHONE", Score: 0.95, BeginOffset: 18, EndOffset: 30 }, + ], + }); + + const redactor = new ComprehendRedactor({ + accessKeyId: "test-key", + secretAccessKey: "test-secret", + }); + + const result = await redactor.redact("user@example.com 123-456-7890"); + + expect(result.redacted).toContain("[EMAIL]"); + expect(result.redacted).toContain("[PHONE]"); + expect(result.piiEntities).toHaveLength(2); + expect(result.piiEntities[0].type).toBe("EMAIL"); + expect(result.piiEntities[1].type).toBe("PHONE"); + }); + + it("should filter by PII types", async () => { + const { __mockSend } = await import("@aws-sdk/client-comprehend"); + + __mockSend.mockResolvedValueOnce({ + Entities: [ + { Type: "EMAIL", Score: 0.99, BeginOffset: 0, EndOffset: 17 }, + { Type: "PHONE", Score: 0.95, BeginOffset: 18, EndOffset: 30 }, + ], + }); + + const redactor = new ComprehendRedactor({}); + const result = await redactor.redact("user@example.com 123-456-7890", ["PHONE"]); + + expect(result.redacted).toBe("user@example.com [PHONE]"); + expect(result.piiEntities).toHaveLength(1); + expect(result.piiEntities[0].type).toBe("PHONE"); + }); + + it("should emit redacted text via observable", async () => { + const { __mockSend } = await import("@aws-sdk/client-comprehend"); + + __mockSend.mockResolvedValueOnce({ + Entities: [ + { Type: "EMAIL", Score: 0.99, BeginOffset: 0, EndOffset: 17 }, + ], + }); + + const redactor = new ComprehendRedactor({}); + const result = await new Promise((resolve, reject) => { + redactor.redact$("user@example.com").subscribe({ + next: resolve, + error: reject, + }); + }); + + expect(result.redacted).toContain("[EMAIL]"); + }); + + it("should chain with mask$ observable", async () => { + const { __mockSend } = await import("@aws-sdk/client-comprehend"); + const { of } = await import("rxjs"); + + __mockSend.mockResolvedValueOnce({ + Entities: [ + { Type: "EMAIL", Score: 0.99, BeginOffset: 0, EndOffset: 17 }, + ], + }); + + const redactor = new ComprehendRedactor({}); + const source$ = of("contact: user@example.com"); + + const result = await new Promise((resolve, reject) => { + redactor.mask$(source$).subscribe({ + next: resolve, + error: reject, + }); + }); + + expect(result).toContain("[EMAIL]"); + expect(result).not.toContain("user@example.com"); + }); +}); diff --git a/JS/edgechains/examples/pii-redaction/src/index.ts b/JS/edgechains/examples/pii-redaction/src/index.ts new file mode 100644 index 00000000..d72d54f8 --- /dev/null +++ b/JS/edgechains/examples/pii-redaction/src/index.ts @@ -0,0 +1,34 @@ +import { OpenAI } from "@arakoodev/edgechains.js"; +import { ComprehendRedactor } from "@arakoodev/edgechains.js"; +import { from } from "rxjs"; + +async function main() { + const llm = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); + const redactor = new ComprehendRedactor({ + region: process.env.AWS_REGION || "us-east-1", + }); + + const userPrompt = "My email is john.doe@example.com and my SSN is 123-45-6789. What services do you offer?"; + + // 1. Redact PII first + const { redacted } = await redactor.redact(userPrompt); + console.log("Original:", userPrompt); + console.log("Redacted:", redacted); + + // 2. Send redacted prompt to LLM + const response = await llm.chat({ prompt: redacted }); + console.log("LLM Response:", response.content); + + // 3. Observable chaining example + const source$ = from([ + "Contact me at jane@test.com or 555-0100", + "My credit card is 4111-1111-1111-1111", + ]); + + redactor.mask$(source$).subscribe({ + next: (masked) => console.log("Masked:", masked), + complete: () => console.log("Done"), + }); +} + +main().catch(console.error);