Find lapsed, registerable domains in adjacent industries — no new vendor

src/client/features/expired-domains/ExpiredDomainsPanel.tsx

@@ -281,6 +281,42 @@ export function ExpiredDomainsPanel({
               </Button>
             ) : null}
 
+            {result.acquirable ? (
+              <div className="mt-2 flex flex-col gap-2 border-t border-base-300 pt-3">
+                <p className="text-xs uppercase tracking-wide text-base-content/60">
+                  Available to register
+                </p>
+                {result.acquirable.rows.length > 0 ? (
+                  <>
+                    <p className="text-xs text-base-content/60">
+                      Lapsed domains in your industry and next to it, free to
+                      register right now.
+                    </p>
+                    <ul className="flex flex-col gap-1">
+                      {result.acquirable.rows.map((row) => (
+                        <li key={row.domain} className="font-medium">
+                          {row.domain}
+                          {row.hadHistory === null ? (
+                            <span className="ml-2 text-xs font-normal text-base-content/60">
+                              (archive check inconclusive)
+                            </span>
+                          ) : null}
+                        </li>
+                      ))}
+                    </ul>
+                  </>
+                ) : (
+                  // Same rule as the table above: say what was examined rather
+                  // than rendering an empty space.
+                  <p className="text-base-content/70">
+                    Generated {result.acquirable.summary.generated} names,{" "}
+                    {result.acquirable.summary.hadHistory} of which ever hosted
+                    a site — none are available to register.
+                  </p>
+                )}
+              </div>
+            ) : null}
+
             {result.sourcesSkipped.length > 0 ? (
               // The bug this fixes: a source that returned nothing was counted
               // as searched, so a run on a project with no competitors reported

src/server/features/expired-domains/acquirableDomains.test.ts

@@ -0,0 +1,169 @@
+import { describe, expect, it, vi } from "vitest";
+import {
+  findAcquirableDomains,
+  MAX_AVAILABILITY_CHECKS,
+} from "@/server/features/expired-domains/acquirableDomains";
+
+const CACHE = {
+  get: () => Promise.resolve(null),
+  put: () => Promise.resolve(),
+};
+
+const BASE = {
+  keywords: ["vending machines dallas", "breakroom services"],
+  profileText: "",
+  adjacentTerms: ["snack", "nutrition"],
+  exclude: ["deliotx.com"],
+  cache: CACHE,
+  limit: 20,
+};
+
+describe("findAcquirableDomains", () => {
+  // The cost ordering the whole feature depends on: Wayback is free, so it
+  // filters first and availability is only paid for on names that had a site.
+  it("only pays for availability on names that were archived", async () => {
+    const archived = vi.fn((domain: string) =>
+      Promise.resolve(domain.startsWith("snack")),
+    );
+    const available = vi.fn().mockResolvedValue(true);
+
+    const result = await findAcquirableDomains({
+      ...BASE,
+      hadArchivedSite: archived,
+      resolveAvailability: available,
+    });
+
+    expect(archived.mock.calls.length).toBeGreaterThan(
+      available.mock.calls.length,
+    );
+    for (const [domain] of available.mock.calls) {
+      expect(String(domain).startsWith("snack")).toBe(true);
+    }
+    expect(result.rows.every((row) => row.domain.startsWith("snack"))).toBe(
+      true,
+    );
+  });
+
+  it("surfaces only domains that are both archived and available", async () => {
+    const result = await findAcquirableDomains({
+      ...BASE,
+      hadArchivedSite: () => Promise.resolve(true),
+      resolveAvailability: (domain: string) =>
+        Promise.resolve(domain.startsWith("nutrition")),
+    });
+
+    expect(result.rows.length).toBeGreaterThan(0);
+    expect(result.rows.every((row) => row.domain.startsWith("nutrition"))).toBe(
+      true,
+    );
+  });
+
+  // An unknown Wayback answer must not be treated as "never existed", which
+  // would silently drop a real target.
+  it("does not discard a name whose archive check was inconclusive", async () => {
+    const available = vi.fn().mockResolvedValue(true);
+    await findAcquirableDomains({
+      ...BASE,
+      hadArchivedSite: () => Promise.resolve(null),
+      resolveAvailability: available,
+    });
+
+    expect(available).toHaveBeenCalled();
+  });
+
+  it("reports what it checked so an empty result is legible", async () => {
+    const result = await findAcquirableDomains({
+      ...BASE,
+      hadArchivedSite: () => Promise.resolve(false),
+      resolveAvailability: vi.fn(),
+    });
+
+    expect(result.rows).toEqual([]);
+    expect(result.summary.generated).toBeGreaterThan(0);
+    expect(result.summary.hadHistory).toBe(0);
+    expect(result.summary.availabilityChecked).toBe(0);
+  });
+
+  it("spends nothing when there is no vocabulary to build from", async () => {
+    const archived = vi.fn();
+    const available = vi.fn();
+
+    const result = await findAcquirableDomains({
+      ...BASE,
+      keywords: [],
+      profileText: "",
+      adjacentTerms: [],
+      hadArchivedSite: archived,
+      resolveAvailability: available,
+    });
+
+    expect(result.rows).toEqual([]);
+    expect(archived).not.toHaveBeenCalled();
+    expect(available).not.toHaveBeenCalled();
+  });
+
+  it("never suggests an excluded domain", async () => {
+    const result = await findAcquirableDomains({
+      ...BASE,
+      exclude: ["snackvending.com"],
+      hadArchivedSite: () => Promise.resolve(true),
+      resolveAvailability: () => Promise.resolve(true),
+    });
+
+    expect(result.rows.map((row) => row.domain)).not.toContain(
+      "snackvending.com",
+    );
+  });
+});
+
+describe("findAcquirableDomains spend guards", () => {
+  // The hazard: archive.org rate-limits (observed 429 in practice). Every
+  // archive check then returns null, and if inconclusive names fall through
+  // freely, a run silently bills availability for the WHOLE generated set.
+  it("caps billed availability checks when the archive service is down", async () => {
+    const available = vi.fn().mockResolvedValue(false);
+
+    const result = await findAcquirableDomains({
+      ...BASE,
+      limit: 60,
+      hadArchivedSite: () => Promise.resolve(null),
+      resolveAvailability: available,
+    });
+
+    expect(available.mock.calls.length).toBeLessThanOrEqual(
+      MAX_AVAILABILITY_CHECKS,
+    );
+    expect(result.summary.archiveUnavailable).toBe(true);
+  });
+
+  it("spends the budget on confirmed history before inconclusive names", async () => {
+    const checked: string[] = [];
+    await findAcquirableDomains({
+      ...BASE,
+      limit: 60,
+      // Only these two are confirmed; everything else is inconclusive.
+      hadArchivedSite: (domain: string) =>
+        Promise.resolve(domain.startsWith("snack") ? true : null),
+      resolveAvailability: (domain: string) => {
+        checked.push(domain);
+        return Promise.resolve(false);
+      },
+    });
+
+    const firstConfirmed = checked.findIndex((d) => d.startsWith("snack"));
+    const firstInconclusive = checked.findIndex((d) => !d.startsWith("snack"));
+    expect(firstConfirmed).toBeGreaterThanOrEqual(0);
+    if (firstInconclusive >= 0) {
+      expect(firstConfirmed).toBeLessThan(firstInconclusive);
+    }
+  });
+
+  it("does not flag the archive as down when it answered", async () => {
+    const result = await findAcquirableDomains({
+      ...BASE,
+      hadArchivedSite: () => Promise.resolve(false),
+      resolveAvailability: vi.fn(),
+    });
+    expect(result.summary.archiveUnavailable).toBe(false);
+  });
+});

src/server/features/expired-domains/acquirableDomains.ts

@@ -0,0 +1,162 @@
+import type { resolveDomainAvailability } from "@/server/lib/apiverve/domainAvailability";
+import type { ExpirationCache } from "@/server/lib/apiverve/domainExpiration";
+import type { hadArchivedSite } from "@/server/lib/wayback";
+import {
+  buildDomainNameCandidates,
+  deriveSeedTerms,
+} from "@/shared/domainNameCandidates";
+
+/**
+ * Domains in the client's industry that are lapsed and registerable today.
+ *
+ * This is the half of the finder that answers "show me expired domains I can
+ * buy", as opposed to the graph sources, which surface domains in the client's
+ * link neighbourhood that are mostly still owned.
+ *
+ * The cost ordering is the design:
+ *
+ *   1. generate names from the industry's vocabulary   -- free
+ *   2. ask Wayback which ever hosted a site            -- free, no key
+ *   3. check availability only for those               -- 5 credits each
+ *
+ * Doing it the other way round would bill for every generated name, most of
+ * which were never registered by anyone. Step 2 is what makes a result an
+ * EXPIRED domain rather than an unregistered string.
+ */
+
+/** Shapes that read like a real small-business domain. */
+const MODIFIERS = [
+  "supply",
+  "hub",
+  "direct",
+  "group",
+  "co",
+  "works",
+  "depot",
+  "partners",
+];
+const TLDS = ["com", "net"];
+/** Wayback is free but public, and it DOES rate-limit (429 observed). Stay
+ *  polite and bounded. */
+const ARCHIVE_CONCURRENCY = 3;
+
+/**
+ * Hard ceiling on billed availability checks per run, regardless of how many
+ * names survive the archive filter.
+ *
+ * This exists because of a real failure mode: when archive.org rate-limits,
+ * every check returns `null`. Inconclusive names are still worth checking --
+ * discarding them would drop real targets over someone else's outage -- but
+ * without a ceiling a throttled archive turns a run into a full-price sweep of
+ * every generated name. The cap is the difference between a degraded run and a
+ * surprise bill.
+ */
+export const MAX_AVAILABILITY_CHECKS = 25;
+
+type AcquirableRow = {
+  domain: string;
+  /** null when the archive lookup could not answer. */
+  hadHistory: boolean | null;
+};
+
+type AcquirableSummary = {
+  generated: number;
+  hadHistory: number;
+  availabilityChecked: number;
+  /** True when the archive service answered for nothing -- results are weaker. */
+  archiveUnavailable: boolean;
+};
+
+export async function findAcquirableDomains(input: {
+  keywords: string[];
+  profileText: string;
+  /** Neighbouring-industry words. Where the reach beyond the vertical comes from. */
+  adjacentTerms: string[];
+  exclude: string[];
+  cache: ExpirationCache;
+  /** Spend guard: at most this many names reach the availability step. */
+  limit: number;
+  hadArchivedSite: typeof hadArchivedSite;
+  resolveAvailability: typeof resolveDomainAvailability;
+}): Promise<{ rows: AcquirableRow[]; summary: AcquirableSummary }> {
+  const heads = deriveSeedTerms(input.keywords, input.profileText);
+  const names = buildDomainNameCandidates({
+    heads,
+    adjacents: input.adjacentTerms,
+    modifiers: MODIFIERS,
+    tlds: TLDS,
+    exclude: input.exclude,
+    limit: input.limit,
+  });
+
+  if (names.length === 0) {
+    return {
+      rows: [],
+      summary: {
+        generated: 0,
+        hadHistory: 0,
+        availabilityChecked: 0,
+        archiveUnavailable: false,
+      },
+    };
+  }
+
+  // Step 2, free: which of these were ever real sites.
+  const confirmed: AcquirableRow[] = [];
+  const inconclusive: AcquirableRow[] = [];
+  let answered = 0;
+  for (let i = 0; i < names.length; i += ARCHIVE_CONCURRENCY) {
+    const batch = names.slice(i, i + ARCHIVE_CONCURRENCY);
+    const settled = await Promise.all(
+      batch.map(async (domain) => {
+        try {
+          return {
+            domain,
+            hadHistory: await input.hadArchivedSite(domain, input.cache),
+          };
+        } catch {
+          return { domain, hadHistory: null };
+        }
+      }),
+    );
+    for (const entry of settled) {
+      if (entry.hadHistory !== null) answered += 1;
+      // `null` is inconclusive, NOT "never existed" -- kept, because discarding
+      // it would drop a real target over a free service's hiccup. But it goes
+      // in a second queue: confirmed history is spent on first.
+      if (entry.hadHistory === true) confirmed.push(entry);
+      else if (entry.hadHistory === null) inconclusive.push(entry);
+    }
+  }
+
+  const archiveUnavailable = answered === 0;
+  // Confirmed first, then inconclusive only while budget remains.
+  const toCheck = [...confirmed, ...inconclusive].slice(
+    0,
+    MAX_AVAILABILITY_CHECKS,
+  );
+
+  // Step 3, billed: only now, and only for survivors.
+  const rows: AcquirableRow[] = [];
+  let availabilityChecked = 0;
+  for (const entry of toCheck) {
+    availabilityChecked += 1;
+    let available: boolean | null = null;
+    try {
+      available = await input.resolveAvailability(entry.domain, input.cache);
+    } catch {
+      available = null;
+    }
+    if (available === true) rows.push(entry);
+  }
+
+  return {
+    rows,
+    summary: {
+      generated: names.length,
+      hadHistory: confirmed.length,
+      availabilityChecked,
+      archiveUnavailable,
+    },
+  };
+}