From ce19dd3b5b8350285e4c7d0a8602a7f3e745fe3c Mon Sep 17 00:00:00 2001 From: Naman Rusia Date: Thu, 24 Sep 2026 16:34:18 -0400 Subject: [PATCH 01/16] Start database core tables and schema (#3) From 3bb910f934a5b7f0062eba02ee470b7787b813f0 Mon Sep 17 00:00:00 2001 From: Naman Rusia Date: Fri, 25 Sep 2026 17:49:53 -0400 Subject: [PATCH 02/16] Add initial database schema and migration (#3) Entities and generated migration for users, imports, daily/hourly/scheduled observations, daily records and audit log, plus docs/database-schema.md mapping each column to its source cell in the example sheets. --- apps/backend/src/audit/audit-log.entity.ts | 37 +++++ apps/backend/src/imports/import.entity.ts | 32 ++++ apps/backend/src/imports/types.ts | 4 + .../migrations/1790372888092-init_schema.ts | 83 ++++++++++ .../observations/daily-observation.entity.ts | 73 +++++++++ .../observations/hourly-observation.entity.ts | 54 +++++++ .../scheduled-observation.entity.ts | 55 +++++++ .../src/records/daily-record.entity.ts | 35 ++++ apps/backend/src/records/types.ts | 7 + docs/database-schema.md | 149 ++++++++++++++++++ 10 files changed, 529 insertions(+) create mode 100644 apps/backend/src/audit/audit-log.entity.ts create mode 100644 apps/backend/src/imports/import.entity.ts create mode 100644 apps/backend/src/imports/types.ts create mode 100644 apps/backend/src/migrations/1790372888092-init_schema.ts create mode 100644 apps/backend/src/observations/daily-observation.entity.ts create mode 100644 apps/backend/src/observations/hourly-observation.entity.ts create mode 100644 apps/backend/src/observations/scheduled-observation.entity.ts create mode 100644 apps/backend/src/records/daily-record.entity.ts create mode 100644 apps/backend/src/records/types.ts create mode 100644 docs/database-schema.md diff --git a/apps/backend/src/audit/audit-log.entity.ts b/apps/backend/src/audit/audit-log.entity.ts new file mode 100644 index 000000000..689a58af3 --- /dev/null +++ b/apps/backend/src/audit/audit-log.entity.ts @@ -0,0 +1,37 @@ +import { + Column, + CreateDateColumn, + Entity, + ManyToOne, + PrimaryGeneratedColumn, +} from 'typeorm'; + +import { User } from '../users/user.entity'; + +@Entity('audit_log') +export class AuditLog { + @PrimaryGeneratedColumn() + id: number; + + @Column({ type: 'text' }) + tableName: string; + + // Primary key of the edited row, e.g. { "date": "2026-07-01", "hour": 0 }. + @Column({ type: 'jsonb' }) + rowKey: Record; + + @Column({ type: 'text' }) + columnName: string; + + @Column({ type: 'text', nullable: true }) + oldValue: string; + + @Column({ type: 'text', nullable: true }) + newValue: string; + + @ManyToOne(() => User, { nullable: false }) + changedBy: User; + + @CreateDateColumn({ type: 'timestamptz' }) + changedAt: Date; +} diff --git a/apps/backend/src/imports/import.entity.ts b/apps/backend/src/imports/import.entity.ts new file mode 100644 index 000000000..0c5663eac --- /dev/null +++ b/apps/backend/src/imports/import.entity.ts @@ -0,0 +1,32 @@ +import { + Column, + CreateDateColumn, + Entity, + ManyToOne, + PrimaryGeneratedColumn, +} from 'typeorm'; + +import { User } from '../users/user.entity'; +import { ImportStatus } from './types'; + +@Entity() +export class Import { + @PrimaryGeneratedColumn() + id: number; + + @Column({ type: 'text' }) + fileName: string; + + @ManyToOne(() => User, { nullable: false }) + uploadedBy: User; + + @CreateDateColumn({ type: 'timestamptz' }) + uploadedAt: Date; + + @Column({ type: 'enum', enum: ImportStatus }) + status: ImportStatus; + + // Rows that failed validation, with the reason for each. + @Column({ type: 'jsonb', nullable: true }) + errors: unknown; +} diff --git a/apps/backend/src/imports/types.ts b/apps/backend/src/imports/types.ts new file mode 100644 index 000000000..16f73880b --- /dev/null +++ b/apps/backend/src/imports/types.ts @@ -0,0 +1,4 @@ +export enum ImportStatus { + SUCCEEDED = 'succeeded', + FAILED = 'failed', +} diff --git a/apps/backend/src/migrations/1790372888092-init_schema.ts b/apps/backend/src/migrations/1790372888092-init_schema.ts new file mode 100644 index 000000000..de5dd8905 --- /dev/null +++ b/apps/backend/src/migrations/1790372888092-init_schema.ts @@ -0,0 +1,83 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class InitSchema1790372888092 implements MigrationInterface { + name = 'InitSchema1790372888092'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "users" ("id" integer NOT NULL, "status" character varying NOT NULL, "firstName" character varying NOT NULL, "lastName" character varying NOT NULL, "email" character varying NOT NULL, CONSTRAINT "PK_a3ffb1c0c8416b9fc6f907b7433" PRIMARY KEY ("id"))`, + ); + await queryRunner.query( + `CREATE TYPE "public"."imports_status_enum" AS ENUM('succeeded', 'failed')`, + ); + await queryRunner.query( + `CREATE TABLE "imports" ("id" SERIAL NOT NULL, "fileName" text NOT NULL, "uploadedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "status" "public"."imports_status_enum" NOT NULL, "errors" jsonb, "uploadedBy" integer NOT NULL, CONSTRAINT "PK_ea10c62f5eb1d75e83d8b5225db" PRIMARY KEY ("id"))`, + ); + await queryRunner.query( + `CREATE TABLE "daily_observations" ("date" date NOT NULL, "maxTempF" smallint, "minTempF" smallint, "precipIn" numeric(5,2), "precipTrace" boolean NOT NULL DEFAULT false, "snowfallIn" numeric(4,1), "snowDepthIn" numeric(4,1), "sunshineMin" smallint, "sunshinePct" smallint, "fastestMileMph" smallint, "fastestMileDir" character varying(3), "fastestMileTime" TIME, "peakGustKts" smallint, "peakGustDir" character varying(3), "peakGustTime" TIME, "avgStationPressureMb" numeric(5,1), "sunrise" TIME, "sunset" TIME, "observerInitials" text, "remarks" text, "import" integer, CONSTRAINT "PK_a7cec69d330c77094266383c7f7" PRIMARY KEY ("date"))`, + ); + await queryRunner.query( + `CREATE TABLE "hourly_observations" ("date" date NOT NULL, "hour" smallint NOT NULL, "tempF" smallint, "precipIn" numeric(4,2), "precipTrace" boolean NOT NULL DEFAULT false, "windDir" character varying(3), "windSpeedMph" smallint, "sunshineMin" smallint, "skyCover" smallint, "visibilityMi" numeric(5,2), "presentWeather" text, "humidityPct" smallint, "mountainVis" text, "remarks" text, CONSTRAINT "PK_91c9c1470c1f91dd1e3dc45de22" PRIMARY KEY ("date", "hour"))`, + ); + await queryRunner.query( + `CREATE TABLE "scheduled_observations" ("date" date NOT NULL, "obsTime" TIME NOT NULL, "stationPressureIn" numeric(5,3), "dryBulbF" numeric(4,1), "wetBulbF" numeric(4,1), "dewpointF" smallint, "humidityPct" smallint, "maxTempF" smallint, "minTempF" smallint, "precipIn" numeric(4,2), "precipTrace" boolean NOT NULL DEFAULT false, "snowfallIn" numeric(4,1), "snowDepthIn" numeric(4,1), "vaporPressureMb" numeric(4,1), CONSTRAINT "PK_ab5b2148436c377435699de1c4c" PRIMARY KEY ("date", "obsTime"))`, + ); + await queryRunner.query( + `CREATE TYPE "public"."daily_records_element_enum" AS ENUM('high', 'low', 'precip', 'snow', 'peak_gust')`, + ); + await queryRunner.query( + `CREATE TABLE "daily_records" ("month" smallint NOT NULL, "day" smallint NOT NULL, "element" "public"."daily_records_element_enum" NOT NULL, "value" numeric(5,2), "years" smallint array NOT NULL, "windDir" character varying(3), "isEstimated" boolean NOT NULL DEFAULT false, "import" integer, CONSTRAINT "PK_54cdb09a0715ffeddbb2193b0ac" PRIMARY KEY ("month", "day", "element"))`, + ); + await queryRunner.query( + `CREATE TABLE "audit_log" ("id" SERIAL NOT NULL, "tableName" text NOT NULL, "rowKey" jsonb NOT NULL, "columnName" text NOT NULL, "oldValue" text, "newValue" text, "changedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "changedBy" integer NOT NULL, CONSTRAINT "PK_07fefa57f7f5ab8fc3f52b3ed0b" PRIMARY KEY ("id"))`, + ); + await queryRunner.query( + `ALTER TABLE "imports" ADD CONSTRAINT "FK_c80e47438b6c9719cab60d65860" FOREIGN KEY ("uploadedBy") REFERENCES "users"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "daily_observations" ADD CONSTRAINT "FK_dcd7b8d6de045fa97944a212f0c" FOREIGN KEY ("import") REFERENCES "imports"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "hourly_observations" ADD CONSTRAINT "FK_04b56339d6c0b00fdf7f1c561c3" FOREIGN KEY ("date") REFERENCES "daily_observations"("date") ON DELETE CASCADE ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "scheduled_observations" ADD CONSTRAINT "FK_7576980c1c64dbea8fff5ad53e0" FOREIGN KEY ("date") REFERENCES "daily_observations"("date") ON DELETE CASCADE ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "daily_records" ADD CONSTRAINT "FK_63812be4d466feba48443349a18" FOREIGN KEY ("import") REFERENCES "imports"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "audit_log" ADD CONSTRAINT "FK_3c20bc116a9a51bab8206ddca1e" FOREIGN KEY ("changedBy") REFERENCES "users"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "audit_log" DROP CONSTRAINT "FK_3c20bc116a9a51bab8206ddca1e"`, + ); + await queryRunner.query( + `ALTER TABLE "daily_records" DROP CONSTRAINT "FK_63812be4d466feba48443349a18"`, + ); + await queryRunner.query( + `ALTER TABLE "scheduled_observations" DROP CONSTRAINT "FK_7576980c1c64dbea8fff5ad53e0"`, + ); + await queryRunner.query( + `ALTER TABLE "hourly_observations" DROP CONSTRAINT "FK_04b56339d6c0b00fdf7f1c561c3"`, + ); + await queryRunner.query( + `ALTER TABLE "daily_observations" DROP CONSTRAINT "FK_dcd7b8d6de045fa97944a212f0c"`, + ); + await queryRunner.query( + `ALTER TABLE "imports" DROP CONSTRAINT "FK_c80e47438b6c9719cab60d65860"`, + ); + await queryRunner.query(`DROP TABLE "audit_log"`); + await queryRunner.query(`DROP TABLE "daily_records"`); + await queryRunner.query(`DROP TYPE "public"."daily_records_element_enum"`); + await queryRunner.query(`DROP TABLE "scheduled_observations"`); + await queryRunner.query(`DROP TABLE "hourly_observations"`); + await queryRunner.query(`DROP TABLE "daily_observations"`); + await queryRunner.query(`DROP TABLE "imports"`); + await queryRunner.query(`DROP TYPE "public"."imports_status_enum"`); + await queryRunner.query(`DROP TABLE "users"`); + } +} diff --git a/apps/backend/src/observations/daily-observation.entity.ts b/apps/backend/src/observations/daily-observation.entity.ts new file mode 100644 index 000000000..8bd338d61 --- /dev/null +++ b/apps/backend/src/observations/daily-observation.entity.ts @@ -0,0 +1,73 @@ +import { Column, Entity, ManyToOne, PrimaryColumn } from 'typeorm'; + +import { Import } from '../imports/import.entity'; + +// Numeric columns come back from pg as strings to avoid float rounding. +@Entity('daily_observations') +export class DailyObservation { + @PrimaryColumn({ type: 'date' }) + date: string; + + @Column({ type: 'smallint', nullable: true }) + maxTempF: number; + + @Column({ type: 'smallint', nullable: true }) + minTempF: number; + + @Column({ type: 'numeric', precision: 5, scale: 2, nullable: true }) + precipIn: string; + + // The sheet writes "T" for a trace; precipIn is 0 on those days. + @Column({ default: false }) + precipTrace: boolean; + + @Column({ type: 'numeric', precision: 4, scale: 1, nullable: true }) + snowfallIn: string; + + @Column({ type: 'numeric', precision: 4, scale: 1, nullable: true }) + snowDepthIn: string; + + @Column({ type: 'smallint', nullable: true }) + sunshineMin: number; + + @Column({ type: 'smallint', nullable: true }) + sunshinePct: number; + + @Column({ type: 'smallint', nullable: true }) + fastestMileMph: number; + + @Column({ type: 'varchar', length: 3, nullable: true }) + fastestMileDir: string; + + // All times are Eastern Standard Time year-round, as in the sheet. + @Column({ type: 'time', nullable: true }) + fastestMileTime: string; + + // Knots, as recorded in the sheet. + @Column({ type: 'smallint', nullable: true }) + peakGustKts: number; + + @Column({ type: 'varchar', length: 3, nullable: true }) + peakGustDir: string; + + @Column({ type: 'time', nullable: true }) + peakGustTime: string; + + @Column({ type: 'numeric', precision: 5, scale: 1, nullable: true }) + avgStationPressureMb: string; + + @Column({ type: 'time', nullable: true }) + sunrise: string; + + @Column({ type: 'time', nullable: true }) + sunset: string; + + @Column({ type: 'text', nullable: true }) + observerInitials: string; + + @Column({ type: 'text', nullable: true }) + remarks: string; + + @ManyToOne(() => Import, { nullable: true }) + import: Import; +} diff --git a/apps/backend/src/observations/hourly-observation.entity.ts b/apps/backend/src/observations/hourly-observation.entity.ts new file mode 100644 index 000000000..ba00bb9b2 --- /dev/null +++ b/apps/backend/src/observations/hourly-observation.entity.ts @@ -0,0 +1,54 @@ +import { Column, Entity, JoinColumn, ManyToOne, PrimaryColumn } from 'typeorm'; + +import { DailyObservation } from './daily-observation.entity'; + +@Entity('hourly_observations') +export class HourlyObservation { + @PrimaryColumn({ type: 'date' }) + date: string; + + // Start of the hour: the sheet's "00-01" row is hour 0. + @PrimaryColumn({ type: 'smallint' }) + hour: number; + + @ManyToOne(() => DailyObservation, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'date' }) + day: DailyObservation; + + @Column({ type: 'smallint', nullable: true }) + tempF: number; + + @Column({ type: 'numeric', precision: 4, scale: 2, nullable: true }) + precipIn: string; + + @Column({ default: false }) + precipTrace: boolean; + + @Column({ type: 'varchar', length: 3, nullable: true }) + windDir: string; + + @Column({ type: 'smallint', nullable: true }) + windSpeedMph: number; + + @Column({ type: 'smallint', nullable: true }) + sunshineMin: number; + + // Eighths of sky covered; only filled at observation hours. + @Column({ type: 'smallint', nullable: true }) + skyCover: number; + + @Column({ type: 'numeric', precision: 5, scale: 2, nullable: true }) + visibilityMi: string; + + @Column({ type: 'text', nullable: true }) + presentWeather: string; + + @Column({ type: 'smallint', nullable: true }) + humidityPct: number; + + @Column({ type: 'text', nullable: true }) + mountainVis: string; + + @Column({ type: 'text', nullable: true }) + remarks: string; +} diff --git a/apps/backend/src/observations/scheduled-observation.entity.ts b/apps/backend/src/observations/scheduled-observation.entity.ts new file mode 100644 index 000000000..3cb9cd570 --- /dev/null +++ b/apps/backend/src/observations/scheduled-observation.entity.ts @@ -0,0 +1,55 @@ +import { Column, Entity, JoinColumn, ManyToOne, PrimaryColumn } from 'typeorm'; + +import { DailyObservation } from './daily-observation.entity'; + +// Readings taken by hand at 0700, 0800, 1000, 1300 and 1900. +@Entity('scheduled_observations') +export class ScheduledObservation { + @PrimaryColumn({ type: 'date' }) + date: string; + + @PrimaryColumn({ type: 'time' }) + obsTime: string; + + @ManyToOne(() => DailyObservation, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'date' }) + day: DailyObservation; + + @Column({ type: 'numeric', precision: 5, scale: 3, nullable: true }) + stationPressureIn: string; + + @Column({ type: 'numeric', precision: 4, scale: 1, nullable: true }) + dryBulbF: string; + + @Column({ type: 'numeric', precision: 4, scale: 1, nullable: true }) + wetBulbF: string; + + @Column({ type: 'smallint', nullable: true }) + dewpointF: number; + + @Column({ type: 'smallint', nullable: true }) + humidityPct: number; + + // The max/min/precip/snow columns are only filled on the 0700 row and + // cover the 24 hours ending at 0700. + @Column({ type: 'smallint', nullable: true }) + maxTempF: number; + + @Column({ type: 'smallint', nullable: true }) + minTempF: number; + + @Column({ type: 'numeric', precision: 4, scale: 2, nullable: true }) + precipIn: string; + + @Column({ default: false }) + precipTrace: boolean; + + @Column({ type: 'numeric', precision: 4, scale: 1, nullable: true }) + snowfallIn: string; + + @Column({ type: 'numeric', precision: 4, scale: 1, nullable: true }) + snowDepthIn: string; + + @Column({ type: 'numeric', precision: 4, scale: 1, nullable: true }) + vaporPressureMb: string; +} diff --git a/apps/backend/src/records/daily-record.entity.ts b/apps/backend/src/records/daily-record.entity.ts new file mode 100644 index 000000000..398d30ba3 --- /dev/null +++ b/apps/backend/src/records/daily-record.entity.ts @@ -0,0 +1,35 @@ +import { Column, Entity, ManyToOne, PrimaryColumn } from 'typeorm'; + +import { Import } from '../imports/import.entity'; +import { RecordElement } from './types'; + +// All-time record per calendar day, one row per element so a "record broken" +// check is a single lookup. Joins to observations on month and day, not a FK. +@Entity('daily_records') +export class DailyRecord { + @PrimaryColumn({ type: 'smallint' }) + month: number; + + @PrimaryColumn({ type: 'smallint' }) + day: number; + + @PrimaryColumn({ type: 'enum', enum: RecordElement }) + element: RecordElement; + + // Null when the sheet says "None". + @Column({ type: 'numeric', precision: 5, scale: 2, nullable: true }) + value: string; + + // Ties list every year the record was set. + @Column({ type: 'smallint', array: true }) + years: number[]; + + @Column({ type: 'varchar', length: 3, nullable: true }) + windDir: string; + + @Column({ default: false }) + isEstimated: boolean; + + @ManyToOne(() => Import, { nullable: true }) + import: Import; +} diff --git a/apps/backend/src/records/types.ts b/apps/backend/src/records/types.ts new file mode 100644 index 000000000..ee6d65047 --- /dev/null +++ b/apps/backend/src/records/types.ts @@ -0,0 +1,7 @@ +export enum RecordElement { + HIGH = 'high', + LOW = 'low', + PRECIP = 'precip', + SNOW = 'snow', + PEAK_GUST = 'peak_gust', +} diff --git a/docs/database-schema.md b/docs/database-schema.md new file mode 100644 index 000000000..b01547eb2 --- /dev/null +++ b/docs/database-schema.md @@ -0,0 +1,149 @@ +# Database schema + +Draft for [#3](https://github.com/Code-4-Community/bho/issues/3). PostgreSQL on AWS RDS. Created by `apps/backend/src/migrations/1790372888092-init_schema.ts`. + +Sources: + +- **Daily sheet**: [DAILY - 07. July2026.xlsx](https://docs.google.com/spreadsheets/d/1cWloMpYHZDGkUFulmeW0g1bY3J97Ts4b/edit), tab `Sheet1`. One 60-row block per day. Cell references are for the July 1 block; add 60 rows for each later day. +- **Historical sheet**: [BHO_WeatherRecords_Master_TestData.xlsx](https://docs.google.com/spreadsheets/d/1WI8CxTJ7edRajo3JHZX0WmTklvCqRy5x/edit), tab `All Daily Records`, rows 4-369. + +Column names are camelCase because the repo's naming strategy uses property names as column names. + +## Relations + +```mermaid +erDiagram + users ||--o{ imports : uploads + users ||--o{ audit_log : edits + imports ||--o{ daily_observations : creates + imports ||--o{ daily_records : creates + daily_observations ||--|{ hourly_observations : has + daily_observations ||--|{ scheduled_observations : has +``` + +`daily_records` joins to observations on month and day, not a foreign key. + +## users + +The scaffold's existing `User` entity, unchanged: `id`, `status`, `firstName`, `lastName`, `email`. Linking to Cognito and dropping `status` (admin and staff are one role) belong to the auth work. + +## imports + +Source: the app, one row per uploaded spreadsheet. + +| Column | Type | Null | Note | +| ---------- | ----------- | ---- | ------------------------------------ | +| id | serial | N | PK | +| fileName | text | N | | +| uploadedBy | int | N | FK users | +| uploadedAt | timestamptz | N | Defaults to now | +| status | enum | N | `succeeded` or `failed` | +| errors | jsonb | Y | Flagged rows and the reason for each | + +## daily_observations + +Source: daily sheet, "Summary of Day" and nearby rows. PK `date`. + +| Column | Type | Null | Source | +| -------------------- | ------------ | ---- | ------------------------------------------- | +| date | date | N | J1 "Wednesday, July 1, 2026" | +| maxTempF | smallint | Y | A44 "24HR Max" | +| minTempF | smallint | Y | B44 "24HR Min" | +| precipIn | numeric(5,2) | Y | C44 "24HR Precip" | +| precipTrace | boolean | N | Our flag for when C44 is `T` (July 5: C284) | +| snowfallIn | numeric(4,1) | Y | E44 "Snowfall" | +| snowDepthIn | numeric(4,1) | Y | F44 "Dpth 0700" | +| sunshineMin | smallint | Y | G44 "Total Min." | +| sunshinePct | smallint | Y | H44 "% poss." (0.77 becomes 77) | +| fastestMileMph | smallint | Y | I44 | +| fastestMileDir | varchar(3) | Y | J44 | +| fastestMileTime | time | Y | K44 "0245E" (24-hour, EST) | +| peakGustKts | smallint | Y | L44 "20 WSW @ 0854E" (knots) | +| peakGustDir | varchar(3) | Y | L44 | +| peakGustTime | time | Y | L44 | +| avgStationPressureMb | numeric(5,1) | Y | E56 "Station Pressure" AVG | +| sunrise | time | Y | K29 | +| sunset | time | Y | K30 | +| observerInitials | text | Y | H59 "MFD/AKJ" | +| remarks | text | Y | G47-G49 "Additional Notes" | +| import | int | Y | FK imports | + +Not stored: average temperature, normal, departure, and degree days (rows 46-50). They can be computed. + +## hourly_observations + +Source: daily sheet, rows 4-27 ("00-01" to "23-24"). PK `(date, hour)`. Deleting a day deletes its hours. + +| Column | Type | Null | Source | +| -------------- | ------------ | ---- | -------------------------------------- | +| date | date | N | FK daily_observations | +| hour | smallint | N | A, start hour ("00-01" is 0) | +| tempF | smallint | Y | B "Temp (°F)" | +| precipIn | numeric(4,2) | Y | C "Precip" | +| precipTrace | boolean | N | Our flag for when C is `T` | +| windDir | varchar(3) | Y | D "Dir." | +| windSpeedMph | smallint | Y | E "Spd (mph)" | +| sunshineMin | smallint | Y | F "Sunshine (min)" | +| skyCover | smallint | Y | G "Sky Cover (0-8)" | +| visibilityMi | numeric(5,2) | Y | H "Lwst VSBL" ("1/8" becomes 0.125) | +| presentWeather | text | Y | I "Prsnt WX" ("R-F") | +| humidityPct | smallint | Y | J "Rel Hum." | +| mountainVis | text | Y | K "Mnts VSBL" ("1@1.5") | +| remarks | text | Y | L "Notes" | + +## scheduled_observations + +Source: daily sheet, "Scheduled Observations", rows 34-38. PK `(date, obsTime)`. Deleting a day deletes its readings. + +| Column | Type | Null | Source | +| ----------------- | ------------ | ---- | ---------------------------------------- | +| date | date | N | FK daily_observations | +| obsTime | time | N | A (700, 800, 1000, 1300, 1900) | +| stationPressureIn | numeric(5,3) | Y | B, 0700 and 1900 only | +| dryBulbF | numeric(4,1) | Y | C | +| wetBulbF | numeric(4,1) | Y | D | +| dewpointF | smallint | Y | E | +| humidityPct | smallint | Y | F (0.8 becomes 80) | +| maxTempF | smallint | Y | G, 0700 only, 24 hours ending 0700 | +| minTempF | smallint | Y | H, 0700 only | +| precipIn | numeric(4,2) | Y | I, 0700 only | +| precipTrace | boolean | N | Our flag for when I is `T` | +| snowfallIn | numeric(4,1) | Y | J, 0700 only | +| snowDepthIn | numeric(4,1) | Y | K, 0700 only | +| vaporPressureMb | numeric(4,1) | Y | L | + +## daily_records + +Source: historical sheet. Each sheet row becomes five rows, one per element. PK `(month, day, element)`. + +| Column | Type | Null | Source | +| ----------- | ------------ | ---- | --------------------------------------------------------- | +| month | smallint | N | "Month #" | +| day | smallint | N | "Day" | +| element | enum | N | `high`, `low`, `precip`, `snow`, `peak_gust` | +| value | numeric(5,2) | Y | "High °F", "Low °F", "Precip in", "Snow in", "Peak Gust mph"; null for "None" | +| years | smallint[] | N | "... Year(s)"; "2014, 1977" becomes `{2014,1977}` | +| windDir | varchar(3) | Y | "Wind Dir", peak_gust only | +| isEstimated | boolean | N | "Estimated?", peak_gust only | +| import | int | Y | FK imports | + +## audit_log + +Source: the app, one row per edited value. + +| Column | Type | Null | Note | +| ---------- | ----------- | ---- | ----------------------------- | +| id | serial | N | PK | +| tableName | text | N | | +| rowKey | jsonb | N | PK of the edited row | +| columnName | text | N | | +| oldValue | text | Y | | +| newValue | text | Y | | +| changedBy | int | N | FK users | +| changedAt | timestamptz | N | Defaults to now | + +## Not in this draft + +- `normals`: no normals data yet. +- `narratives`: no current source. +- `live_readings`, `ingestion_runs`: waiting on the WeatherLink API format. From db39a063e0139573664958a802aa264f5d4067fa Mon Sep 17 00:00:00 2001 From: Naman Rusia Date: Fri, 25 Sep 2026 18:10:44 -0400 Subject: [PATCH 03/16] Remove unclear users note from schema doc --- docs/database-schema.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/database-schema.md b/docs/database-schema.md index b01547eb2..613aae358 100644 --- a/docs/database-schema.md +++ b/docs/database-schema.md @@ -25,7 +25,7 @@ erDiagram ## users -The scaffold's existing `User` entity, unchanged: `id`, `status`, `firstName`, `lastName`, `email`. Linking to Cognito and dropping `status` (admin and staff are one role) belong to the auth work. +The scaffold's existing `User` entity, unchanged: `id`, `status`, `firstName`, `lastName`, `email`. ## imports From dbba10e209a87d91a9b0d0b6e5395aea47c71e88 Mon Sep 17 00:00:00 2001 From: Naman Rusia Date: Fri, 25 Sep 2026 18:18:41 -0400 Subject: [PATCH 04/16] Keep PR to the schema doc; remove entities and migration Add s3Key to imports in the doc. --- apps/backend/src/audit/audit-log.entity.ts | 37 --------- apps/backend/src/imports/import.entity.ts | 32 ------- apps/backend/src/imports/types.ts | 4 - .../migrations/1790372888092-init_schema.ts | 83 ------------------- .../observations/daily-observation.entity.ts | 73 ---------------- .../observations/hourly-observation.entity.ts | 54 ------------ .../scheduled-observation.entity.ts | 55 ------------ .../src/records/daily-record.entity.ts | 35 -------- apps/backend/src/records/types.ts | 7 -- docs/database-schema.md | 5 +- 10 files changed, 3 insertions(+), 382 deletions(-) delete mode 100644 apps/backend/src/audit/audit-log.entity.ts delete mode 100644 apps/backend/src/imports/import.entity.ts delete mode 100644 apps/backend/src/imports/types.ts delete mode 100644 apps/backend/src/migrations/1790372888092-init_schema.ts delete mode 100644 apps/backend/src/observations/daily-observation.entity.ts delete mode 100644 apps/backend/src/observations/hourly-observation.entity.ts delete mode 100644 apps/backend/src/observations/scheduled-observation.entity.ts delete mode 100644 apps/backend/src/records/daily-record.entity.ts delete mode 100644 apps/backend/src/records/types.ts diff --git a/apps/backend/src/audit/audit-log.entity.ts b/apps/backend/src/audit/audit-log.entity.ts deleted file mode 100644 index 689a58af3..000000000 --- a/apps/backend/src/audit/audit-log.entity.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { - Column, - CreateDateColumn, - Entity, - ManyToOne, - PrimaryGeneratedColumn, -} from 'typeorm'; - -import { User } from '../users/user.entity'; - -@Entity('audit_log') -export class AuditLog { - @PrimaryGeneratedColumn() - id: number; - - @Column({ type: 'text' }) - tableName: string; - - // Primary key of the edited row, e.g. { "date": "2026-07-01", "hour": 0 }. - @Column({ type: 'jsonb' }) - rowKey: Record; - - @Column({ type: 'text' }) - columnName: string; - - @Column({ type: 'text', nullable: true }) - oldValue: string; - - @Column({ type: 'text', nullable: true }) - newValue: string; - - @ManyToOne(() => User, { nullable: false }) - changedBy: User; - - @CreateDateColumn({ type: 'timestamptz' }) - changedAt: Date; -} diff --git a/apps/backend/src/imports/import.entity.ts b/apps/backend/src/imports/import.entity.ts deleted file mode 100644 index 0c5663eac..000000000 --- a/apps/backend/src/imports/import.entity.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { - Column, - CreateDateColumn, - Entity, - ManyToOne, - PrimaryGeneratedColumn, -} from 'typeorm'; - -import { User } from '../users/user.entity'; -import { ImportStatus } from './types'; - -@Entity() -export class Import { - @PrimaryGeneratedColumn() - id: number; - - @Column({ type: 'text' }) - fileName: string; - - @ManyToOne(() => User, { nullable: false }) - uploadedBy: User; - - @CreateDateColumn({ type: 'timestamptz' }) - uploadedAt: Date; - - @Column({ type: 'enum', enum: ImportStatus }) - status: ImportStatus; - - // Rows that failed validation, with the reason for each. - @Column({ type: 'jsonb', nullable: true }) - errors: unknown; -} diff --git a/apps/backend/src/imports/types.ts b/apps/backend/src/imports/types.ts deleted file mode 100644 index 16f73880b..000000000 --- a/apps/backend/src/imports/types.ts +++ /dev/null @@ -1,4 +0,0 @@ -export enum ImportStatus { - SUCCEEDED = 'succeeded', - FAILED = 'failed', -} diff --git a/apps/backend/src/migrations/1790372888092-init_schema.ts b/apps/backend/src/migrations/1790372888092-init_schema.ts deleted file mode 100644 index de5dd8905..000000000 --- a/apps/backend/src/migrations/1790372888092-init_schema.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { MigrationInterface, QueryRunner } from 'typeorm'; - -export class InitSchema1790372888092 implements MigrationInterface { - name = 'InitSchema1790372888092'; - - public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query( - `CREATE TABLE "users" ("id" integer NOT NULL, "status" character varying NOT NULL, "firstName" character varying NOT NULL, "lastName" character varying NOT NULL, "email" character varying NOT NULL, CONSTRAINT "PK_a3ffb1c0c8416b9fc6f907b7433" PRIMARY KEY ("id"))`, - ); - await queryRunner.query( - `CREATE TYPE "public"."imports_status_enum" AS ENUM('succeeded', 'failed')`, - ); - await queryRunner.query( - `CREATE TABLE "imports" ("id" SERIAL NOT NULL, "fileName" text NOT NULL, "uploadedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "status" "public"."imports_status_enum" NOT NULL, "errors" jsonb, "uploadedBy" integer NOT NULL, CONSTRAINT "PK_ea10c62f5eb1d75e83d8b5225db" PRIMARY KEY ("id"))`, - ); - await queryRunner.query( - `CREATE TABLE "daily_observations" ("date" date NOT NULL, "maxTempF" smallint, "minTempF" smallint, "precipIn" numeric(5,2), "precipTrace" boolean NOT NULL DEFAULT false, "snowfallIn" numeric(4,1), "snowDepthIn" numeric(4,1), "sunshineMin" smallint, "sunshinePct" smallint, "fastestMileMph" smallint, "fastestMileDir" character varying(3), "fastestMileTime" TIME, "peakGustKts" smallint, "peakGustDir" character varying(3), "peakGustTime" TIME, "avgStationPressureMb" numeric(5,1), "sunrise" TIME, "sunset" TIME, "observerInitials" text, "remarks" text, "import" integer, CONSTRAINT "PK_a7cec69d330c77094266383c7f7" PRIMARY KEY ("date"))`, - ); - await queryRunner.query( - `CREATE TABLE "hourly_observations" ("date" date NOT NULL, "hour" smallint NOT NULL, "tempF" smallint, "precipIn" numeric(4,2), "precipTrace" boolean NOT NULL DEFAULT false, "windDir" character varying(3), "windSpeedMph" smallint, "sunshineMin" smallint, "skyCover" smallint, "visibilityMi" numeric(5,2), "presentWeather" text, "humidityPct" smallint, "mountainVis" text, "remarks" text, CONSTRAINT "PK_91c9c1470c1f91dd1e3dc45de22" PRIMARY KEY ("date", "hour"))`, - ); - await queryRunner.query( - `CREATE TABLE "scheduled_observations" ("date" date NOT NULL, "obsTime" TIME NOT NULL, "stationPressureIn" numeric(5,3), "dryBulbF" numeric(4,1), "wetBulbF" numeric(4,1), "dewpointF" smallint, "humidityPct" smallint, "maxTempF" smallint, "minTempF" smallint, "precipIn" numeric(4,2), "precipTrace" boolean NOT NULL DEFAULT false, "snowfallIn" numeric(4,1), "snowDepthIn" numeric(4,1), "vaporPressureMb" numeric(4,1), CONSTRAINT "PK_ab5b2148436c377435699de1c4c" PRIMARY KEY ("date", "obsTime"))`, - ); - await queryRunner.query( - `CREATE TYPE "public"."daily_records_element_enum" AS ENUM('high', 'low', 'precip', 'snow', 'peak_gust')`, - ); - await queryRunner.query( - `CREATE TABLE "daily_records" ("month" smallint NOT NULL, "day" smallint NOT NULL, "element" "public"."daily_records_element_enum" NOT NULL, "value" numeric(5,2), "years" smallint array NOT NULL, "windDir" character varying(3), "isEstimated" boolean NOT NULL DEFAULT false, "import" integer, CONSTRAINT "PK_54cdb09a0715ffeddbb2193b0ac" PRIMARY KEY ("month", "day", "element"))`, - ); - await queryRunner.query( - `CREATE TABLE "audit_log" ("id" SERIAL NOT NULL, "tableName" text NOT NULL, "rowKey" jsonb NOT NULL, "columnName" text NOT NULL, "oldValue" text, "newValue" text, "changedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "changedBy" integer NOT NULL, CONSTRAINT "PK_07fefa57f7f5ab8fc3f52b3ed0b" PRIMARY KEY ("id"))`, - ); - await queryRunner.query( - `ALTER TABLE "imports" ADD CONSTRAINT "FK_c80e47438b6c9719cab60d65860" FOREIGN KEY ("uploadedBy") REFERENCES "users"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`, - ); - await queryRunner.query( - `ALTER TABLE "daily_observations" ADD CONSTRAINT "FK_dcd7b8d6de045fa97944a212f0c" FOREIGN KEY ("import") REFERENCES "imports"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`, - ); - await queryRunner.query( - `ALTER TABLE "hourly_observations" ADD CONSTRAINT "FK_04b56339d6c0b00fdf7f1c561c3" FOREIGN KEY ("date") REFERENCES "daily_observations"("date") ON DELETE CASCADE ON UPDATE NO ACTION`, - ); - await queryRunner.query( - `ALTER TABLE "scheduled_observations" ADD CONSTRAINT "FK_7576980c1c64dbea8fff5ad53e0" FOREIGN KEY ("date") REFERENCES "daily_observations"("date") ON DELETE CASCADE ON UPDATE NO ACTION`, - ); - await queryRunner.query( - `ALTER TABLE "daily_records" ADD CONSTRAINT "FK_63812be4d466feba48443349a18" FOREIGN KEY ("import") REFERENCES "imports"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`, - ); - await queryRunner.query( - `ALTER TABLE "audit_log" ADD CONSTRAINT "FK_3c20bc116a9a51bab8206ddca1e" FOREIGN KEY ("changedBy") REFERENCES "users"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`, - ); - } - - public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query( - `ALTER TABLE "audit_log" DROP CONSTRAINT "FK_3c20bc116a9a51bab8206ddca1e"`, - ); - await queryRunner.query( - `ALTER TABLE "daily_records" DROP CONSTRAINT "FK_63812be4d466feba48443349a18"`, - ); - await queryRunner.query( - `ALTER TABLE "scheduled_observations" DROP CONSTRAINT "FK_7576980c1c64dbea8fff5ad53e0"`, - ); - await queryRunner.query( - `ALTER TABLE "hourly_observations" DROP CONSTRAINT "FK_04b56339d6c0b00fdf7f1c561c3"`, - ); - await queryRunner.query( - `ALTER TABLE "daily_observations" DROP CONSTRAINT "FK_dcd7b8d6de045fa97944a212f0c"`, - ); - await queryRunner.query( - `ALTER TABLE "imports" DROP CONSTRAINT "FK_c80e47438b6c9719cab60d65860"`, - ); - await queryRunner.query(`DROP TABLE "audit_log"`); - await queryRunner.query(`DROP TABLE "daily_records"`); - await queryRunner.query(`DROP TYPE "public"."daily_records_element_enum"`); - await queryRunner.query(`DROP TABLE "scheduled_observations"`); - await queryRunner.query(`DROP TABLE "hourly_observations"`); - await queryRunner.query(`DROP TABLE "daily_observations"`); - await queryRunner.query(`DROP TABLE "imports"`); - await queryRunner.query(`DROP TYPE "public"."imports_status_enum"`); - await queryRunner.query(`DROP TABLE "users"`); - } -} diff --git a/apps/backend/src/observations/daily-observation.entity.ts b/apps/backend/src/observations/daily-observation.entity.ts deleted file mode 100644 index 8bd338d61..000000000 --- a/apps/backend/src/observations/daily-observation.entity.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { Column, Entity, ManyToOne, PrimaryColumn } from 'typeorm'; - -import { Import } from '../imports/import.entity'; - -// Numeric columns come back from pg as strings to avoid float rounding. -@Entity('daily_observations') -export class DailyObservation { - @PrimaryColumn({ type: 'date' }) - date: string; - - @Column({ type: 'smallint', nullable: true }) - maxTempF: number; - - @Column({ type: 'smallint', nullable: true }) - minTempF: number; - - @Column({ type: 'numeric', precision: 5, scale: 2, nullable: true }) - precipIn: string; - - // The sheet writes "T" for a trace; precipIn is 0 on those days. - @Column({ default: false }) - precipTrace: boolean; - - @Column({ type: 'numeric', precision: 4, scale: 1, nullable: true }) - snowfallIn: string; - - @Column({ type: 'numeric', precision: 4, scale: 1, nullable: true }) - snowDepthIn: string; - - @Column({ type: 'smallint', nullable: true }) - sunshineMin: number; - - @Column({ type: 'smallint', nullable: true }) - sunshinePct: number; - - @Column({ type: 'smallint', nullable: true }) - fastestMileMph: number; - - @Column({ type: 'varchar', length: 3, nullable: true }) - fastestMileDir: string; - - // All times are Eastern Standard Time year-round, as in the sheet. - @Column({ type: 'time', nullable: true }) - fastestMileTime: string; - - // Knots, as recorded in the sheet. - @Column({ type: 'smallint', nullable: true }) - peakGustKts: number; - - @Column({ type: 'varchar', length: 3, nullable: true }) - peakGustDir: string; - - @Column({ type: 'time', nullable: true }) - peakGustTime: string; - - @Column({ type: 'numeric', precision: 5, scale: 1, nullable: true }) - avgStationPressureMb: string; - - @Column({ type: 'time', nullable: true }) - sunrise: string; - - @Column({ type: 'time', nullable: true }) - sunset: string; - - @Column({ type: 'text', nullable: true }) - observerInitials: string; - - @Column({ type: 'text', nullable: true }) - remarks: string; - - @ManyToOne(() => Import, { nullable: true }) - import: Import; -} diff --git a/apps/backend/src/observations/hourly-observation.entity.ts b/apps/backend/src/observations/hourly-observation.entity.ts deleted file mode 100644 index ba00bb9b2..000000000 --- a/apps/backend/src/observations/hourly-observation.entity.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { Column, Entity, JoinColumn, ManyToOne, PrimaryColumn } from 'typeorm'; - -import { DailyObservation } from './daily-observation.entity'; - -@Entity('hourly_observations') -export class HourlyObservation { - @PrimaryColumn({ type: 'date' }) - date: string; - - // Start of the hour: the sheet's "00-01" row is hour 0. - @PrimaryColumn({ type: 'smallint' }) - hour: number; - - @ManyToOne(() => DailyObservation, { onDelete: 'CASCADE' }) - @JoinColumn({ name: 'date' }) - day: DailyObservation; - - @Column({ type: 'smallint', nullable: true }) - tempF: number; - - @Column({ type: 'numeric', precision: 4, scale: 2, nullable: true }) - precipIn: string; - - @Column({ default: false }) - precipTrace: boolean; - - @Column({ type: 'varchar', length: 3, nullable: true }) - windDir: string; - - @Column({ type: 'smallint', nullable: true }) - windSpeedMph: number; - - @Column({ type: 'smallint', nullable: true }) - sunshineMin: number; - - // Eighths of sky covered; only filled at observation hours. - @Column({ type: 'smallint', nullable: true }) - skyCover: number; - - @Column({ type: 'numeric', precision: 5, scale: 2, nullable: true }) - visibilityMi: string; - - @Column({ type: 'text', nullable: true }) - presentWeather: string; - - @Column({ type: 'smallint', nullable: true }) - humidityPct: number; - - @Column({ type: 'text', nullable: true }) - mountainVis: string; - - @Column({ type: 'text', nullable: true }) - remarks: string; -} diff --git a/apps/backend/src/observations/scheduled-observation.entity.ts b/apps/backend/src/observations/scheduled-observation.entity.ts deleted file mode 100644 index 3cb9cd570..000000000 --- a/apps/backend/src/observations/scheduled-observation.entity.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { Column, Entity, JoinColumn, ManyToOne, PrimaryColumn } from 'typeorm'; - -import { DailyObservation } from './daily-observation.entity'; - -// Readings taken by hand at 0700, 0800, 1000, 1300 and 1900. -@Entity('scheduled_observations') -export class ScheduledObservation { - @PrimaryColumn({ type: 'date' }) - date: string; - - @PrimaryColumn({ type: 'time' }) - obsTime: string; - - @ManyToOne(() => DailyObservation, { onDelete: 'CASCADE' }) - @JoinColumn({ name: 'date' }) - day: DailyObservation; - - @Column({ type: 'numeric', precision: 5, scale: 3, nullable: true }) - stationPressureIn: string; - - @Column({ type: 'numeric', precision: 4, scale: 1, nullable: true }) - dryBulbF: string; - - @Column({ type: 'numeric', precision: 4, scale: 1, nullable: true }) - wetBulbF: string; - - @Column({ type: 'smallint', nullable: true }) - dewpointF: number; - - @Column({ type: 'smallint', nullable: true }) - humidityPct: number; - - // The max/min/precip/snow columns are only filled on the 0700 row and - // cover the 24 hours ending at 0700. - @Column({ type: 'smallint', nullable: true }) - maxTempF: number; - - @Column({ type: 'smallint', nullable: true }) - minTempF: number; - - @Column({ type: 'numeric', precision: 4, scale: 2, nullable: true }) - precipIn: string; - - @Column({ default: false }) - precipTrace: boolean; - - @Column({ type: 'numeric', precision: 4, scale: 1, nullable: true }) - snowfallIn: string; - - @Column({ type: 'numeric', precision: 4, scale: 1, nullable: true }) - snowDepthIn: string; - - @Column({ type: 'numeric', precision: 4, scale: 1, nullable: true }) - vaporPressureMb: string; -} diff --git a/apps/backend/src/records/daily-record.entity.ts b/apps/backend/src/records/daily-record.entity.ts deleted file mode 100644 index 398d30ba3..000000000 --- a/apps/backend/src/records/daily-record.entity.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { Column, Entity, ManyToOne, PrimaryColumn } from 'typeorm'; - -import { Import } from '../imports/import.entity'; -import { RecordElement } from './types'; - -// All-time record per calendar day, one row per element so a "record broken" -// check is a single lookup. Joins to observations on month and day, not a FK. -@Entity('daily_records') -export class DailyRecord { - @PrimaryColumn({ type: 'smallint' }) - month: number; - - @PrimaryColumn({ type: 'smallint' }) - day: number; - - @PrimaryColumn({ type: 'enum', enum: RecordElement }) - element: RecordElement; - - // Null when the sheet says "None". - @Column({ type: 'numeric', precision: 5, scale: 2, nullable: true }) - value: string; - - // Ties list every year the record was set. - @Column({ type: 'smallint', array: true }) - years: number[]; - - @Column({ type: 'varchar', length: 3, nullable: true }) - windDir: string; - - @Column({ default: false }) - isEstimated: boolean; - - @ManyToOne(() => Import, { nullable: true }) - import: Import; -} diff --git a/apps/backend/src/records/types.ts b/apps/backend/src/records/types.ts deleted file mode 100644 index ee6d65047..000000000 --- a/apps/backend/src/records/types.ts +++ /dev/null @@ -1,7 +0,0 @@ -export enum RecordElement { - HIGH = 'high', - LOW = 'low', - PRECIP = 'precip', - SNOW = 'snow', - PEAK_GUST = 'peak_gust', -} diff --git a/docs/database-schema.md b/docs/database-schema.md index 613aae358..da32cf75e 100644 --- a/docs/database-schema.md +++ b/docs/database-schema.md @@ -1,13 +1,13 @@ # Database schema -Draft for [#3](https://github.com/Code-4-Community/bho/issues/3). PostgreSQL on AWS RDS. Created by `apps/backend/src/migrations/1790372888092-init_schema.ts`. +Draft for [#3](https://github.com/Code-4-Community/bho/issues/3). PostgreSQL on AWS RDS. Sources: - **Daily sheet**: [DAILY - 07. July2026.xlsx](https://docs.google.com/spreadsheets/d/1cWloMpYHZDGkUFulmeW0g1bY3J97Ts4b/edit), tab `Sheet1`. One 60-row block per day. Cell references are for the July 1 block; add 60 rows for each later day. - **Historical sheet**: [BHO_WeatherRecords_Master_TestData.xlsx](https://docs.google.com/spreadsheets/d/1WI8CxTJ7edRajo3JHZX0WmTklvCqRy5x/edit), tab `All Daily Records`, rows 4-369. -Column names are camelCase because the repo's naming strategy uses property names as column names. +Column names are camelCase to match the repo's TypeORM naming strategy, which uses property names as column names. ## Relations @@ -35,6 +35,7 @@ Source: the app, one row per uploaded spreadsheet. | ---------- | ----------- | ---- | ------------------------------------ | | id | serial | N | PK | | fileName | text | N | | +| s3Key | text | N | Key of the uploaded file in S3 | | uploadedBy | int | N | FK users | | uploadedAt | timestamptz | N | Defaults to now | | status | enum | N | `succeeded` or `failed` | From 6300fdf78450cf18fee80a47d79ab298c14eb82e Mon Sep 17 00:00:00 2001 From: ryaken-nakamoto Date: Mon, 28 Sep 2026 19:50:38 -0400 Subject: [PATCH 05/16] simplify types --- docs/database-schema.md | 161 +++++++++++++++++++++------------------- 1 file changed, 84 insertions(+), 77 deletions(-) diff --git a/docs/database-schema.md b/docs/database-schema.md index da32cf75e..690a2a3c3 100644 --- a/docs/database-schema.md +++ b/docs/database-schema.md @@ -45,29 +45,29 @@ Source: the app, one row per uploaded spreadsheet. Source: daily sheet, "Summary of Day" and nearby rows. PK `date`. -| Column | Type | Null | Source | -| -------------------- | ------------ | ---- | ------------------------------------------- | -| date | date | N | J1 "Wednesday, July 1, 2026" | -| maxTempF | smallint | Y | A44 "24HR Max" | -| minTempF | smallint | Y | B44 "24HR Min" | -| precipIn | numeric(5,2) | Y | C44 "24HR Precip" | -| precipTrace | boolean | N | Our flag for when C44 is `T` (July 5: C284) | -| snowfallIn | numeric(4,1) | Y | E44 "Snowfall" | -| snowDepthIn | numeric(4,1) | Y | F44 "Dpth 0700" | -| sunshineMin | smallint | Y | G44 "Total Min." | -| sunshinePct | smallint | Y | H44 "% poss." (0.77 becomes 77) | -| fastestMileMph | smallint | Y | I44 | -| fastestMileDir | varchar(3) | Y | J44 | -| fastestMileTime | time | Y | K44 "0245E" (24-hour, EST) | -| peakGustKts | smallint | Y | L44 "20 WSW @ 0854E" (knots) | -| peakGustDir | varchar(3) | Y | L44 | -| peakGustTime | time | Y | L44 | -| avgStationPressureMb | numeric(5,1) | Y | E56 "Station Pressure" AVG | -| sunrise | time | Y | K29 | -| sunset | time | Y | K30 | -| observerInitials | text | Y | H59 "MFD/AKJ" | -| remarks | text | Y | G47-G49 "Additional Notes" | -| import | int | Y | FK imports | +| Column | Type | Null | Source | +| -------------------- | ---------- | ---- | ------------------------------------------- | +| date | date | N | J1 "Wednesday, July 1, 2026" | +| maxTempF | smallint | Y | A44 "24HR Max" | +| minTempF | smallint | Y | B44 "24HR Min" | +| precipIn | numeric | Y | C44 "24HR Precip" | +| precipTrace | boolean | N | Our flag for when C44 is `T` (July 5: C284) | +| snowfallIn | numeric | Y | E44 "Snowfall" | +| snowDepthIn | numeric | Y | F44 "Dpth 0700" | +| sunshineMin | smallint | Y | G44 "Total Min." | +| sunshinePct | smallint | Y | H44 "% poss." (0.77 becomes 77) | +| fastestMileMph | smallint | Y | I44 | +| fastestMileDir | varchar(3) | Y | J44 | +| fastestMileTime | time | Y | K44 "0245E" (24-hour, EST) | +| peakGustKts | smallint | Y | L44 "20 WSW @ 0854E" (knots) | +| peakGustDir | varchar(3) | Y | L44 | +| peakGustTime | time | Y | L44 | +| avgStationPressureMb | numeric | Y | E56 "Station Pressure" AVG | +| sunrise | time | Y | K29 | +| sunset | time | Y | K30 | +| observerInitials | text | Y | H59 "MFD/AKJ" | +| remarks | text | Y | G47-G49 "Additional Notes" | +| import | int | Y | FK imports | Not stored: average temperature, normal, departure, and degree days (rows 46-50). They can be computed. @@ -75,73 +75,80 @@ Not stored: average temperature, normal, departure, and degree days (rows 46-50) Source: daily sheet, rows 4-27 ("00-01" to "23-24"). PK `(date, hour)`. Deleting a day deletes its hours. -| Column | Type | Null | Source | -| -------------- | ------------ | ---- | -------------------------------------- | -| date | date | N | FK daily_observations | -| hour | smallint | N | A, start hour ("00-01" is 0) | -| tempF | smallint | Y | B "Temp (°F)" | -| precipIn | numeric(4,2) | Y | C "Precip" | -| precipTrace | boolean | N | Our flag for when C is `T` | -| windDir | varchar(3) | Y | D "Dir." | -| windSpeedMph | smallint | Y | E "Spd (mph)" | -| sunshineMin | smallint | Y | F "Sunshine (min)" | -| skyCover | smallint | Y | G "Sky Cover (0-8)" | -| visibilityMi | numeric(5,2) | Y | H "Lwst VSBL" ("1/8" becomes 0.125) | -| presentWeather | text | Y | I "Prsnt WX" ("R-F") | -| humidityPct | smallint | Y | J "Rel Hum." | -| mountainVis | text | Y | K "Mnts VSBL" ("1@1.5") | -| remarks | text | Y | L "Notes" | +| Column | Type | Null | Source | +| -------------- | ---------- | ---- | ----------------------------------- | +| date | date | N | FK daily_observations | +| hour | smallint | N | A, start hour ("00-01" is 0) | +| tempF | smallint | Y | B "Temp (°F)" | +| precipIn | numeric | Y | C "Precip" | +| precipTrace | boolean | N | Our flag for when C is `T` | +| windDir | varchar(3) | Y | D "Dir." | +| windSpeedMph | smallint | Y | E "Spd (mph)" | +| sunshineMin | smallint | Y | F "Sunshine (min)" | +| skyCover | smallint | Y | G "Sky Cover (0-8)" | +| visibilityMi | numeric | Y | H "Lwst VSBL" ("1/8" becomes 0.125) | +| presentWeather | text | Y | I "Prsnt WX" ("R-F") | +| humidityPct | smallint | Y | J "Rel Hum." | +| mountainVis | text | Y | K "Mnts VSBL" ("1@1.5") | +| remarks | text | Y | L "Notes" | ## scheduled_observations Source: daily sheet, "Scheduled Observations", rows 34-38. PK `(date, obsTime)`. Deleting a day deletes its readings. -| Column | Type | Null | Source | -| ----------------- | ------------ | ---- | ---------------------------------------- | -| date | date | N | FK daily_observations | -| obsTime | time | N | A (700, 800, 1000, 1300, 1900) | -| stationPressureIn | numeric(5,3) | Y | B, 0700 and 1900 only | -| dryBulbF | numeric(4,1) | Y | C | -| wetBulbF | numeric(4,1) | Y | D | -| dewpointF | smallint | Y | E | -| humidityPct | smallint | Y | F (0.8 becomes 80) | -| maxTempF | smallint | Y | G, 0700 only, 24 hours ending 0700 | -| minTempF | smallint | Y | H, 0700 only | -| precipIn | numeric(4,2) | Y | I, 0700 only | -| precipTrace | boolean | N | Our flag for when I is `T` | -| snowfallIn | numeric(4,1) | Y | J, 0700 only | -| snowDepthIn | numeric(4,1) | Y | K, 0700 only | -| vaporPressureMb | numeric(4,1) | Y | L | +| Column | Type | Null | Source | +| ----------------- | -------- | ---- | ---------------------------------- | +| date | date | N | FK daily_observations | +| obsTime | time | N | A (700, 800, 1000, 1300, 1900) | +| stationPressureIn | numeric | Y | B, 0700 and 1900 only | +| dryBulbF | numeric | Y | C | +| wetBulbF | numeric | Y | D | +| dewpointF | smallint | Y | E | +| humidityPct | smallint | Y | F (0.8 becomes 80) | +| maxTempF | smallint | Y | G, 0700 only, 24 hours ending 0700 | +| minTempF | smallint | Y | H, 0700 only | +| precipIn | numeric | Y | I, 0700 only | +| precipTrace | boolean | N | Our flag for when I is `T` | +| snowfallIn | numeric | Y | J, 0700 only | +| snowDepthIn | numeric | Y | K, 0700 only | +| vaporPressureMb | numeric | Y | L | ## daily_records -Source: historical sheet. Each sheet row becomes five rows, one per element. PK `(month, day, element)`. - -| Column | Type | Null | Source | -| ----------- | ------------ | ---- | --------------------------------------------------------- | -| month | smallint | N | "Month #" | -| day | smallint | N | "Day" | -| element | enum | N | `high`, `low`, `precip`, `snow`, `peak_gust` | -| value | numeric(5,2) | Y | "High °F", "Low °F", "Precip in", "Snow in", "Peak Gust mph"; null for "None" | -| years | smallint[] | N | "... Year(s)"; "2014, 1977" becomes `{2014,1977}` | -| windDir | varchar(3) | Y | "Wind Dir", peak_gust only | -| isEstimated | boolean | N | "Estimated?", peak_gust only | -| import | int | Y | FK imports | +Source: historical sheet. One sheet row becomes one row. PK `(month, day)`. Each element's value is null when the sheet says "None", and its years are null too. + +| Column | Type | Null | Source | +| ------------------- | ---------- | ---- | -------------------------------------------------- | +| month | smallint | N | "Month #" | +| day | smallint | N | "Day" | +| highF | smallint | Y | "High °F" | +| highYears | smallint[] | Y | "High Year(s)"; "2014, 1977" becomes `{2014,1977}` | +| lowF | smallint | Y | "Low °F" | +| lowYears | smallint[] | Y | "Low Year(s)" | +| precipIn | numeric | Y | "Precip in" | +| precipYears | smallint[] | Y | "Precip Year(s)" | +| snowIn | numeric | Y | "Snow in"; null for "None" | +| snowYears | smallint[] | Y | "Snow Year(s)"; null when snow is "None" | +| peakGustMph | smallint | Y | "Peak Gust mph" | +| peakGustDir | varchar(3) | Y | "Wind Dir" | +| peakGustIsEstimated | boolean | N | "Estimated?" | +| peakGustYears | smallint[] | Y | "Gust Year(s)" | +| import | int | Y | FK imports | ## audit_log Source: the app, one row per edited value. -| Column | Type | Null | Note | -| ---------- | ----------- | ---- | ----------------------------- | -| id | serial | N | PK | -| tableName | text | N | | -| rowKey | jsonb | N | PK of the edited row | -| columnName | text | N | | -| oldValue | text | Y | | -| newValue | text | Y | | -| changedBy | int | N | FK users | -| changedAt | timestamptz | N | Defaults to now | +| Column | Type | Null | Note | +| ---------- | ----------- | ---- | -------------------- | +| id | serial | N | PK | +| tableName | text | N | | +| rowKey | jsonb | N | PK of the edited row | +| columnName | text | N | | +| oldValue | text | Y | | +| newValue | text | Y | | +| changedBy | int | N | FK users | +| changedAt | timestamptz | N | Defaults to now | ## Not in this draft From 4271e9225e9c50b0a68112448e22d136000bb933 Mon Sep 17 00:00:00 2001 From: ryaken-nakamoto Date: Mon, 28 Sep 2026 20:10:32 -0400 Subject: [PATCH 06/16] update observibility --- docs/database-schema.md | 65 +++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/docs/database-schema.md b/docs/database-schema.md index 690a2a3c3..3e29e0e4e 100644 --- a/docs/database-schema.md +++ b/docs/database-schema.md @@ -14,7 +14,8 @@ Column names are camelCase to match the repo's TypeORM naming strategy, which us ```mermaid erDiagram users ||--o{ imports : uploads - users ||--o{ audit_log : edits + users ||--o{ audit_log : changes + imports ||--o{ audit_log : produces imports ||--o{ daily_observations : creates imports ||--o{ daily_records : creates daily_observations ||--|{ hourly_observations : has @@ -29,18 +30,26 @@ The scaffold's existing `User` entity, unchanged: `id`, `status`, `firstName`, ` ## imports -Source: the app, one row per uploaded spreadsheet. +Source: the app, one row per uploaded spreadsheet. The uploaded file stays in S3 as the raw source of truth. -| Column | Type | Null | Note | -| ---------- | ----------- | ---- | ------------------------------------ | -| id | serial | N | PK | -| fileName | text | N | | -| s3Key | text | N | Key of the uploaded file in S3 | -| uploadedBy | int | N | FK users | -| uploadedAt | timestamptz | N | Defaults to now | -| status | enum | N | `succeeded` or `failed` | -| errors | jsonb | Y | Flagged rows and the reason for each | +| Column | Type | Null | Note | +| ---------- | ----------- | ---- | ----------------------------------------------- | +| id | serial | N | PK | +| kind | text | N | `daily` or `historical`; text, | +| fileName | text | N | | +| s3Key | text | N | Key of the uploaded file in S3 | +| fileHash | text | N | SHA-256 of the file, to catch duplicate uploads | +| uploadedBy | int | N | FK users | +| uploadedAt | timestamptz | N | Defaults to now | +| status | enum | N | `succeeded`, or `failed` | +| errors | jsonb | Y | Flagged rows and the reason for each | + + +Ideas for later: + +- `rowCount` (rows imported): not sure what the existing parser looks like +- `partial`? Do we want to allow for this, feels simpler to force a re-import and log that import failed ## daily_observations Source: daily sheet, "Summary of Day" and nearby rows. PK `date`. @@ -67,7 +76,9 @@ Source: daily sheet, "Summary of Day" and nearby rows. PK `date`. | sunset | time | Y | K30 | | observerInitials | text | Y | H59 "MFD/AKJ" | | remarks | text | Y | G47-G49 "Additional Notes" | -| import | int | Y | FK imports | +| importId | int | N | FK imports; null for hand-entered rows | + +Questions: how often are rows really "added"? Seems like they would only be edited, and what can we depend on being available and not? What should we require? (same applies to daily_records) Not stored: average temperature, normal, departure, and degree days (rows 46-50). They can be computed. @@ -133,25 +144,23 @@ Source: historical sheet. One sheet row becomes one row. PK `(month, day)`. Each | peakGustDir | varchar(3) | Y | "Wind Dir" | | peakGustIsEstimated | boolean | N | "Estimated?" | | peakGustYears | smallint[] | Y | "Gust Year(s)" | -| import | int | Y | FK imports | +| importId | int | Y | FK imports; null for hand-entered rows | ## audit_log -Source: the app, one row per edited value. +Source: a Postgres trigger, one row per inserted, updated, or deleted row in `daily_observations`, `hourly_observations`, `scheduled_observations`, and `daily_records`. -| Column | Type | Null | Note | -| ---------- | ----------- | ---- | -------------------- | -| id | serial | N | PK | -| tableName | text | N | | -| rowKey | jsonb | N | PK of the edited row | -| columnName | text | N | | -| oldValue | text | Y | | -| newValue | text | Y | | -| changedBy | int | N | FK users | -| changedAt | timestamptz | N | Defaults to now | +| Column | Type | Null | Note | +| --------- | ----------- | ---- | ---------------------------------------------- | +| id | serial | N | PK | +| tableName | text | N | | +| rowKey | jsonb | N | PK of the affected row | +| action | enum | N | `insert`, `update`, or `delete` | +| oldData | jsonb | Y | Whole row before; null on insert | +| newData | jsonb | Y | Whole row after; null on delete | +| changedBy | int | Y | FK users; null for system writes | +| importId | int | Y | FK imports; set when an import made the change | +| changedAt | timestamptz | N | Defaults to now | -## Not in this draft +One SQL function is attached to the four data tables, so no code path can skip the log. -- `normals`: no normals data yet. -- `narratives`: no current source. -- `live_readings`, `ingestion_runs`: waiting on the WeatherLink API format. From 49f3faf9a691b515c0a9694095fa0491753159f3 Mon Sep 17 00:00:00 2001 From: Naman Rusia Date: Fri, 2 Oct 2026 20:39:29 -0400 Subject: [PATCH 07/16] Add stations and possible sunshine to schema doc; require importId Client confirmed more stations are coming, so stationId joins the keys of the three daily tables now rather than in a later migration. Every row comes from an upload, so importId is not null on observations and records. --- docs/database-schema.md | 104 ++++++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 46 deletions(-) diff --git a/docs/database-schema.md b/docs/database-schema.md index 3e29e0e4e..c6955b84a 100644 --- a/docs/database-schema.md +++ b/docs/database-schema.md @@ -18,6 +18,7 @@ erDiagram imports ||--o{ audit_log : produces imports ||--o{ daily_observations : creates imports ||--o{ daily_records : creates + stations ||--o{ daily_observations : records daily_observations ||--|{ hourly_observations : has daily_observations ||--|{ scheduled_observations : has ``` @@ -28,6 +29,15 @@ erDiagram The scaffold's existing `User` entity, unchanged: `id`, `status`, `firstName`, `lastName`, `email`. +## stations + +Source: the app. Starts with one row, `19737-02`; the client plans to add more. A table rather than an enum, so a new station is a new row, not a migration. + +| Column | Type | Null | Note | +| ------ | ------ | ---- | ------------------ | +| id | serial | N | PK | +| code | text | N | Unique, `19737-02` | + ## imports Source: the app, one row per uploaded spreadsheet. The uploaded file stays in S3 as the raw source of truth. @@ -35,27 +45,27 @@ Source: the app, one row per uploaded spreadsheet. The uploaded file stays in S3 | Column | Type | Null | Note | | ---------- | ----------- | ---- | ----------------------------------------------- | | id | serial | N | PK | -| kind | text | N | `daily` or `historical`; text, | +| kind | text | N | `daily` or `historical` | | fileName | text | N | | | s3Key | text | N | Key of the uploaded file in S3 | | fileHash | text | N | SHA-256 of the file, to catch duplicate uploads | | uploadedBy | int | N | FK users | | uploadedAt | timestamptz | N | Defaults to now | -| status | enum | N | `succeeded`, or `failed` | +| status | enum | N | `succeeded` or `failed` | | errors | jsonb | Y | Flagged rows and the reason for each | - - Ideas for later: - `rowCount` (rows imported): not sure what the existing parser looks like - `partial`? Do we want to allow for this, feels simpler to force a re-import and log that import failed + ## daily_observations -Source: daily sheet, "Summary of Day" and nearby rows. PK `date`. +Source: daily sheet, "Summary of Day" and nearby rows. PK `(stationId, date)`. | Column | Type | Null | Source | | -------------------- | ---------- | ---- | ------------------------------------------- | +| stationId | int | N | FK stations; the station the upload is for | | date | date | N | J1 "Wednesday, July 1, 2026" | | maxTempF | smallint | Y | A44 "24HR Max" | | minTempF | smallint | Y | B44 "24HR Min" | @@ -64,6 +74,7 @@ Source: daily sheet, "Summary of Day" and nearby rows. PK `date`. | snowfallIn | numeric | Y | E44 "Snowfall" | | snowDepthIn | numeric | Y | F44 "Dpth 0700" | | sunshineMin | smallint | Y | G44 "Total Min." | +| sunshinePossibleMin | smallint | Y | F29 "Poss 914" (possible sunshine minutes) | | sunshinePct | smallint | Y | H44 "% poss." (0.77 becomes 77) | | fastestMileMph | smallint | Y | I44 | | fastestMileDir | varchar(3) | Y | J44 | @@ -76,57 +87,59 @@ Source: daily sheet, "Summary of Day" and nearby rows. PK `date`. | sunset | time | Y | K30 | | observerInitials | text | Y | H59 "MFD/AKJ" | | remarks | text | Y | G47-G49 "Additional Notes" | -| importId | int | N | FK imports; null for hand-entered rows | +| importId | int | N | FK imports; the last upload that wrote it | -Questions: how often are rows really "added"? Seems like they would only be edited, and what can we depend on being available and not? What should we require? (same applies to daily_records) +Every row comes from an upload. Staff re-upload the month's sheet several times a day, and the most recent upload overwrites each day it contains. Weather values are nullable: an empty cell means no data and is stored as null, never 0. The same applies to `daily_records`. Not stored: average temperature, normal, departure, and degree days (rows 46-50). They can be computed. ## hourly_observations -Source: daily sheet, rows 4-27 ("00-01" to "23-24"). PK `(date, hour)`. Deleting a day deletes its hours. - -| Column | Type | Null | Source | -| -------------- | ---------- | ---- | ----------------------------------- | -| date | date | N | FK daily_observations | -| hour | smallint | N | A, start hour ("00-01" is 0) | -| tempF | smallint | Y | B "Temp (°F)" | -| precipIn | numeric | Y | C "Precip" | -| precipTrace | boolean | N | Our flag for when C is `T` | -| windDir | varchar(3) | Y | D "Dir." | -| windSpeedMph | smallint | Y | E "Spd (mph)" | -| sunshineMin | smallint | Y | F "Sunshine (min)" | -| skyCover | smallint | Y | G "Sky Cover (0-8)" | -| visibilityMi | numeric | Y | H "Lwst VSBL" ("1/8" becomes 0.125) | -| presentWeather | text | Y | I "Prsnt WX" ("R-F") | -| humidityPct | smallint | Y | J "Rel Hum." | -| mountainVis | text | Y | K "Mnts VSBL" ("1@1.5") | -| remarks | text | Y | L "Notes" | +Source: daily sheet, rows 4-27 ("00-01" to "23-24"). PK `(stationId, date, hour)`. Deleting a day deletes its hours. + +| Column | Type | Null | Source | +| -------------- | ---------- | ---- | --------------------------------------- | +| stationId | int | N | FK daily_observations, with `date` | +| date | date | N | FK daily_observations, with `stationId` | +| hour | smallint | N | A, start hour ("00-01" is 0) | +| tempF | smallint | Y | B "Temp (°F)" | +| precipIn | numeric | Y | C "Precip" | +| precipTrace | boolean | N | Our flag for when C is `T` | +| windDir | varchar(3) | Y | D "Dir." | +| windSpeedMph | smallint | Y | E "Spd (mph)" | +| sunshineMin | smallint | Y | F "Sunshine (min)" | +| skyCover | smallint | Y | G "Sky Cover (0-8)" | +| visibilityMi | numeric | Y | H "Lwst VSBL" ("1/8" becomes 0.125) | +| presentWeather | text | Y | I "Prsnt WX" ("R-F") | +| humidityPct | smallint | Y | J "Rel Hum." | +| mountainVis | text | Y | K "Mnts VSBL" ("1@1.5") | +| remarks | text | Y | L "Notes" | ## scheduled_observations -Source: daily sheet, "Scheduled Observations", rows 34-38. PK `(date, obsTime)`. Deleting a day deletes its readings. - -| Column | Type | Null | Source | -| ----------------- | -------- | ---- | ---------------------------------- | -| date | date | N | FK daily_observations | -| obsTime | time | N | A (700, 800, 1000, 1300, 1900) | -| stationPressureIn | numeric | Y | B, 0700 and 1900 only | -| dryBulbF | numeric | Y | C | -| wetBulbF | numeric | Y | D | -| dewpointF | smallint | Y | E | -| humidityPct | smallint | Y | F (0.8 becomes 80) | -| maxTempF | smallint | Y | G, 0700 only, 24 hours ending 0700 | -| minTempF | smallint | Y | H, 0700 only | -| precipIn | numeric | Y | I, 0700 only | -| precipTrace | boolean | N | Our flag for when I is `T` | -| snowfallIn | numeric | Y | J, 0700 only | -| snowDepthIn | numeric | Y | K, 0700 only | -| vaporPressureMb | numeric | Y | L | +Source: daily sheet, "Scheduled Observations", rows 34-38. PK `(stationId, date, obsTime)`. Deleting a day deletes its readings. + +| Column | Type | Null | Source | +| ----------------- | -------- | ---- | --------------------------------------- | +| stationId | int | N | FK daily_observations, with `date` | +| date | date | N | FK daily_observations, with `stationId` | +| obsTime | time | N | A (700, 800, 1000, 1300, 1900) | +| stationPressureIn | numeric | Y | B, 0700 and 1900 only | +| dryBulbF | numeric | Y | C | +| wetBulbF | numeric | Y | D | +| dewpointF | smallint | Y | E | +| humidityPct | smallint | Y | F (0.8 becomes 80) | +| maxTempF | smallint | Y | G, 0700 only, 24 hours ending 0700 | +| minTempF | smallint | Y | H, 0700 only | +| precipIn | numeric | Y | I, 0700 only | +| precipTrace | boolean | N | Our flag for when I is `T` | +| snowfallIn | numeric | Y | J, 0700 only | +| snowDepthIn | numeric | Y | K, 0700 only | +| vaporPressureMb | numeric | Y | L | ## daily_records -Source: historical sheet. One sheet row becomes one row. PK `(month, day)`. Each element's value is null when the sheet says "None", and its years are null too. +Source: historical sheet. One sheet row becomes one row. PK `(month, day)`. Each element's value is null when the sheet says "None", and its years are null too. No `stationId`: these are the records for the one official dataset. | Column | Type | Null | Source | | ------------------- | ---------- | ---- | -------------------------------------------------- | @@ -144,7 +157,7 @@ Source: historical sheet. One sheet row becomes one row. PK `(month, day)`. Each | peakGustDir | varchar(3) | Y | "Wind Dir" | | peakGustIsEstimated | boolean | N | "Estimated?" | | peakGustYears | smallint[] | Y | "Gust Year(s)" | -| importId | int | Y | FK imports; null for hand-entered rows | +| importId | int | N | FK imports; the last upload that wrote it | ## audit_log @@ -163,4 +176,3 @@ Source: a Postgres trigger, one row per inserted, updated, or deleted row in `da | changedAt | timestamptz | N | Defaults to now | One SQL function is attached to the four data tables, so no code path can skip the log. - From 85d171e141bb1cf922584e45be50ac9899c9bd41 Mon Sep 17 00:00:00 2001 From: Naman Rusia Date: Fri, 2 Oct 2026 20:57:48 -0400 Subject: [PATCH 08/16] Add stationId to daily_records; explain visibility and present weather Keying records by station now avoids changing their primary key later if records become per-station. The hourly notes now say what column H's fractions and column I's weather codes mean. --- docs/database-schema.md | 42 ++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/docs/database-schema.md b/docs/database-schema.md index c6955b84a..f7c2e7896 100644 --- a/docs/database-schema.md +++ b/docs/database-schema.md @@ -19,11 +19,12 @@ erDiagram imports ||--o{ daily_observations : creates imports ||--o{ daily_records : creates stations ||--o{ daily_observations : records + stations ||--o{ daily_records : holds daily_observations ||--|{ hourly_observations : has daily_observations ||--|{ scheduled_observations : has ``` -`daily_records` joins to observations on month and day, not a foreign key. +`daily_records` joins to observations on station, month, and day, not a foreign key. ## users @@ -97,23 +98,25 @@ Not stored: average temperature, normal, departure, and degree days (rows 46-50) Source: daily sheet, rows 4-27 ("00-01" to "23-24"). PK `(stationId, date, hour)`. Deleting a day deletes its hours. -| Column | Type | Null | Source | -| -------------- | ---------- | ---- | --------------------------------------- | -| stationId | int | N | FK daily_observations, with `date` | -| date | date | N | FK daily_observations, with `stationId` | -| hour | smallint | N | A, start hour ("00-01" is 0) | -| tempF | smallint | Y | B "Temp (°F)" | -| precipIn | numeric | Y | C "Precip" | -| precipTrace | boolean | N | Our flag for when C is `T` | -| windDir | varchar(3) | Y | D "Dir." | -| windSpeedMph | smallint | Y | E "Spd (mph)" | -| sunshineMin | smallint | Y | F "Sunshine (min)" | -| skyCover | smallint | Y | G "Sky Cover (0-8)" | -| visibilityMi | numeric | Y | H "Lwst VSBL" ("1/8" becomes 0.125) | -| presentWeather | text | Y | I "Prsnt WX" ("R-F") | -| humidityPct | smallint | Y | J "Rel Hum." | -| mountainVis | text | Y | K "Mnts VSBL" ("1@1.5") | -| remarks | text | Y | L "Notes" | +| Column | Type | Null | Source | +| -------------- | ---------- | ---- | ------------------------------------------------ | +| stationId | int | N | FK daily_observations, with `date` | +| date | date | N | FK daily_observations, with `stationId` | +| hour | smallint | N | A, start hour ("00-01" is 0) | +| tempF | smallint | Y | B "Temp (°F)" | +| precipIn | numeric | Y | C "Precip" | +| precipTrace | boolean | N | Our flag for when C is `T` | +| windDir | varchar(3) | Y | D "Dir." | +| windSpeedMph | smallint | Y | E "Spd (mph)" | +| sunshineMin | smallint | Y | F "Sunshine (min)" | +| skyCover | smallint | Y | G "Sky Cover (0-8)" | +| visibilityMi | numeric | Y | H "Lwst VSBL (mls)"; "1/8" mile stored as 0.125 | +| presentWeather | text | Y | I "Prsnt WX"; codes, "RW-" is light rain showers | +| humidityPct | smallint | Y | J "Rel Hum." | +| mountainVis | text | Y | K "Mnts VSBL" ("1@1.5") | +| remarks | text | Y | L "Notes" | + +G, H, and I are only recorded at a few hours each day (06-07, 09-10, and 12-13 on July 21). An empty cell there means nobody observed that hour; `0` in I means they observed and there was no weather. Codes in I: `R` rain, `W` showers, `L` drizzle, `F` fog, `-` light, `+` heavy, so `R-F` (I370, July 7) is light rain and fog. These are the standard US codes; BHO has not confirmed them. ## scheduled_observations @@ -139,10 +142,11 @@ Source: daily sheet, "Scheduled Observations", rows 34-38. PK `(stationId, date, ## daily_records -Source: historical sheet. One sheet row becomes one row. PK `(month, day)`. Each element's value is null when the sheet says "None", and its years are null too. No `stationId`: these are the records for the one official dataset. +Source: historical sheet. One sheet row becomes one row. PK `(stationId, month, day)`. Each element's value is null when the sheet says "None", and its years are null too. | Column | Type | Null | Source | | ------------------- | ---------- | ---- | -------------------------------------------------- | +| stationId | int | N | FK stations; the station the upload is for | | month | smallint | N | "Month #" | | day | smallint | N | "Day" | | highF | smallint | Y | "High °F" | From bb21e6dac9d24d6681ba88b556ffe540cb382f48 Mon Sep 17 00:00:00 2001 From: Naman Rusia Date: Mon, 5 Oct 2026 18:13:38 -0400 Subject: [PATCH 09/16] Add average and normal temperature to daily_observations --- docs/database-schema.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/database-schema.md b/docs/database-schema.md index f7c2e7896..3af3079e5 100644 --- a/docs/database-schema.md +++ b/docs/database-schema.md @@ -70,6 +70,8 @@ Source: daily sheet, "Summary of Day" and nearby rows. PK `(stationId, date)`. | date | date | N | J1 "Wednesday, July 1, 2026" | | maxTempF | smallint | Y | A44 "24HR Max" | | minTempF | smallint | Y | B44 "24HR Min" | +| avgTempF | numeric | Y | D46 "Average Temperature"; 81.5 | +| normalTempF | smallint | Y | D47 "Normal Temperature" | | precipIn | numeric | Y | C44 "24HR Precip" | | precipTrace | boolean | N | Our flag for when C44 is `T` (July 5: C284) | | snowfallIn | numeric | Y | E44 "Snowfall" | @@ -92,7 +94,7 @@ Source: daily sheet, "Summary of Day" and nearby rows. PK `(stationId, date)`. Every row comes from an upload. Staff re-upload the month's sheet several times a day, and the most recent upload overwrites each day it contains. Weather values are nullable: an empty cell means no data and is stored as null, never 0. The same applies to `daily_records`. -Not stored: average temperature, normal, departure, and degree days (rows 46-50). They can be computed. +Not stored: departure and degree days (rows 48-50). They can be computed. ## hourly_observations From 4fbb598c4eb3d729bac4da76c36696a650483ae3 Mon Sep 17 00:00:00 2001 From: Naman Rusia Date: Mon, 5 Oct 2026 18:33:42 -0400 Subject: [PATCH 10/16] Add barometer readings to scheduled_observations --- docs/database-schema.md | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/docs/database-schema.md b/docs/database-schema.md index 3af3079e5..7e533253c 100644 --- a/docs/database-schema.md +++ b/docs/database-schema.md @@ -124,23 +124,26 @@ G, H, and I are only recorded at a few hours each day (06-07, 09-10, and 12-13 o Source: daily sheet, "Scheduled Observations", rows 34-38. PK `(stationId, date, obsTime)`. Deleting a day deletes its readings. -| Column | Type | Null | Source | -| ----------------- | -------- | ---- | --------------------------------------- | -| stationId | int | N | FK daily_observations, with `date` | -| date | date | N | FK daily_observations, with `stationId` | -| obsTime | time | N | A (700, 800, 1000, 1300, 1900) | -| stationPressureIn | numeric | Y | B, 0700 and 1900 only | -| dryBulbF | numeric | Y | C | -| wetBulbF | numeric | Y | D | -| dewpointF | smallint | Y | E | -| humidityPct | smallint | Y | F (0.8 becomes 80) | -| maxTempF | smallint | Y | G, 0700 only, 24 hours ending 0700 | -| minTempF | smallint | Y | H, 0700 only | -| precipIn | numeric | Y | I, 0700 only | -| precipTrace | boolean | N | Our flag for when I is `T` | -| snowfallIn | numeric | Y | J, 0700 only | -| snowDepthIn | numeric | Y | K, 0700 only | -| vaporPressureMb | numeric | Y | L | +| Column | Type | Null | Source | +| -------------------- | -------- | ---- | ----------------------------------------------------- | +| stationId | int | N | FK daily_observations, with `date` | +| date | date | N | FK daily_observations, with `stationId` | +| obsTime | time | N | A (700, 800, 1000, 1300, 1900); F52 1200 on some days | +| stationPressureIn | numeric | Y | B, 0700 and 1900 only | +| attachedThermC | numeric | Y | C53 "Attached Therm."; F53 at 1200 | +| observedBarometerMb | numeric | Y | C54 "Observed Barometer"; F54 at 1200 | +| pressureCorrectionMb | numeric | Y | C55 "Total Correction"; F55 at 1200 | +| dryBulbF | numeric | Y | C | +| wetBulbF | numeric | Y | D | +| dewpointF | smallint | Y | E | +| humidityPct | smallint | Y | F (0.8 becomes 80) | +| maxTempF | smallint | Y | G, 0700 only, 24 hours ending 0700 | +| minTempF | smallint | Y | H, 0700 only | +| precipIn | numeric | Y | I, 0700 only | +| precipTrace | boolean | N | Our flag for when I is `T` | +| snowfallIn | numeric | Y | J, 0700 only | +| snowDepthIn | numeric | Y | K, 0700 only | +| vaporPressureMb | numeric | Y | L | ## daily_records From 55b4830764b1254851f54fd6abe6b5c8c6398c35 Mon Sep 17 00:00:00 2001 From: Naman Rusia Date: Mon, 5 Oct 2026 18:34:33 -0400 Subject: [PATCH 11/16] Add notes to daily_records --- docs/database-schema.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/database-schema.md b/docs/database-schema.md index 7e533253c..4714deabb 100644 --- a/docs/database-schema.md +++ b/docs/database-schema.md @@ -166,6 +166,7 @@ Source: historical sheet. One sheet row becomes one row. PK `(stationId, month, | peakGustDir | varchar(3) | Y | "Wind Dir" | | peakGustIsEstimated | boolean | N | "Estimated?" | | peakGustYears | smallint[] | Y | "Gust Year(s)" | +| notes | text | Y | "Notes"; internal to staff | | importId | int | N | FK imports; the last upload that wrote it | ## audit_log From 610cf51d4660ed06279ecb803862bf0875bc0cb6 Mon Sep 17 00:00:00 2001 From: Naman Rusia Date: Mon, 5 Oct 2026 18:42:54 -0400 Subject: [PATCH 12/16] Define users table in schema doc --- docs/database-schema.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/database-schema.md b/docs/database-schema.md index 4714deabb..8b9e79a49 100644 --- a/docs/database-schema.md +++ b/docs/database-schema.md @@ -28,7 +28,15 @@ erDiagram ## users -The scaffold's existing `User` entity, unchanged: `id`, `status`, `firstName`, `lastName`, `email`. +Source: the app. `main` no longer has the scaffold's `User` entity, so this schema creates the table, with the same columns. + +| Column | Type | Null | Note | +| --------- | ------ | ---- | ---------------------------------------- | +| id | serial | N | PK | +| email | text | N | Unique | +| firstName | text | N | | +| lastName | text | N | | +| status | text | N | `ADMIN` or `STANDARD`; text, not an enum | ## stations From a21e43deb7c4e4a0586cc7a1db3870ac44a06bff Mon Sep 17 00:00:00 2001 From: Naman Rusia Date: Mon, 5 Oct 2026 21:04:27 -0400 Subject: [PATCH 13/16] Add initial schema migration and Kysely table types --- db/migrations/0001_init_schema.ts | 246 ++++++++++++++++++++++++++++++ db/types.ts | 143 +++++++++++++++++ docs/database-schema.md | 8 +- package.json | 1 + yarn.lock | 5 + 5 files changed, 399 insertions(+), 4 deletions(-) create mode 100644 db/migrations/0001_init_schema.ts create mode 100644 db/types.ts diff --git a/db/migrations/0001_init_schema.ts b/db/migrations/0001_init_schema.ts new file mode 100644 index 000000000..02a566a54 --- /dev/null +++ b/db/migrations/0001_init_schema.ts @@ -0,0 +1,246 @@ +import { Kysely, sql } from 'kysely'; + +// Tables, columns, and sources are documented in docs/database-schema.md. +export async function up(db: Kysely): Promise { + await db.schema + .createTable('users') + .addColumn('id', 'serial', (col) => col.primaryKey()) + .addColumn('email', 'text', (col) => col.notNull().unique()) + .addColumn('firstName', 'text', (col) => col.notNull()) + .addColumn('lastName', 'text', (col) => col.notNull()) + .addColumn('status', 'text', (col) => col.notNull()) + .execute(); + + await db.schema + .createTable('stations') + .addColumn('id', 'serial', (col) => col.primaryKey()) + .addColumn('code', 'text', (col) => col.notNull().unique()) + .execute(); + + await db.insertInto('stations').values({ code: '19737-02' }).execute(); + + await db.schema + .createTable('imports') + .addColumn('id', 'serial', (col) => col.primaryKey()) + .addColumn('kind', 'text', (col) => col.notNull()) + .addColumn('fileName', 'text', (col) => col.notNull()) + .addColumn('s3Key', 'text', (col) => col.notNull()) + .addColumn('fileHash', 'text', (col) => col.notNull()) + .addColumn('uploadedBy', 'integer', (col) => + col.notNull().references('users.id'), + ) + .addColumn('uploadedAt', 'timestamptz', (col) => + col.notNull().defaultTo(sql`now()`), + ) + .addColumn('status', 'text', (col) => col.notNull()) + .addColumn('errors', 'jsonb') + .execute(); + + await db.schema + .createTable('daily_observations') + .addColumn('stationId', 'integer', (col) => + col.notNull().references('stations.id'), + ) + .addColumn('date', 'date', (col) => col.notNull()) + .addColumn('maxTempF', 'smallint') + .addColumn('minTempF', 'smallint') + .addColumn('avgTempF', 'numeric') + .addColumn('normalTempF', 'smallint') + .addColumn('precipIn', 'numeric') + .addColumn('precipTrace', 'boolean', (col) => col.notNull()) + .addColumn('snowfallIn', 'numeric') + .addColumn('snowDepthIn', 'numeric') + .addColumn('sunshineMin', 'smallint') + .addColumn('sunshinePossibleMin', 'smallint') + .addColumn('sunshinePct', 'smallint') + .addColumn('fastestMileMph', 'smallint') + .addColumn('fastestMileDir', 'varchar(3)') + .addColumn('fastestMileTime', 'time') + .addColumn('peakGustKts', 'smallint') + .addColumn('peakGustDir', 'varchar(3)') + .addColumn('peakGustTime', 'time') + .addColumn('avgStationPressureMb', 'numeric') + .addColumn('sunrise', 'time') + .addColumn('sunset', 'time') + .addColumn('observerInitials', 'text') + .addColumn('remarks', 'text') + .addColumn('importId', 'integer', (col) => + col.notNull().references('imports.id'), + ) + .addPrimaryKeyConstraint('daily_observations_pkey', ['stationId', 'date']) + .execute(); + + await db.schema + .createTable('hourly_observations') + .addColumn('stationId', 'integer', (col) => col.notNull()) + .addColumn('date', 'date', (col) => col.notNull()) + .addColumn('hour', 'smallint', (col) => col.notNull()) + .addColumn('tempF', 'smallint') + .addColumn('precipIn', 'numeric') + .addColumn('precipTrace', 'boolean', (col) => col.notNull()) + .addColumn('windDir', 'varchar(3)') + .addColumn('windSpeedMph', 'smallint') + .addColumn('sunshineMin', 'smallint') + .addColumn('skyCover', 'smallint') + .addColumn('visibilityMi', 'numeric') + .addColumn('presentWeather', 'text') + .addColumn('humidityPct', 'smallint') + .addColumn('mountainVis', 'text') + .addColumn('remarks', 'text') + .addPrimaryKeyConstraint('hourly_observations_pkey', [ + 'stationId', + 'date', + 'hour', + ]) + .addForeignKeyConstraint( + 'hourly_observations_day_fkey', + ['stationId', 'date'], + 'daily_observations', + ['stationId', 'date'], + (fk) => fk.onDelete('cascade'), + ) + .execute(); + + await db.schema + .createTable('scheduled_observations') + .addColumn('stationId', 'integer', (col) => col.notNull()) + .addColumn('date', 'date', (col) => col.notNull()) + .addColumn('obsTime', 'time', (col) => col.notNull()) + .addColumn('stationPressureIn', 'numeric') + .addColumn('attachedThermC', 'numeric') + .addColumn('observedBarometerMb', 'numeric') + .addColumn('pressureCorrectionMb', 'numeric') + .addColumn('dryBulbF', 'numeric') + .addColumn('wetBulbF', 'numeric') + .addColumn('dewpointF', 'smallint') + .addColumn('humidityPct', 'smallint') + .addColumn('maxTempF', 'smallint') + .addColumn('minTempF', 'smallint') + .addColumn('precipIn', 'numeric') + .addColumn('precipTrace', 'boolean', (col) => col.notNull()) + .addColumn('snowfallIn', 'numeric') + .addColumn('snowDepthIn', 'numeric') + .addColumn('vaporPressureMb', 'numeric') + .addPrimaryKeyConstraint('scheduled_observations_pkey', [ + 'stationId', + 'date', + 'obsTime', + ]) + .addForeignKeyConstraint( + 'scheduled_observations_day_fkey', + ['stationId', 'date'], + 'daily_observations', + ['stationId', 'date'], + (fk) => fk.onDelete('cascade'), + ) + .execute(); + + await db.schema + .createTable('daily_records') + .addColumn('stationId', 'integer', (col) => + col.notNull().references('stations.id'), + ) + .addColumn('month', 'smallint', (col) => col.notNull()) + .addColumn('day', 'smallint', (col) => col.notNull()) + .addColumn('highF', 'smallint') + .addColumn('highYears', sql`smallint[]`) + .addColumn('lowF', 'smallint') + .addColumn('lowYears', sql`smallint[]`) + .addColumn('precipIn', 'numeric') + .addColumn('precipYears', sql`smallint[]`) + .addColumn('snowIn', 'numeric') + .addColumn('snowYears', sql`smallint[]`) + .addColumn('peakGustMph', 'smallint') + .addColumn('peakGustDir', 'varchar(3)') + .addColumn('peakGustIsEstimated', 'boolean', (col) => col.notNull()) + .addColumn('peakGustYears', sql`smallint[]`) + .addColumn('notes', 'text') + .addColumn('importId', 'integer', (col) => + col.notNull().references('imports.id'), + ) + .addPrimaryKeyConstraint('daily_records_pkey', [ + 'stationId', + 'month', + 'day', + ]) + .execute(); + + await db.schema + .createType('audit_log_action') + .asEnum(['insert', 'update', 'delete']) + .execute(); + + await db.schema + .createTable('audit_log') + .addColumn('id', 'serial', (col) => col.primaryKey()) + .addColumn('tableName', 'text', (col) => col.notNull()) + .addColumn('rowKey', 'jsonb', (col) => col.notNull()) + .addColumn('action', sql`audit_log_action`, (col) => col.notNull()) + .addColumn('oldData', 'jsonb') + .addColumn('newData', 'jsonb') + .addColumn('changedBy', 'integer', (col) => col.references('users.id')) + .addColumn('importId', 'integer', (col) => col.references('imports.id')) + .addColumn('changedAt', 'timestamptz', (col) => + col.notNull().defaultTo(sql`now()`), + ) + .execute(); + + // Trigger arguments are the table's primary key columns, used for rowKey. + // Writers identify themselves with set_config('app.user_id' / 'app.import_id', + // ..., true) inside their transaction; hourly and scheduled rows have no + // importId column, so the row itself can't say which import wrote it. + await sql` + CREATE FUNCTION audit_row() RETURNS trigger AS $$ + DECLARE + rec jsonb := to_jsonb(CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END); + key jsonb := '{}'; + col text; + BEGIN + FOREACH col IN ARRAY TG_ARGV LOOP + key := key || jsonb_build_object(col, rec -> col); + END LOOP; + + INSERT INTO audit_log ("tableName", "rowKey", action, "oldData", "newData", "changedBy", "importId") + VALUES ( + TG_TABLE_NAME, + key, + lower(TG_OP)::audit_log_action, + CASE WHEN TG_OP <> 'INSERT' THEN to_jsonb(OLD) END, + CASE WHEN TG_OP <> 'DELETE' THEN to_jsonb(NEW) END, + nullif(current_setting('app.user_id', true), '')::int, + nullif(current_setting('app.import_id', true), '')::int + ); + RETURN NULL; + END; + $$ LANGUAGE plpgsql + `.execute(db); + + const audited: [string, string[]][] = [ + ['daily_observations', ['stationId', 'date']], + ['hourly_observations', ['stationId', 'date', 'hour']], + ['scheduled_observations', ['stationId', 'date', 'obsTime']], + ['daily_records', ['stationId', 'month', 'day']], + ]; + for (const [table, key] of audited) { + await sql` + CREATE TRIGGER ${sql.id(`${table}_audit`)} + AFTER INSERT OR UPDATE OR DELETE ON ${sql.table(table)} + FOR EACH ROW EXECUTE FUNCTION audit_row(${sql.raw( + key.map((col) => `'${col}'`).join(', '), + )}) + `.execute(db); + } +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable('audit_log').execute(); + await db.schema.dropType('audit_log_action').execute(); + await db.schema.dropTable('daily_records').execute(); + await db.schema.dropTable('scheduled_observations').execute(); + await db.schema.dropTable('hourly_observations').execute(); + await db.schema.dropTable('daily_observations').execute(); + await sql`DROP FUNCTION audit_row()`.execute(db); + await db.schema.dropTable('imports').execute(); + await db.schema.dropTable('stations').execute(); + await db.schema.dropTable('users').execute(); +} diff --git a/db/types.ts b/db/types.ts new file mode 100644 index 000000000..8f76d8713 --- /dev/null +++ b/db/types.ts @@ -0,0 +1,143 @@ +import type { ColumnType, Generated } from 'kysely'; + +// Table types for Kysely, matching db/migrations. pg returns numeric as a +// string so no precision is lost, and date/timestamptz as a Date. +type Numeric = ColumnType; +type DateColumn = ColumnType; +type Json = ColumnType; + +export interface UsersTable { + id: Generated; + email: string; + firstName: string; + lastName: string; + status: string; +} + +export interface StationsTable { + id: Generated; + code: string; +} + +export interface ImportsTable { + id: Generated; + kind: string; + fileName: string; + s3Key: string; + fileHash: string; + uploadedBy: number; + uploadedAt: ColumnType; + status: string; + errors: Json | null; +} + +export interface DailyObservationsTable { + stationId: number; + date: DateColumn; + maxTempF: number | null; + minTempF: number | null; + avgTempF: Numeric | null; + normalTempF: number | null; + precipIn: Numeric | null; + precipTrace: boolean; + snowfallIn: Numeric | null; + snowDepthIn: Numeric | null; + sunshineMin: number | null; + sunshinePossibleMin: number | null; + sunshinePct: number | null; + fastestMileMph: number | null; + fastestMileDir: string | null; + fastestMileTime: string | null; + peakGustKts: number | null; + peakGustDir: string | null; + peakGustTime: string | null; + avgStationPressureMb: Numeric | null; + sunrise: string | null; + sunset: string | null; + observerInitials: string | null; + remarks: string | null; + importId: number; +} + +export interface HourlyObservationsTable { + stationId: number; + date: DateColumn; + hour: number; + tempF: number | null; + precipIn: Numeric | null; + precipTrace: boolean; + windDir: string | null; + windSpeedMph: number | null; + sunshineMin: number | null; + skyCover: number | null; + visibilityMi: Numeric | null; + presentWeather: string | null; + humidityPct: number | null; + mountainVis: string | null; + remarks: string | null; +} + +export interface ScheduledObservationsTable { + stationId: number; + date: DateColumn; + obsTime: string; + stationPressureIn: Numeric | null; + attachedThermC: Numeric | null; + observedBarometerMb: Numeric | null; + pressureCorrectionMb: Numeric | null; + dryBulbF: Numeric | null; + wetBulbF: Numeric | null; + dewpointF: number | null; + humidityPct: number | null; + maxTempF: number | null; + minTempF: number | null; + precipIn: Numeric | null; + precipTrace: boolean; + snowfallIn: Numeric | null; + snowDepthIn: Numeric | null; + vaporPressureMb: Numeric | null; +} + +export interface DailyRecordsTable { + stationId: number; + month: number; + day: number; + highF: number | null; + highYears: number[] | null; + lowF: number | null; + lowYears: number[] | null; + precipIn: Numeric | null; + precipYears: number[] | null; + snowIn: Numeric | null; + snowYears: number[] | null; + peakGustMph: number | null; + peakGustDir: string | null; + peakGustIsEstimated: boolean; + peakGustYears: number[] | null; + notes: string | null; + importId: number; +} + +// Written only by the audit_row() trigger. +export interface AuditLogTable { + id: Generated; + tableName: string; + rowKey: unknown; + action: 'insert' | 'update' | 'delete'; + oldData: unknown; + newData: unknown; + changedBy: number | null; + importId: number | null; + changedAt: Generated; +} + +export interface Database { + users: UsersTable; + stations: StationsTable; + imports: ImportsTable; + daily_observations: DailyObservationsTable; + hourly_observations: HourlyObservationsTable; + scheduled_observations: ScheduledObservationsTable; + daily_records: DailyRecordsTable; + audit_log: AuditLogTable; +} diff --git a/docs/database-schema.md b/docs/database-schema.md index 8b9e79a49..5d8cac6bb 100644 --- a/docs/database-schema.md +++ b/docs/database-schema.md @@ -1,13 +1,13 @@ # Database schema -Draft for [#3](https://github.com/Code-4-Community/bho/issues/3). PostgreSQL on AWS RDS. +Draft for [#3](https://github.com/Code-4-Community/bho/issues/3). PostgreSQL on AWS RDS. Created by `db/migrations/0001_init_schema.ts`; Kysely table types are in `db/types.ts`. Sources: - **Daily sheet**: [DAILY - 07. July2026.xlsx](https://docs.google.com/spreadsheets/d/1cWloMpYHZDGkUFulmeW0g1bY3J97Ts4b/edit), tab `Sheet1`. One 60-row block per day. Cell references are for the July 1 block; add 60 rows for each later day. - **Historical sheet**: [BHO_WeatherRecords_Master_TestData.xlsx](https://docs.google.com/spreadsheets/d/1WI8CxTJ7edRajo3JHZX0WmTklvCqRy5x/edit), tab `All Daily Records`, rows 4-369. -Column names are camelCase to match the repo's TypeORM naming strategy, which uses property names as column names. +Column names are camelCase, so Kysely queries use the same names as the TypeScript types. Raw SQL has to quote them (`"maxTempF"`). ## Relations @@ -60,7 +60,7 @@ Source: the app, one row per uploaded spreadsheet. The uploaded file stays in S3 | fileHash | text | N | SHA-256 of the file, to catch duplicate uploads | | uploadedBy | int | N | FK users | | uploadedAt | timestamptz | N | Defaults to now | -| status | enum | N | `succeeded` or `failed` | +| status | text | N | `succeeded` or `failed`; text, not an enum | | errors | jsonb | Y | Flagged rows and the reason for each | Ideas for later: @@ -193,4 +193,4 @@ Source: a Postgres trigger, one row per inserted, updated, or deleted row in `da | importId | int | Y | FK imports; set when an import made the change | | changedAt | timestamptz | N | Defaults to now | -One SQL function is attached to the four data tables, so no code path can skip the log. +One SQL function is attached to the four data tables, so no code path can skip the log. Writers set `app.user_id` and `app.import_id` with `set_config(..., true)` inside their transaction, and the trigger copies them into `changedBy` and `importId`. Hourly and scheduled rows have no `importId` column, so the row alone can't say which import wrote it. diff --git a/package.json b/package.json index c9dd38d2d..e30b19f63 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "global": "^4.4.0", "jsonwebtoken": "^9.0.3", "jwks-rsa": "^4.1.0", + "kysely": "^0.27.0", "listr2": "11", "nodemailer": "^9.0.3", "passport": "^0.7.0", diff --git a/yarn.lock b/yarn.lock index d7ac9b68a..4a1687870 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9392,6 +9392,11 @@ keyv@^4.5.3: dependencies: json-buffer "3.0.1" +kysely@^0.27.0: + version "0.27.6" + resolved "https://registry.yarnpkg.com/kysely/-/kysely-0.27.6.tgz#b2a908e160e510f9945586ca9de1616c187a2a8e" + integrity sha512-FIyV/64EkKhJmjgC0g2hygpBv5RNWVPyNCqSAD7eTCv6eFWNIi4PN1UvdSJGicN/o35bnevgis4Y0UDC0qi8jQ== + language-subtag-registry@^0.3.20: version "0.3.23" resolved "https://registry.yarnpkg.com/language-subtag-registry/-/language-subtag-registry-0.3.23.tgz#23529e04d9e3b74679d70142df3fd2eb6ec572e7" From 9dfa6bf33610597aa281631d4a5bc68ff62e70fd Mon Sep 17 00:00:00 2001 From: Naman Rusia Date: Thu, 8 Oct 2026 14:20:08 -0400 Subject: [PATCH 14/16] Return date columns as YYYY-MM-DD strings --- db/client.ts | 13 +++++++++++++ db/types.ts | 10 +++++----- docs/database-schema.md | 2 +- 3 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 db/client.ts diff --git a/db/client.ts b/db/client.ts new file mode 100644 index 000000000..2456bbc00 --- /dev/null +++ b/db/client.ts @@ -0,0 +1,13 @@ +import { Kysely, PostgresDialect } from 'kysely'; +import { Pool, PoolConfig, types } from 'pg'; +import type { Database } from './types'; + +// pg turns a date into a Date at local midnight, so outside UTC (laptops run +// Eastern) 2026-07-01 reads back as June 30 in UTC. Keep dates as 'YYYY-MM-DD'. +types.setTypeParser(types.builtins.DATE, (value) => value); + +export function createDb(config: PoolConfig): Kysely { + return new Kysely({ + dialect: new PostgresDialect({ pool: new Pool(config) }), + }); +} diff --git a/db/types.ts b/db/types.ts index 8f76d8713..a550120a0 100644 --- a/db/types.ts +++ b/db/types.ts @@ -1,9 +1,9 @@ import type { ColumnType, Generated } from 'kysely'; // Table types for Kysely, matching db/migrations. pg returns numeric as a -// string so no precision is lost, and date/timestamptz as a Date. +// string so no precision is lost, timestamptz as a Date, and date as +// 'YYYY-MM-DD' when connected through createDb() in db/client.ts. type Numeric = ColumnType; -type DateColumn = ColumnType; type Json = ColumnType; export interface UsersTable { @@ -33,7 +33,7 @@ export interface ImportsTable { export interface DailyObservationsTable { stationId: number; - date: DateColumn; + date: string; maxTempF: number | null; minTempF: number | null; avgTempF: Numeric | null; @@ -61,7 +61,7 @@ export interface DailyObservationsTable { export interface HourlyObservationsTable { stationId: number; - date: DateColumn; + date: string; hour: number; tempF: number | null; precipIn: Numeric | null; @@ -79,7 +79,7 @@ export interface HourlyObservationsTable { export interface ScheduledObservationsTable { stationId: number; - date: DateColumn; + date: string; obsTime: string; stationPressureIn: Numeric | null; attachedThermC: Numeric | null; diff --git a/docs/database-schema.md b/docs/database-schema.md index 5d8cac6bb..fa3f8d230 100644 --- a/docs/database-schema.md +++ b/docs/database-schema.md @@ -1,6 +1,6 @@ # Database schema -Draft for [#3](https://github.com/Code-4-Community/bho/issues/3). PostgreSQL on AWS RDS. Created by `db/migrations/0001_init_schema.ts`; Kysely table types are in `db/types.ts`. +Draft for [#3](https://github.com/Code-4-Community/bho/issues/3). PostgreSQL on AWS RDS. Created by `db/migrations/0001_init_schema.ts`; Kysely table types are in `db/types.ts`. Connect with `createDb()` in `db/client.ts`, which returns `date` columns as `'YYYY-MM-DD'` strings rather than a `Date` at local midnight. Sources: From 8867ca38c177f4bd02b0cd28e04958c5b71da073 Mon Sep 17 00:00:00 2001 From: Naman Rusia Date: Thu, 8 Oct 2026 14:20:09 -0400 Subject: [PATCH 15/16] Note that imports rows are never updated or deleted --- docs/database-schema.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/database-schema.md b/docs/database-schema.md index fa3f8d230..e94144be9 100644 --- a/docs/database-schema.md +++ b/docs/database-schema.md @@ -49,7 +49,7 @@ Source: the app. Starts with one row, `19737-02`; the client plans to add more. ## imports -Source: the app, one row per uploaded spreadsheet. The uploaded file stays in S3 as the raw source of truth. +Source: the app, one row per uploaded spreadsheet. The uploaded file stays in S3 as the raw source of truth. Rows are never updated or deleted. Each upload, including a re-upload of the same file, adds a new row. | Column | Type | Null | Note | | ---------- | ----------- | ---- | ----------------------------------------------- | From 3552010f63b1d9da5dada3cac521d20cd6ed0c14 Mon Sep 17 00:00:00 2001 From: Naman Rusia Date: Thu, 8 Oct 2026 14:28:00 -0400 Subject: [PATCH 16/16] Correct date parser comment --- db/client.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/client.ts b/db/client.ts index 2456bbc00..d0446b347 100644 --- a/db/client.ts +++ b/db/client.ts @@ -2,8 +2,8 @@ import { Kysely, PostgresDialect } from 'kysely'; import { Pool, PoolConfig, types } from 'pg'; import type { Database } from './types'; -// pg turns a date into a Date at local midnight, so outside UTC (laptops run -// Eastern) 2026-07-01 reads back as June 30 in UTC. Keep dates as 'YYYY-MM-DD'. +// pg turns a date into a Date at local midnight, so 2026-07-01 is 04:00Z on a +// laptop in Eastern time and 00:00Z in Lambda. Keep dates as 'YYYY-MM-DD'. types.setTypeParser(types.builtins.DATE, (value) => value); export function createDb(config: PoolConfig): Kysely {