From ea3eb2d215378536dc41bb23f4f5ed1d930cdb20 Mon Sep 17 00:00:00 2001 From: mshekh-123 Date: Fri, 27 Mar 2026 19:13:48 +0530 Subject: [PATCH 01/10] testing github action --- .github/workflows/data-pr-checks.yml | 30 +-- .../conftest.cpython-314-pytest-9.0.2.pyc | Bin 0 -> 592 bytes submission/catalog/catalog.json | 77 ++++++ submission/conftest.py | 6 + submission/pipeline/__init__.py | 0 .../__pycache__/__init__.cpython-314.pyc | Bin 0 -> 186 bytes .../pipeline/__pycache__/cdc.cpython-314.pyc | Bin 0 -> 5653 bytes .../pipeline/__pycache__/lake.cpython-314.pyc | Bin 0 -> 3364 bytes .../__pycache__/warehouse.cpython-314.pyc | Bin 0 -> 5351 bytes submission/pipeline/cdc.py | 89 +++++++ submission/pipeline/lake.py | 69 +++++ submission/pipeline/warehouse.py | 124 +++++++++ submission/requirements.txt | 2 + submission/scripts/check_schema_contracts.py | 60 +++++ submission/scripts/run_data_quality_checks.py | 212 +++++++++++++++ submission/scripts/validate_catalog.py | 77 ++++++ submission/source/__init__.py | 0 .../__pycache__/__init__.cpython-314.pyc | Bin 0 -> 184 bytes .../source/__pycache__/models.cpython-314.pyc | Bin 0 -> 3369 bytes submission/source/models.py | 84 ++++++ .../conftest.cpython-314-pytest-9.0.2.pyc | Bin 0 -> 3910 bytes .../test_catalog.cpython-314-pytest-9.0.2.pyc | Bin 0 -> 19689 bytes .../test_cdc.cpython-314-pytest-9.0.2.pyc | Bin 0 -> 20869 bytes ..._data_quality.cpython-314-pytest-9.0.2.pyc | Bin 0 -> 21789 bytes ...ema_contracts.cpython-314-pytest-9.0.2.pyc | Bin 0 -> 16244 bytes submission/tests/conftest.py | 125 +++++++++ submission/tests/test_catalog.py | 134 ++++++++++ submission/tests/test_cdc.py | 135 ++++++++++ submission/tests/test_data_quality.py | 252 ++++++++++++++++++ submission/tests/test_schema_contracts.py | 170 ++++++++++++ 30 files changed, 1631 insertions(+), 15 deletions(-) create mode 100644 submission/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc create mode 100644 submission/catalog/catalog.json create mode 100644 submission/conftest.py create mode 100644 submission/pipeline/__init__.py create mode 100644 submission/pipeline/__pycache__/__init__.cpython-314.pyc create mode 100644 submission/pipeline/__pycache__/cdc.cpython-314.pyc create mode 100644 submission/pipeline/__pycache__/lake.cpython-314.pyc create mode 100644 submission/pipeline/__pycache__/warehouse.cpython-314.pyc create mode 100644 submission/pipeline/cdc.py create mode 100644 submission/pipeline/lake.py create mode 100644 submission/pipeline/warehouse.py create mode 100644 submission/requirements.txt create mode 100644 submission/scripts/check_schema_contracts.py create mode 100644 submission/scripts/run_data_quality_checks.py create mode 100644 submission/scripts/validate_catalog.py create mode 100644 submission/source/__init__.py create mode 100644 submission/source/__pycache__/__init__.cpython-314.pyc create mode 100644 submission/source/__pycache__/models.cpython-314.pyc create mode 100644 submission/source/models.py create mode 100644 submission/tests/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc create mode 100644 submission/tests/__pycache__/test_catalog.cpython-314-pytest-9.0.2.pyc create mode 100644 submission/tests/__pycache__/test_cdc.cpython-314-pytest-9.0.2.pyc create mode 100644 submission/tests/__pycache__/test_data_quality.cpython-314-pytest-9.0.2.pyc create mode 100644 submission/tests/__pycache__/test_schema_contracts.cpython-314-pytest-9.0.2.pyc create mode 100644 submission/tests/conftest.py create mode 100644 submission/tests/test_catalog.py create mode 100644 submission/tests/test_cdc.py create mode 100644 submission/tests/test_data_quality.py create mode 100644 submission/tests/test_schema_contracts.py diff --git a/.github/workflows/data-pr-checks.yml b/.github/workflows/data-pr-checks.yml index c655483..cd22328 100644 --- a/.github/workflows/data-pr-checks.yml +++ b/.github/workflows/data-pr-checks.yml @@ -22,13 +22,13 @@ jobs: pip install ruff black sqlfluff - name: Ruff - run: ruff check . + run: ruff check submission/ - name: Black - run: black --check . + run: black --check submission/ - name: SQLFluff - run: sqlfluff lint . + run: sqlfluff lint submission/ || true continue-on-error: true test-pipeline: @@ -44,11 +44,11 @@ jobs: - name: Install test dependencies run: | python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip install pytest + if [ -f submission/requirements.txt ]; then pip install -r submission/requirements.txt; fi - name: Run tests - run: pytest -q + run: pytest submission/ -q validate-models: name: validate-models @@ -95,12 +95,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f submission/requirements.txt ]; then pip install -r submission/requirements.txt; fi - name: Run schema contract checks run: | - if [ -f scripts/check_schema_contracts.py ]; then - python scripts/check_schema_contracts.py + if [ -f submission/scripts/check_schema_contracts.py ]; then + python submission/scripts/check_schema_contracts.py else echo "No schema contract script found. Add one or replace this step." exit 1 @@ -119,12 +119,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f submission/requirements.txt ]; then pip install -r submission/requirements.txt; fi - name: Run data quality checks run: | - if [ -f scripts/run_data_quality_checks.py ]; then - python scripts/run_data_quality_checks.py + if [ -f submission/scripts/run_data_quality_checks.py ]; then + python submission/scripts/run_data_quality_checks.py else echo "No data quality script found. Add one or replace this step." exit 1 @@ -143,12 +143,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f submission/requirements.txt ]; then pip install -r submission/requirements.txt; fi - name: Validate catalog metadata run: | - if [ -f scripts/validate_catalog.py ]; then - python scripts/validate_catalog.py + if [ -f submission/scripts/validate_catalog.py ]; then + python submission/scripts/validate_catalog.py else echo "No catalog validation script found. Add one or replace this step." exit 1 @@ -173,7 +173,7 @@ jobs: run: pip-audit || true - name: bandit - run: bandit -r . -x tests || true + run: bandit -r submission/ -x submission/tests || true - name: gitleaks uses: gitleaks/gitleaks-action@v2 diff --git a/submission/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc b/submission/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7309004e9fba8bc1f3e0c7323e666a6180a4045a GIT binary patch literal 592 zcmZutO=}cE5bc?bJCh}GF+s%BHh2n}84Q7V2*zW)gs|el!={>@6`u zt_cXa`Em4*_y>eQ+KYJcD&!AXlXbEl>_ff!c=f8f?%$m6F?yfhepMd{^CoCt_pfvL zft&+2WXN{dFnAhZc!izK85a)N<_+u)CY1=It4fUVDBfOZg{HWDd+N8|4jV=P<=$x- zQyk5$29sztoZx^qy2qA6_l)A9*2r^R@1X?bKM&sXtSEqk+pQFU(sjyFa~MNnGTi5& zc|!!AXEj%q(H1G^Y%Wuyj4V|xldYwB9|=+DTnHcUI#r^o!OuN0V{skb6=vKhB^iLU zNP`k9-&lpx&t9i|)aH8B+|Wg4eW<~AXd^$Gh zek*-W3`kMt`>!R<(CI5zy3|UeVPYjZlNK3f%QRfQtSea~r2kx@JyBxDK3=bM;Y#_~ z-Xi(ty6}+9?@kZ|7wp9^#@?}`+3&3PgLRJM!}z#=*gpwQR?gN=*UmOhH$FZ3w)o(j JE!#eI{Q*8Dq6Gi| literal 0 HcmV?d00001 diff --git a/submission/catalog/catalog.json b/submission/catalog/catalog.json new file mode 100644 index 0000000..83dfa55 --- /dev/null +++ b/submission/catalog/catalog.json @@ -0,0 +1,77 @@ +{ + "datasets": [ + { + "name": "lake_cdc_events", + "layer": "lake", + "description": "Append-only log of every CDC event captured from the payments source system. Preserves full change history for replay and point-in-time recovery.", + "owner": "data-platform", + "consumers": ["data-platform", "analytics", "audit"], + "update_cadence": "real-time", + "schema": { + "sequence": "INTEGER — monotonic capture offset", + "operation": "VARCHAR — insert | update | delete", + "table_name": "VARCHAR — source table name", + "primary_key": "VARCHAR — source row PK value", + "data": "VARCHAR (JSON) — full row snapshot at capture time", + "captured_at": "TIMESTAMP — UTC capture time" + } + }, + { + "name": "wh_customers", + "layer": "warehouse", + "description": "Current-state customer snapshot. Reflects the latest known state of each customer row. Soft-deleted rows are flagged with _deleted=true.", + "owner": "data-platform", + "consumers": ["analytics", "product", "finance"], + "update_cadence": "near-real-time", + "schema": { + "customer_id": "VARCHAR — primary key", + "name": "VARCHAR", + "email": "VARCHAR", + "status": "VARCHAR — active | suspended | closed", + "created_at": "TIMESTAMP", + "updated_at": "TIMESTAMP", + "_cdc_seq": "INTEGER — sequence of last CDC event", + "_deleted": "BOOLEAN — true if source row was deleted" + } + }, + { + "name": "wh_wallets", + "layer": "warehouse", + "description": "Current-state wallet snapshot. Balance reflects the latest value written via CDC. Negative balances are a data quality violation.", + "owner": "data-platform", + "consumers": ["analytics", "finance"], + "update_cadence": "near-real-time", + "schema": { + "wallet_id": "VARCHAR — primary key", + "customer_id": "VARCHAR — FK to wh_customers", + "balance": "DECIMAL(18,2)", + "currency": "VARCHAR", + "status": "VARCHAR — active | frozen | closed", + "created_at": "TIMESTAMP", + "updated_at": "TIMESTAMP", + "_cdc_seq": "INTEGER", + "_deleted": "BOOLEAN" + } + }, + { + "name": "wh_transactions", + "layer": "warehouse", + "description": "Current-state transaction snapshot. Each row is the latest state of a transaction from the source. Amount must always be positive.", + "owner": "data-platform", + "consumers": ["analytics", "finance", "audit"], + "update_cadence": "near-real-time", + "schema": { + "transaction_id": "VARCHAR — primary key", + "wallet_id": "VARCHAR — FK to wh_wallets", + "amount": "DECIMAL(18,2) — always > 0", + "direction": "VARCHAR — credit | debit", + "status": "VARCHAR — pending | settled | failed | reversed", + "reference": "VARCHAR — nullable external reference", + "created_at": "TIMESTAMP", + "settled_at": "TIMESTAMP — nullable until settled", + "_cdc_seq": "INTEGER", + "_deleted": "BOOLEAN" + } + } + ] +} diff --git a/submission/conftest.py b/submission/conftest.py new file mode 100644 index 0000000..ce3c67b --- /dev/null +++ b/submission/conftest.py @@ -0,0 +1,6 @@ +"""Root conftest — adds submission/ to sys.path so tests can import source/pipeline.""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) diff --git a/submission/pipeline/__init__.py b/submission/pipeline/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/submission/pipeline/__pycache__/__init__.cpython-314.pyc b/submission/pipeline/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1523f32b32810272b170ad39e00932cfc189806a GIT binary patch literal 186 zcmdPq_I|p@<2{{|u766|NszoLW?@ zUy_=fQI=YiS(2}xU7Ay>UzA#qUko8rOG*p$QxZ!ObrXw=Gt={OQ}arS^@~fBax;Pa z{5<`F%!1UM%)C_n`1s7c%#!$cy@JYH95%W6DWy57c15f}dq6HJ1~EP{Gcqz3F#}lu D5y~+p literal 0 HcmV?d00001 diff --git a/submission/pipeline/__pycache__/cdc.cpython-314.pyc b/submission/pipeline/__pycache__/cdc.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a17fa0bcc202261717534949b737d20e8d1f1dd1 GIT binary patch literal 5653 zcmb_gU2GiH6}~e&yWSsr*YQvOF!3ZN!3&8OqKII}xI|9CCSc5jETY6&jd#cPD(ji$ z&RxJ!q*|qFleRRWszm5ZqdrvRi9A=8+DcXX(uagZsym`8RV!7z*;*p2JoP*G?yNV# zG*!e%d+)jb=boSMoV$;wJCY3FM;HEBUQ03dPill;R1sSJ0*Ga1F~QC-OPkatbfNbX ztxrZK3}H+}MO3yUld*}Ih)uMKHrX~N;}Z#y=w~y@-RucwB^R|rn%Y&QW{h68n_1Cq z%!(b-Rqr8HG0`r^+AtOm$3%yf7+~3yJ%%lkBrEe)!FK1JaO#NZyyfsEv+Vi~7r9!sgz34GQ*nf2dafxJ9Mkt|yyzsW z+$+_J*vmF?mgN*ZUNYY>- zRKcF<+jEXs8UY`bio<=>E?9GlXRBV>74qDQ?F$p9;!I+^xs~nW7Fd?#k_W$llZwuu zH#g@yB5MlIP9BgyP7d96|$*dNnY!`R1McH%xtR5sF2)hUt z{2)44b}A+G#Ys4#yyygRD$jYYgN2c!?ozR#CRrJ(q1E33vCL+e&|oV1nH~{&1m-eh zSo%JUNGj5#3`s>X8Y5~Z3R95kJ{%fI%t~N1Nuy_&XtQ82R=Zr)0kcSNW0g)3w=yDO zb;2yR1c|1af`~78knpMwmlq78!k(=-L8{8ji#A^>oOPDIOqa?<@dhUDH*uqxz>pHa zVa+cB3cp1C1rW>ZbaP<3VXe&$5WCswh9DT5j`I<)%%c1@eg8l0Rfh`8R{oF)g6 znkZglb($Mkxl^!CKPQ~x0<5(xyxiGZrIv%5JztfMs8w^s@BwJ7Jnt?#uJChyZFaGY zw&&)m<*HLDyH2iHDh^ebg7|2~UYsr2V?kF_OhaMcPMqp*L9xt!Gx*#6zuy0ccm8z# z_vdeqyvlpg3#DgD&YWGVh{Bv*6dqqv?57$ISSr{eh`^46cxbgn(qcoCjKl>&YV|&O zMxLF~pZ*?WCG_>NSF{Dq(kwlvK{S!iX>JhXxFB}}V+uCMd(Z_QVsjo}w1rQ*?%`W$ zUVEX?Fk`1s2)1tAthN7P%=EiaEVFeMji%SAxFTdF?=GJBJ3DBEG^C=#t|RBRPP%W|m2!z- z$ZXoQd9G{7e5nS=m?h-GZ*IgS!vc!4pfi8;#mQrZm#2?gM_-+MdFo`4%-fZkbDVRJ z2QlB{!YN(T0>gJIb5f%FlteoU1sItql-;s`NVm0$v}HibbQ~uwvz1il(%8kZ+q<8< zoqF=quD+G7!H;@A-gRx)&B2jXrX3mhbRc{Cv5{K?M{evHU5#j4#_EwscY2jYI?`(q z)_zdxKq`4En+)QGf@?35I+KOMA|SXzG!*@2&8{>?GKIoinfrna-1RVwjutp86u1G_ zIGHskljD!02zm33gP z+0t{%=vhf_zh!LyYv1mh2VcG2cWU{>WwcMN^z<%ITqxc$de@E6%+$uTXA!^V2Lq!| ztN#>b3uNvU5la&QOaNR&0$kKGKqE#1>|$07G(Zw`n-vG0Xn7nFt zySRWM8C{Du=VcjjLl|kqQ4?Xc3?~4}QM|KFwYLg^rpJ@bJzqRIWdbK!VrwH0P%&~S z`C!1rcR^7NAmV#Kx1bOoKwElBLtsNqYlP_z(c}<)^yout48@g4Wc5|_AzXoRB>ld{o8=$ZRVpYrST0;C|y;&k{vf@@dgNd=gIi zI9kD$7NM(c`L}Su|FT&r)5A3BC>6v{ z@f(qqN~`%X+VwdU_Xy<#y6zH+nlCw-pf!>c(o3kD2eP~W@BSln;5U4K^`7|J`@eke z{ndQQ$>#Sbt-xxvx(R}klFbq*_??lXlp!kpEr=<2&M zvy?ioJ^eGr=CM~FI{}->>r?uP9Y9C63+t-5Y7Ng&qRMd5b7PiF8cjTv8Ue*u@3m$y z8AFmpk0x`l<>{pYLv|>ubEyGJ7_t%N+ZO0R<33n`Q=5$x>8^KJu)W2*?;4T9BPd>G zpFPz7i{YOS-^z@BIDFauIC(93J%4lX$OqbG?Ze@lnb8~Z(f@ps8THA|FLmXNU&V69 zg9R#ZF2UBsPD=Cl1Jz@g*$hxxbO!-pAtJqw*1B}yaBCGKu zsJ;@8GV+ZsC{)SEaSg!hb%V*{Te2ZRWuT)Mb)Bffoutq>p@ar6E!jJxD>lDNB{ ze8?kPAzf%)dnM&BtNngDWCly@Ma7dq$feElc?D&O!h;GA&!%eDoeaKj7KQK zr(0#^%|DNgd=xqcAlTD?9pDBUJAlRQ1r4svdq`)}JN1)}k0! z_iDbiunRej6!|aZmU(tW+w9GDM%&+k35~C&Z{$^{Non%SYk55%$s3!SwWS?yoLh}z zh7qpXsHdB6yN$klG*ogw%ab(q&q*UdX(>;c6%k`floEXm6{qBq=csrB1(K&i$t&X3MGq^P9dXJAIE6%>rx|fPH1NNTSmNKN zz5*3=4StXca#nSie5{cXaVqxW=M!3gpRMV!X#7qmvg|wU$OP}i!eTv%sI?krvGf-@ zAg&$(wMrC3p!)ioef{gn&@k0;bjZ{g8H(a{oG4;Fo96SV=(RdmqkkiDIdb$M)%-`z z@wr?`1|$DCQGfZw#8MUSAx<_EB<8qx&cVNug72egQup$DJ=CO(NJXFcsCpES@kR8J zCHbUCJghZsO^<8FogStQ{F@E_ojv+Dw)G#~eV;shZx_TLqm>?0+-cO RtJGXK4{6s z4co&OH>4kOR|w`&cE_#>oc6sMo21nZc{@IV9RhUdH+hXwe~k*>6dUqxBk)~MoO8W% z!fi0hSpadU?KyPAl_Iz`yAE#6_d=Yh3D@`V`91rwLdZcY7bsM?I#ZdTfz5?0uV4;cZ5hH|tuxEB@m=}BnjMHHXO3kH>evGEFZh)E zA4=(^1!%)}+%*?xai7ELOfYBC7}uh_?Rmbi<;CFw=2>}KsslO)tC((0Du-XO`a)n>K)vPy*u(}_{ zJuV1}kl#LMhoQUfHSpeHF>J0jT+Ds17`OqeyB;gbhn);sQNgl0`ZLSoz4*{8_=n#@ zw?!TkL(3h=u08zbz8pV)T}=!25G|6JuSaP+~OmeZqh2v{5O~Rf7 z!=RluW0t0?C0d=nGFPJIcj&^+DlPq_T&Y%+BU&}5W+@*I>Di-0@@&Z9J8)WcPkEtQ zx>_=+I=-+pH#hOrUO!-5y|&!EJZ;Wgn||TmV?OrMy#aR{Hg8$CSnH*GhYDZ>G$2F^PB1aCt^a!?U2G zj06SlgqwYJ0L4#_!t7@}Czq2z6`ltL6$%*Q2v900A64BMxiE@OI_i-2%7N_KL+AJ9 z__?Kq_PbTZj+N+OHjcpQ@P`tq!+v-m99H|TViU3s?lACs6uc8;DbY+!l4jzaMM8qa zastD14cy~5ch4n4%laL%aO@bt( zB#ZLRo@u7GoV?hPbA+s%>@I`b3CdCEIf8$Xa1olD_l#+kQB>;aycZR|$^{BO48v52 z6{iQU*;ob4B0Y5XcGAkEQFd4(@R4C^Eefm!+PfYNj{0PRo0d6fYH zf$vHV*IS=7Hs35SR7z%*BFb+*hYHKnb4#TPJ^S_qed;$po{O?wkce_e5D=wQ3`F^3 z!5bMCVAi$k?k0*I8 zSOrV?D)hMYUHK31qJ#hTlET1W`c532BK<>$14bcz&=2bF^nM@d>D{@$eSNRcvvXtn zh8(VLUzNlB_WXmwsof8Lcl$TDzbc&Wunv1NIpg324(|_;R1TnBNbjElbzdL+QXkyQ zzP*<%e4R?B(qCsuq3^Ev_)=RR{P#acurHLJxifw0impEu?@|>N?^Fr<;&_kKT^#Gf z*bDQyHPH!aQ~)mx>9=qwd`*4jG10UO2eR9(?#to#H8pzzN7$xX3cV8VGKej*5_8di zQ&fHjscA(jq?$>7*og>$lqR9mVQY4?F2<9Q{(k6tQOapHf-vf_YFw&!3|P5L zn5t7M#8CPm-?wx5*OzylPv-xeKi}5RKasr?apeCoZItv^-;dJhP&g>#@;BtyUxt@g zq{yeyNxV#IOWJ|zcW>{j@eiZ%{E3j*s@f6W{}xjixGTIgWygGkbfKjSqYeOOSk?jZ zd7fz(JM&SpA`_HqCOHOzM|HwCfnfo)!}%l7+Ry-1eK(acRBhS{7~LF_Wgno1Ua#C- z0G`D0EGx$GbY!@pi-Ocf!NB1-B+9v=AD3EDu8Q*|&V9tO02yU1RbyHfQqcr9AAj`k zVa{czO#34cTjW9J z5tq)S(z$|{*d>*S(lgZcF&Jfkj9R!^XK(Q-EXdake}Ha3scG6lBCF|-28lNMkmUbL fzWtE&{GFV6lt%6M=r!ckM^gyJ$64t<>c;;9zA^H* literal 0 HcmV?d00001 diff --git a/submission/pipeline/__pycache__/warehouse.cpython-314.pyc b/submission/pipeline/__pycache__/warehouse.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da28d229906ec66ff5cf0d96b82699210bbe1efa GIT binary patch literal 5351 zcmdrQU2GgjdG>DaZ|}}`{t1rbB%a2>^`(xl4WWcIad7N&?bx|o*XK*Bx;?G$#`ZRQ zyVu>lI5r53RNx8^#1v%rsYW2xRgs8zX$nH>3#d=Xk*R7{m55sLfW(7?Qvy8k&Ft-+ z&vi)4BO~q3%s1bBGxL4ld_VJ8Ls&pi{_yhe<*N~dKBgU~cYdk&&?7QOL#CbmPjY; zBVan4nlr=z8B}JPlk<<9RJo8M6==1S$Rht7^S1X616Q^}z&|OT; z16{+^iByo)8KFS*8G>KbrtnoV55qvOsLKUOpQnIcJU%dZ0>j2l4Iqp&2ALpS^pVoA2$^~*9_;`&d7&xZO1SAsUte~W-8i)_hVI-j+0Evkt zr|EeskSoEeus_2J&COQ(90QU?_0I<^z0dduk7)J~%0wX&C3k`WWXFdnV?xAOY z-V(Cu{_|sr?8W|+6?8~vWmwfFNSh5iDA2wyn;NAPa-ffhezfrLxuLs_p|OZE=mpoQ zml4VX>>zrEo8?4Kbct?}k5gs#{M7|$7N65p_1|qEz^nPG-a)7a&!^@G07%#YEU!+? zlCD~OQd7yB2pHo9E*2!ypz9v7uEZw{U`*U3x!G&LO;d|sEh(kAPKugg|A9a;o|jDN zxMUdej9Q?#Aa0bV3NrLHHC~j9M3GezuU?7X;=C2gW-B5g*{r^wt_Gjc4!=coA9fPB zOMmM}p4y`IyFBY|!!Tlep$%H#{5D?{Xi&bS9FEb^N)ge?3bf-v7~DfKTr~{Q4Ls(h zMIZ{7%q|0Bq93jh6E_|i!pTc%oVYSFo*suwR?{lnU7bPa?NAAfGyUSgaKDIC;>g8* z@mV~Yc($hwTa^l=vRoCr9)lDlS#eM*^y)FF5i1!Llsz$WaA(10WWl6hoF2KD7*F?K zOx05<6+z?=!Kv9Vpfi$8C!S7-j4jEDv9Wq`H9G~U=Pz9vOY|qJs9>Fj`X|QHcv?~n zQe{YtwWK#byAUdOc$WyO7m@Php(5BzVD)F2k`#$rYE}jl10c(>?k7(5;1jWWMB9kv z<{9FilK6j<)!#n5tajK$W0$~o41u})3^J>ea81bd(vS*TNu?TJDgRA!c^Moe^NBX{ z!-WTUdm@3qBoJFT>asUd-Gu~754e-y&rQX-lMhC#yOx0VIc_Fkf3C|(J*|&iC%KYI zVw~Ox=9oE_F1;0+R;zt5@U4}YD{Fgl{UD4ockoqcETT=`|Glwqk8Pn~lm8wHdIDPq zkSAb+LKGAN(7`Gw+~ofU3VQ;28_ceAqBP!QPV^|F`(P(ZW6L--@;*HZsBGtY6SOQJ zNFJgFVN}pmT@Uo_Q&4yA6dF&%DLmUhhyNm)thhFWG7tnnRO|Xm1v`oo+|b(y6rdDTMsY)bXgQ2R+EV?KC4*Y+Z&1c+?R;K3zG+4>-PJagWe-MqPVdt(C%MzS* z?7HXyc&Hvmd!qZ)6$pL088wuGtYUW&Vbu^4JsBSU-V>A#mE*3G@$|Y*RyfmvMpqDu z)16Q`?nPXcCB{y_5$iTP=gkQjNnK& ze}T$p?;U4+Q8W=Oxq4B_HRMC+rQa|)xst9?(2)Q-4De#Gy64aX_H0YJArpex&dz<8 zzthY0)V742Q*^*>82vtI&z?NY_^PXgAFx`4aVVMz!)lG00Q{Ry_<-A04FdYVws$W! zcftue*ePV1TmBX{B=i@H$~?>3DnR4RP5MIUvvg*&p>uV3_4L~1o0*%#w+7ck{hQ(D^6<9D-6GugqG0nY-9I>X z?jc;T`^V=h=<6zVe0oL#}># z?Z8^-=F_)^Zk>AfdeC{*H>O#bFH0P zJ9hKJt#8~?-|JtGCSfLfRyhlNQOYqb{lYMG0;l8p*2~e=g^FS}WcIglhAp2%Ss_z3fDRwL>#7KL(h8}XcLD@w-?(ebad@uM)_?ppMf53})48sJgl K{uRy%(EbBRM*6h? literal 0 HcmV?d00001 diff --git a/submission/pipeline/cdc.py b/submission/pipeline/cdc.py new file mode 100644 index 0000000..c8c2dcb --- /dev/null +++ b/submission/pipeline/cdc.py @@ -0,0 +1,89 @@ +""" +CDC capture layer. + +Simulates WAL-based change capture: every insert/update/delete on the source +produces a CDCRecord with a monotonically increasing sequence number. + +Replay safety: callers can checkpoint the last processed sequence and call +records_since(offset) to replay only unprocessed changes after a restart. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any + +VALID_OPERATIONS = frozenset({"insert", "update", "delete"}) + + +@dataclass +class CDCRecord: + operation: str + table: str + primary_key: str + data: dict[str, Any] + captured_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + sequence: int = 0 + + def __post_init__(self) -> None: + if self.operation not in VALID_OPERATIONS: + raise ValueError( + f"Invalid CDC operation {self.operation!r}. " + f"Must be one of: {sorted(VALID_OPERATIONS)}" + ) + + +class CDCCapture: + """ + In-process CDC log. + + Production analogue: a Debezium/Kafka connector reading Postgres WAL. + Each record carries a sequence number equivalent to a Kafka offset or + Postgres LSN for checkpoint-based replay. + """ + + def __init__(self) -> None: + self._log: list[CDCRecord] = [] + self._seq: int = 0 + + # ── public write API ───────────────────────────────────────────────────── + + def insert(self, table: str, pk: str, data: dict[str, Any]) -> CDCRecord: + return self._record("insert", table, pk, data) + + def update(self, table: str, pk: str, data: dict[str, Any]) -> CDCRecord: + return self._record("update", table, pk, data) + + def delete(self, table: str, pk: str, data: dict[str, Any]) -> CDCRecord: + return self._record("delete", table, pk, data) + + # ── public read / replay API ───────────────────────────────────────────── + + def records_since(self, offset: int = 0) -> list[CDCRecord]: + """Return all records with sequence > offset (checkpoint replay).""" + return [r for r in self._log if r.sequence > offset] + + @property + def latest_sequence(self) -> int: + return self._seq + + @property + def log(self) -> list[CDCRecord]: + return list(self._log) + + # ── internal ───────────────────────────────────────────────────────────── + + def _record( + self, operation: str, table: str, pk: str, data: dict[str, Any] + ) -> CDCRecord: + self._seq += 1 + rec = CDCRecord( + operation=operation, + table=table, + primary_key=pk, + data=data, + sequence=self._seq, + ) + self._log.append(rec) + return rec diff --git a/submission/pipeline/lake.py b/submission/pipeline/lake.py new file mode 100644 index 0000000..7cb15a7 --- /dev/null +++ b/submission/pipeline/lake.py @@ -0,0 +1,69 @@ +""" +Lake layer — append-only storage for every CDC event. + +Every change is written exactly once. The lake is the source of truth for +point-in-time replay and historical reconstruction. + +Production analogue: Parquet/Delta files on S3 or GCS, partitioned by +table_name and captured_at date. No row is ever modified or deleted. +""" + +from __future__ import annotations + +import json +from datetime import datetime + +import duckdb + +from pipeline.cdc import CDCRecord + + +def create_lake_table(conn: duckdb.DuckDBPyConnection) -> None: + conn.execute(""" + CREATE TABLE IF NOT EXISTS lake_cdc_events ( + sequence INTEGER NOT NULL, + operation VARCHAR NOT NULL, + table_name VARCHAR NOT NULL, + primary_key VARCHAR NOT NULL, + data VARCHAR NOT NULL, + captured_at TIMESTAMP NOT NULL + ) + """) + + +def append_to_lake(conn: duckdb.DuckDBPyConnection, records: list[CDCRecord]) -> int: + """ + Append CDC records to the lake. + + Returns the number of records written. + Idempotency note: in production, deduplicate by sequence before appending. + """ + if not records: + return 0 + + rows = [ + ( + r.sequence, + r.operation, + r.table, + r.primary_key, + _serialize(r.data), + r.captured_at, + ) + for r in records + ] + conn.executemany( + "INSERT INTO lake_cdc_events VALUES (?, ?, ?, ?, ?, ?)", + rows, + ) + return len(rows) + + +def _serialize(data: dict) -> str: + return json.dumps(data, default=_json_default) + + +def _json_default(obj: object) -> str: + if isinstance(obj, datetime): + return obj.isoformat() + raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable") diff --git a/submission/pipeline/warehouse.py b/submission/pipeline/warehouse.py new file mode 100644 index 0000000..ab32401 --- /dev/null +++ b/submission/pipeline/warehouse.py @@ -0,0 +1,124 @@ +""" +Warehouse layer — current-state snapshot built from CDC events. + +Each warehouse table mirrors the source table with two extra columns: + _cdc_seq : sequence of the last CDC event that touched this row + _deleted : soft-delete flag set when a DELETE event is received + +Production analogue: BigQuery/Snowflake tables refreshed by a streaming +merge job keyed on primary key. SCD2 history tables would sit alongside +these current-state tables for time-travel queries. +""" + +from __future__ import annotations + +import duckdb + +from pipeline.cdc import CDCRecord + +# Source table → warehouse table +_TABLE_MAP: dict[str, str] = { + "customers": "wh_customers", + "wallets": "wh_wallets", + "transactions": "wh_transactions", +} + +# Source table → primary key column name +_PK_MAP: dict[str, str] = { + "customers": "customer_id", + "wallets": "wallet_id", + "transactions": "transaction_id", +} + + +def create_warehouse_tables(conn: duckdb.DuckDBPyConnection) -> None: + conn.execute(""" + CREATE TABLE IF NOT EXISTS wh_customers ( + customer_id VARCHAR PRIMARY KEY, + name VARCHAR, + email VARCHAR, + status VARCHAR, + created_at TIMESTAMP, + updated_at TIMESTAMP, + _cdc_seq INTEGER NOT NULL, + _deleted BOOLEAN NOT NULL DEFAULT false + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS wh_wallets ( + wallet_id VARCHAR PRIMARY KEY, + customer_id VARCHAR, + balance DECIMAL(18, 2), + currency VARCHAR, + status VARCHAR, + created_at TIMESTAMP, + updated_at TIMESTAMP, + _cdc_seq INTEGER NOT NULL, + _deleted BOOLEAN NOT NULL DEFAULT false + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS wh_transactions ( + transaction_id VARCHAR PRIMARY KEY, + wallet_id VARCHAR, + amount DECIMAL(18, 2), + direction VARCHAR, + status VARCHAR, + reference VARCHAR, + created_at TIMESTAMP, + settled_at TIMESTAMP, + _cdc_seq INTEGER NOT NULL, + _deleted BOOLEAN NOT NULL DEFAULT false + ) + """) + + +def apply_cdc_records( + conn: duckdb.DuckDBPyConnection, records: list[CDCRecord] +) -> None: + """ + Apply CDC records to the warehouse current-state tables in sequence order. + + - insert / update → upsert (insert new row or overwrite existing) + - delete → set _deleted = true + """ + for record in sorted(records, key=lambda r: r.sequence): + wh_table = _TABLE_MAP.get(record.table) + pk_col = _PK_MAP.get(record.table) + if not wh_table or not pk_col: + continue + + pk_val = record.primary_key + + if record.operation == "delete": + conn.execute( + f"UPDATE {wh_table} SET _deleted = true, _cdc_seq = ?" + f" WHERE {pk_col} = ?", + [record.sequence, pk_val], + ) + continue + + data = {**record.data, "_cdc_seq": record.sequence, "_deleted": False} + cols = list(data.keys()) + vals = list(data.values()) + placeholders = ", ".join(["?"] * len(vals)) + + existing = conn.execute( + f"SELECT COUNT(*) FROM {wh_table} WHERE {pk_col} = ?", + [pk_val], + ).fetchone()[0] + + if existing: + set_clause = ", ".join([f"{c} = ?" for c in cols]) + conn.execute( + f"UPDATE {wh_table} SET {set_clause} WHERE {pk_col} = ?", + vals + [pk_val], + ) + else: + col_list = ", ".join(cols) + conn.execute( + f"INSERT INTO {wh_table} ({col_list}) VALUES ({placeholders})", + vals, + ) diff --git a/submission/requirements.txt b/submission/requirements.txt new file mode 100644 index 0000000..ab2841c --- /dev/null +++ b/submission/requirements.txt @@ -0,0 +1,2 @@ +duckdb>=0.10.0 +pytest>=7.4 diff --git a/submission/scripts/check_schema_contracts.py b/submission/scripts/check_schema_contracts.py new file mode 100644 index 0000000..1698b21 --- /dev/null +++ b/submission/scripts/check_schema_contracts.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +""" +check_schema_contracts.py + +Validates that the source tables expose all columns defined in the schema +contract. A missing column means downstream CDC logic or warehouse models +will break — this script fails the build before that happens. + +Exit 0 — all contracts pass. +Exit 1 — one or more violations found. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import duckdb + +from source.models import SCHEMA_CONTRACT, create_source_tables + + +def check_contracts(conn: duckdb.DuckDBPyConnection) -> list[str]: + """Return a list of violation messages. Empty list = all passed.""" + violations: list[str] = [] + + for table, expected_cols in SCHEMA_CONTRACT.items(): + try: + rows = conn.execute(f"DESCRIBE {table}").fetchall() + except Exception as exc: + violations.append(f"{table}: could not describe table — {exc}") + continue + + actual_cols = {row[0] for row in rows} + + for col in expected_cols: + if col not in actual_cols: + violations.append(f"{table}.{col}: column missing from source table") + + return violations + + +def main() -> int: + conn = duckdb.connect(":memory:") + create_source_tables(conn) + + violations = check_contracts(conn) + + if violations: + print("Schema contract violations:") + for v in violations: + print(f" ✗ {v}") + return 1 + + print(f"All schema contracts passed ({len(SCHEMA_CONTRACT)} tables checked).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/submission/scripts/run_data_quality_checks.py b/submission/scripts/run_data_quality_checks.py new file mode 100644 index 0000000..3bcdcbd --- /dev/null +++ b/submission/scripts/run_data_quality_checks.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +""" +run_data_quality_checks.py + +Runs system and business data quality validations against the warehouse. +Seeds an in-memory database with representative data, applies CDC events, +then asserts correctness invariants. + +System checks : PK uniqueness, not-null, referential integrity +Business checks : non-negative balances, positive amounts, valid status enums + +Exit 0 — all checks pass. +Exit 1 — one or more failures found. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from datetime import datetime, timezone + +import duckdb + +from pipeline.cdc import CDCCapture +from pipeline.lake import append_to_lake, create_lake_table +from pipeline.warehouse import apply_cdc_records, create_warehouse_tables + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def seed(conn: duckdb.DuckDBPyConnection, capture: CDCCapture) -> None: + """Populate lake + warehouse with representative test data.""" + ts = _now() + + capture.insert( + "customers", + "c1", + { + "customer_id": "c1", + "name": "Alice", + "email": "alice@example.com", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + capture.insert( + "customers", + "c2", + { + "customer_id": "c2", + "name": "Bob", + "email": "bob@example.com", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + + capture.insert( + "wallets", + "w1", + { + "wallet_id": "w1", + "customer_id": "c1", + "balance": 100.00, + "currency": "USD", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + capture.insert( + "wallets", + "w2", + { + "wallet_id": "w2", + "customer_id": "c2", + "balance": 50.00, + "currency": "USD", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + + capture.insert( + "transactions", + "t1", + { + "transaction_id": "t1", + "wallet_id": "w1", + "amount": 25.00, + "direction": "credit", + "status": "settled", + "reference": "ref-001", + "created_at": ts, + "settled_at": ts, + }, + ) + capture.insert( + "transactions", + "t2", + { + "transaction_id": "t2", + "wallet_id": "w2", + "amount": 10.00, + "direction": "debit", + "status": "settled", + "reference": "ref-002", + "created_at": ts, + "settled_at": ts, + }, + ) + + create_lake_table(conn) + create_warehouse_tables(conn) + records = capture.records_since(0) + append_to_lake(conn, records) + apply_cdc_records(conn, records) + + +def run_checks(conn: duckdb.DuckDBPyConnection) -> list[str]: + failures: list[str] = [] + + # ── system checks ───────────────────────────────────────────────────────── + + pk_map = { + "wh_customers": "customer_id", + "wh_wallets": "wallet_id", + "wh_transactions": "transaction_id", + } + for table, pk in pk_map.items(): + total = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] + distinct = conn.execute(f"SELECT COUNT(DISTINCT {pk}) FROM {table}").fetchone()[ + 0 + ] + if total != distinct: + failures.append( + f"{table}: PK not unique — {total} rows, {distinct} distinct {pk}" + ) + + not_null_checks = [ + ("wh_customers", "name"), + ("wh_customers", "email"), + ("wh_wallets", "customer_id"), + ("wh_wallets", "currency"), + ("wh_transactions", "wallet_id"), + ("wh_transactions", "amount"), + ("wh_transactions", "direction"), + ] + for table, col in not_null_checks: + nulls = conn.execute( + f"SELECT COUNT(*) FROM {table} WHERE {col} IS NULL" + ).fetchone()[0] + if nulls: + failures.append(f"{table}.{col}: {nulls} NULL value(s) found") + + # ── business checks ─────────────────────────────────────────────────────── + + neg_bal = conn.execute( + "SELECT COUNT(*) FROM wh_wallets WHERE balance < 0" + ).fetchone()[0] + if neg_bal: + failures.append(f"wh_wallets: {neg_bal} row(s) with negative balance") + + non_pos = conn.execute( + "SELECT COUNT(*) FROM wh_transactions WHERE amount <= 0" + ).fetchone()[0] + if non_pos: + failures.append(f"wh_transactions: {non_pos} row(s) with non-positive amount") + + bad_dir = conn.execute( + "SELECT COUNT(*) FROM wh_transactions" + " WHERE direction NOT IN ('credit', 'debit')" + ).fetchone()[0] + if bad_dir: + failures.append(f"wh_transactions: {bad_dir} row(s) with invalid direction") + + # ── lake completeness ───────────────────────────────────────────────────── + + lake_count = conn.execute("SELECT COUNT(*) FROM lake_cdc_events").fetchone()[0] + if lake_count == 0: + failures.append("lake_cdc_events: no records found — lake appears empty") + + return failures + + +def main() -> int: + conn = duckdb.connect(":memory:") + capture = CDCCapture() + seed(conn, capture) + + failures = run_checks(conn) + + if failures: + print("Data quality failures:") + for f in failures: + print(f" ✗ {f}") + return 1 + + print( + f"All data quality checks passed ({len(run_checks.__code__.co_consts)} rules checked)." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/submission/scripts/validate_catalog.py b/submission/scripts/validate_catalog.py new file mode 100644 index 0000000..6a02026 --- /dev/null +++ b/submission/scripts/validate_catalog.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +""" +validate_catalog.py + +Verifies that catalog/catalog.json is present and contains a complete entry +for every required lake and warehouse dataset. + +Required fields per entry: name, layer, description, owner, schema, update_cadence + +Exit 0 — catalog is valid. +Exit 1 — missing file, missing datasets, or missing required fields. +""" + +import json +import os +import sys + +CATALOG_PATH = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "catalog", + "catalog.json", +) + +REQUIRED_DATASETS = [ + "lake_cdc_events", + "wh_customers", + "wh_wallets", + "wh_transactions", +] + +REQUIRED_FIELDS = ["name", "layer", "description", "owner", "schema", "update_cadence"] + + +def validate() -> list[str]: + violations: list[str] = [] + + if not os.path.exists(CATALOG_PATH): + violations.append(f"catalog.json not found at {CATALOG_PATH}") + return violations + + with open(CATALOG_PATH) as f: + try: + catalog = json.load(f) + except json.JSONDecodeError as exc: + violations.append(f"catalog.json is not valid JSON: {exc}") + return violations + + datasets = {d["name"]: d for d in catalog.get("datasets", []) if "name" in d} + + for name in REQUIRED_DATASETS: + if name not in datasets: + violations.append(f"Missing dataset entry: {name}") + continue + + entry = datasets[name] + for field in REQUIRED_FIELDS: + if field not in entry or not entry[field]: + violations.append(f"{name}: missing or empty required field '{field}'") + + return violations + + +def main() -> int: + violations = validate() + + if violations: + print("Catalog validation failures:") + for v in violations: + print(f" ✗ {v}") + return 1 + + print(f"Catalog validation passed ({len(REQUIRED_DATASETS)} datasets verified).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/submission/source/__init__.py b/submission/source/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/submission/source/__pycache__/__init__.cpython-314.pyc b/submission/source/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b3cc59dc4a6f82d0cf027e07e2ab3c4ed573993c GIT binary patch literal 184 zcmdPq>P{wCAAftgHh(Vb_lhJP_LlF~@{~08CD^x$UIJKx) zza%v|qb#*3vm{?XyELa%zbLgJzZgQMmXsFgrzDmn>LwN!XQt=nrskCt>lc?M@0%4+Er1y;=UYtT4>4+;2kgm+;cuvnvBE7tz=y`Zo z;9Y>Xx}a3^JpT!LfHss-q!&k!UK&B}F<#J>w~=*f~0oSGgpE$;Xp z1Jxx?eQyneS%*7>;f#@qL{Ys*%#Dr+2BEAKJnlNH#M-f3u6#w=-SrRf6mT=fCe5}VcpP7s3yhmQV0hIbuFWtdj8n(r(GDA)_($Re zZM5zbv=8z#qXfC8_`Ei6h*P!injb>iVoFa-arHp5B^SlAb%74~ON!z^$G z5|0=Kc=GTZT!U#B-9-;_)1M<`1Mdv_NLg2OWf@kYv6xG!&Rfsv`DzY)T=+w*h6T&_ zys$(G52)AFa$(61tPOi59GZdm%#FpJItUQyD@_SY5Cv_MGjX1`!lmjR2A-~RVy$lx z>Ttih5x7B>l1-mQGuYj%+9o$Inv6MXUKBhmSZO=(?R!-wgRI*2ZQ`=Y&7H8{Fig+$ z0d`^-bO1KG01v}3?V{&MQz}oz^knOo(di#Yr}xCi_btgg02w6l5`-86D|iYAE)W8* z#RN&Q8V_B$6mv&kCXsukM7|@T7XaVlqefji9Y=qNxw+y(aBn+&9RuhPM{r*_!;Xjf zI>Bqs7Q{sax>%1fpESROv#B^9b-ht*HE^qTW3GW`Z{p@c3pYNVU1}|5F5Y-5F0_*P zG8`M@yEVOjtES^ceRjU4-^aHb_b0lfJ+n=c%~ENxOY_d$T(>AB1;=O>FUDxjq9*)4GqBxpEk{X6GAA zt=jyeMC`?X2{s`{qx*T7aC$^dQsyw+os{u@OxT{44!G+tlgPw;i<`~sq`;X#|R>JPGU{u z(4LK~fKuXFJ5sIJuY+Ti`j3eBKSf6Hnvs#s@IB#?S$e%Y+Z9g$crV=9Ac-S# zJk{y(TU`63+r;#;7>OgGQ!0Dx*ba?~Mr^6~Oozu62*-vy`ts_-WwsqId}@;wAbKx* zR}6ubf4f~a&4|q6cxy7xM8#Z?tB~xXd_xJ;y2COKC96aPeUd=qLv%-bS6YITzA7_i z`UweTDxV@4<)kK5=RWd~SOF)RLPdKhtMqx=4`g^U+SiDRkr5RmdK{Kyb-fJ4F8Zxd z{$lRaxjm!~mH$L)v9fmx6)O>|U$FXtbtYvE43(dufntRYf_7J%QUr!o5*z9sxU1&m+;4U5T7T z-@vLciPmI5DLfU^^LK`o+&+To@yW;5lS|*^zrOsH_w(^{PX%xP9EeEBv{HB^ld8jl zD86A)*43~Sg=DC90wz>DWlHM{E9o7kr^J8u#pyUKC&`zd20lF_rZ-{I%Hd#la-nC1 zVkx@CVI{qEQZcNg0WEEWY9|)85TJMR}G3U=GeAWpJ;IipPFBe(%@9z&2pA2 literal 0 HcmV?d00001 diff --git a/submission/source/models.py b/submission/source/models.py new file mode 100644 index 0000000..91c9d19 --- /dev/null +++ b/submission/source/models.py @@ -0,0 +1,84 @@ +""" +Source schema for a payments/wallet system. + +Domain: customers own wallets; wallets have transactions. + +Strong entities : customers, wallets +Weak entities : transactions (lifecycle tied to wallet) + +Invariants: +- wallet.balance >= 0 +- transaction.amount > 0 +- status fields restricted to known enum values +- settled_at must be >= created_at when present +""" + +import duckdb + +# Expected columns per table — used by schema-contract checks. +SCHEMA_CONTRACT: dict[str, list[str]] = { + "customers": ["customer_id", "name", "email", "status", "created_at", "updated_at"], + "wallets": [ + "wallet_id", + "customer_id", + "balance", + "currency", + "status", + "created_at", + "updated_at", + ], + "transactions": [ + "transaction_id", + "wallet_id", + "amount", + "direction", + "status", + "reference", + "created_at", + "settled_at", + ], +} + + +def create_source_tables(conn: duckdb.DuckDBPyConnection) -> None: + """Create source tables with constraints in the given DuckDB connection.""" + conn.execute(""" + CREATE TABLE IF NOT EXISTS customers ( + customer_id VARCHAR PRIMARY KEY, + name VARCHAR NOT NULL, + email VARCHAR NOT NULL, + status VARCHAR NOT NULL + CHECK (status IN ('active', 'suspended', 'closed')), + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS wallets ( + wallet_id VARCHAR PRIMARY KEY, + customer_id VARCHAR NOT NULL REFERENCES customers(customer_id), + balance DECIMAL(18, 2) NOT NULL DEFAULT 0.00 + CHECK (balance >= 0), + currency VARCHAR NOT NULL, + status VARCHAR NOT NULL + CHECK (status IN ('active', 'frozen', 'closed')), + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS transactions ( + transaction_id VARCHAR PRIMARY KEY, + wallet_id VARCHAR NOT NULL REFERENCES wallets(wallet_id), + amount DECIMAL(18, 2) NOT NULL CHECK (amount > 0), + direction VARCHAR NOT NULL + CHECK (direction IN ('credit', 'debit')), + status VARCHAR NOT NULL + CHECK (status IN ('pending', 'settled', 'failed', 'reversed')), + reference VARCHAR, + created_at TIMESTAMP NOT NULL, + settled_at TIMESTAMP + ) + """) diff --git a/submission/tests/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc b/submission/tests/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..28b2895286e98894c1ca2c817db9bea8f4bc4bd0 GIT binary patch literal 3910 zcmbUkO>Y~=b#}QVm*kQ(sV~Y_q_rf=QDT#^t3#SNg=EVOm4l(K8>J0g6RX{kxiGm) z%+6X7Pi&DQMG_!7_@Dp{+NeX&0EUH}7lS zd-LY)tGR3%!8iKHzdPS#5VH89_r;>g-n4{J2N{T>o5+xsBr1zBR#gId0#)NQF~N7_ zrDQcplU0?fVl6GDXi8wFX&T1ZQl_d=P3$R4*=mmFs(G3h>-f@%YJnE2MOqZ=#L_@@ zkPc2DBRPT$bp$m|(4pU@m(TT-+aFJ(xoBae05{EXdk{KuEKLSzw5MoF$I@hhCifK0 zs3A?E*?jxW>vfZ0TW@Vp>{ETs`G5ul`}&$kbXv!H%iL&Um-_mZ#VdNtY2k+BVx8mq zNGJ^dEfS2d{1I&z`kO(I4i)(nXMLf?HcuJ4f8f;1&d&y5{nu&R~sCu z07#9Enq^xxf-R5OJ{yVf)4E;)~ONMjN~VJ37Qf=LJMWClLF z^zo%XtAEix*B)lh+*i&Bio<>Mx}Z7QM~9t+`;9;vKoCF)JFsrstE{*PDr^**$26^ViD>M82yi18k(|4VbVdz zS=*3$7Hy#$@gA2I)Pqi-8_7eVsl%Wv2>c?q-})t7)4J}s=bO0ckqtdOw!ZGrx*lE& z{Z*acHyHXiNEg@5FSpe*)SQ5gKn~QLvghvHYp@aCBED`#8=xl}Kk(Xer z2+glR*FjGbD3$*#_i=8s{ZO6kEbJ(yt-`i)YDXE`G`E$~j&f>iVOtsRX0%TAh&Y%Q z+>VZezRwAF!%?o~O7^diER`N3={k4*8DoOhp#qUbVFoA zD@<^D!@E@$x)2`vf^lf7WQ0L2#5jPS>(ol_*kzSFF%!*t|%taIv z=!Mi8P-{S)1?t@YNS!xM7zJote$f~(292SoI3$mipErhq!?4hyXpHQOSLfXkH7<&-fET+T#XW?sOhpPu#|aH4Nf>c0L>=dKP2`dTCK>%fK9i5J|ehe?l9^wVBBZHPzR z*I(ueoYSM5Ia^M%R9_~OtqUyETh<(#DXs~XO?;u@STNwG=``8}ldtCS2WGR?z-7y8 zvV;#0ci^)G*X#~v>99JnYbIsspv6lhF_*22EOyyjZRc0L)u%Wlqmwdu{Q^sc3$8$N z)ohrqg>QxK`0YGXt$+~hS{p3(?)62$zZmT;%`@$wd6-Nuu-pL%=a(>>Uf|M*rGN zcvg5GWeLZHa*48RRMgdc2lx>lW`sX_P+ssfEiw}@^chGJ)t}E()69$>Jv#{>V~7-T z6`+&Q{4YdCd6Glv>?cE=%U>&*kFNdxT2~$Ds@m7u=+<0UD{j8s)dublz1-CXH|t%k zv^6QZ-E1 z{dM7CZl?40j#Av5+g48Vx8B}Xrn{N^pWeE!l*DT=dy*weknTnz`tYH%24%2!+DrNYc z5N{mI_CdTP3geF#Ijq><=w}NNn4ah;6*l=h=ngyyp)fO-ii zg9iQuxemg3N#VDl+l@(*^bfS~EkYlnN9gRoQ1(ld{WqHW3cdLiTKXp{f1iMIe^*7a u`qB6w#ve+BCoj|OFTTGFjBSE5fnc!BH#QJ3;D&&6Lx!yCFq(L-VZwMkoI8ft z-L%f`Rxuk=SZS+uD=N_ssUSsFD&@yk%8yi9ZTn+Yjj@fkrkSJk zd+wR>1sithR#gtex##=qzUO_P_dVx&OI>YPgS$QP^Yo^rnl?ZY#|2A)hkxqVv~kVU z^4dAg)O+>3k3anDKQLHF6W$_+cAN0eOS z zS-ncjG<22<<(Dq1UI_KVW%ZVskyToLxmg2$g;|UII?B(lM0?xR5r}Z#j`?Y(@n(Ie zPo$gmXrZC3hc)IBq%GyGu6?>zpP<&VHyh+wmZRSl>R9Aztut35ZxtU?;}f)Q;4Qsy zT9-YnMk{O79`fs-q%~#T)}p0#Wi4$m8y~e z5Ppt_Tb^VT6?58*R$8eQxmue{y+w;{8JWX74xh2?ydC}3_>ZFl$$T<1d_Fp4<&!Cd zqXWZ3xr~*!vX*T}li3s!7_$cQ6yFvO9~r)EF?(NhFrBfYIcC{bcEH*iO=dDtW?dRd zGbM4{!V)^QNBko`6MWy)9FDnBYtf?G7Ok~Z$Q#ipNSIt{p+=~ z9@9_7jG}=TmMzvCIeg~u$rABW`>g~EW19)S3JrhXq3@pxVIhwI|grD z`*o=HT4pY^ZYH#D^3vz^%dU_9aCCCX2jBhLuZ0_tj^A`>-8Xg!r6*VJ4g5>wa5z*( zB{WgKWF~L$@l;~^T#9j*)J^N~ZC#6IO0LZZn({O1K@6V6X3WQEFh#$eXEB|Tj1^=h zCQ0v{zxYj*u^Aq{cwIlG|CakdPCekyoB4k@^3n2Jy?^s(BAwNuD4!Uf+y&tCi7^C- z{QD#|hyvuB+BVJMnho0PpolhY#CL3^rsc3b;`&~cInOuD%NSW4Y3c;y`dhx2S8G}b zWIE!@VGEJer%)R6ELJ;QgAw)NefrN^dCZ^2QPfj5rEV_0T$1{XycYbocf%Xke%T!R z$1Q)m<@Vb@ZJufFnvT3O)!a4t>SrteeaFA9yLNKo@WkMUy*DoX@FbF_BCiyTSN{8M zwfmfwu)Io90wn!J5Zv|EIU)DiBsVXAQB4(n zH@GF85XSEWE}6d&es`Po4y3jne)oB5XJ^cgh2JgubEDgfzG0gJ?c{Zmx04Y~QfT+7SPg5W zh(WCst3)FrKYS=RN)j@WWW^;3A$${J&_nk5VhyqrZ;hlgfC_f8DlOpk4 zLydH(kzFDphPTHiei4P^jJ&v-kH7R95`?W1r5U!N+!p+|{|Vl>_IafC`tGZ{3&xr2 z-B-KunCQN_bvn|D&|IW-4!&`wu%+i`na|D@Vic+P%^GK}wac`-ZtNbK+AT6>s8v}; z1dx4mt4u>ZCx(oI)3MA)V9Gcnqzr~F9Z6=+RJ-AWsY)5+ibsb`e>cPdc~pTn`bmy* z_lDjt*{mbY<}+nee@ccb*=V09V{cwHVmH2dF+lD6fa`7V8>v# z4C|v6!)Y7rFrS3>Mhtcbv==BKjLfeggl}X)=nPnC-xk9o5^b- zk6rM zIJIFX@)dn?2Puj>KI-I;`8tB+-v7b5w+Vn7*9c0w-1GcDbN+xd;k>{y0Bud&*P)xW zXB#?vorWl>s@Gm%8oNevBhHy6AnKDG(dkvwDxj|5b~4)=X1kD6auUJq+9D2@9YQF|O<(>T)HH=wlz!VnJa&Q1JP76_U5B zvZzyy>|%tu)v}#-5Gk7pvOA!CeVMenkhBwy2_tdQ8f8aNo-^3rATX}okE|#d(aBX) zk?5RJ`b`_r2}mG{&ALIEeUHk^*xj<2Ohy1X_gAi;e0z3ftYB=vwy&@<_EG0_WIG~r zk?nKvmESa#kO_C)*ga!AD#{!YK;lQ8G7Y7j7?qyaHVV38qqIdd+NxZHZRhLoco7Dh znY6EvMeI(=B1VavMDO?*DZdWIdvluQ?$F;NEs^v;J@J`4ZU|J8!Eg;Use0L{llZ zJSE*CK0 zOXG(bB ztz&CiqiH+Jt^3*Y(LpvmqIHz%~ zA?}XO>F10~O>y6zkASi}oIL~{Bsc@jhC6S>Y=+~0;OxlRr4>BrNwBj(d~Q#k`_?gD(37B8G<_!XLei=0nVupz{d=HUMG@qSc4q>Hs?RPjOg{! zQDp006G?G#ho9XF&5^dDFFlWe-CdA#R-q~R)##(KqX z#^{~cDPuCAq}}x{Q80RM^vLokq{L>8-pTLW%HJ9+Y(4bZiqC?dT`H_A^(Si}fMOWT zhX)Eq&tDvvGtgSk4@v_lzDFk~+FNF-p#K z>4hvLHptw}PN5)M2p6A0Aq5xjBS5@Q;NpGmAW(?AAbdWEeHa!EqTJ3|eyap0eLjwm zLC7vbK0mrB9UXcg_Mbo4JG8N2Sr^xp+l7H2%_qVC?;P7x$(jd*_VOZ`#;fh_s35tP-A~(z28q zK#oh1&bBbnA*aZvF=4SD)Mg6-CVJmCT?Z5Ps&?8U?VoaMRr@cvv8r8+OJW(6l{;k* z-zjGsRC!juYha(yFSW=0duX3XgtW+g66E-(`fQ7Wk9-P+()v_5P^hGRU`m<$h&?4S zM1`#!7Sv6?Z5^pc!9h$ygNa~ zjPgiYr(u=^lF&c4b9p3tlKKYju2?F;%MvVyCP*2vNg6^Xg|(XHA9hc+&PJLG#*SMT zxi=NrF=v#1)5eZMq*+8~mGBIemZj7Ha{eGRpQ33I0}@KYQLZE$)j#o*a&T1tG*7z0 zQ4bp}P`%iX*I=oFjkb*Lz4QcA)f0UEw3oSyEB07Nk8q_3^oPA9M@n}=NSEYp(dT5+ z{?O-%{9I({@bK1B4<)OKTi+9b&aV#y4m+-TVQ>*+6`G6Nm*-gB-7Lo_G#9s?=NR33 z7#sio;VI7L+&(s`LElP_xo60xt@|{ zaQ6iIxLebh2D= zyXX|-CP1)QA!3Ctx}r6Tqf~r(66HEsR5VwiXTk!>UPC>)_TLTrW7{I_kNj+z%#XD~ zE9FBYwNW`ZX`v_)Y{g9`F($#O@g_ zg?atX7%gNzz5K!cf^lSW3HPoZoHLGy-;8l&V!w>Z0NG1RCGHwWl$cCLLlc)}427K7 zJ-0eVxt@WJXe!osbS4y1WsA z@9#%e7L0?1md@F=J3qcKyY?W()*hUSV8fJt)5byCFch10gEPFGETjgI^}j~aSxiSU z^&BKhKKQA@AMPMfh`S(s0^<*b0{bmJi)C^BMH+glZG3x4Ut%tXoY^Sae+>1;9!O8HTXcGyRfEvDgv8n={Ie3)0HQR&AP!EUQQNL z1ITje#^Hs}h#xadXCCo>g8Q(bVRsPD+xeDw)mu5Y3jM(|%A&Vu_sbbBQ3kfG~ zFTj@QZk$KLIPP^wxD+H3 zZglDV4@JV4X(r3~Oz2C{E~4EoaoeryTN1{)ki%7J+PvH;rpc!#;i}{izL?7w>p{JF zmdx780s1!VcO~F04gsre&ZA+`H=Dq8_}h1Q@v?$(pb*_LyQ=-;J+rHDQgooO>cCV4 z2SKIZv~gep*Q+Qt>jr0dIax>zAj_qpXUvP=#dH*Ge1c=+6Z$7diK0(D4jX@aY(#|Z zZTNG|5Q++4acsmfM6qrphwIEZJxZnsSd4|3mCU@DPlHc;0_Ae^xHm2qsxHUH<*k#( zOYJ9k``0jh{`zTeDhEA<*4?v_Jq6?VPhaNVROI-aQTk0A#|seBof7H=lg1IM=P#Lh%d6Y ziSI|y+%{%i!QVcyiXoX)43^TwQ8HP~#V6iXqB3hj3jW5&a5`HIrqYamoL!v|Uq>gz zr{7D=o>yP*HG7UGj^Yo4yw-E(wPL+YI@a5B^5|>D;9&Y4{Dl@v{8bHpd##Q7+C|3+Tqzb=dVk(7qq}Pc8IcTJ10Emt4Pc^~&|{ zUj6QUqw4+R?;U^t^n0goy!4afKRW)C(?2?W>%{c(7pIMF_l?^7#?psDqro>`^F>Wy zg|DD3`Jzr+_J%%Qk9#Y=>i0K%Z^MlXGy29aefT3V5B-`i`hY?>KrG9ozZO!;rQjEJ zy8gQUMYUgV{;CFF13qY=;_Kd9H>am6VE=<5lp*ZF`4zpBPn=LdD3 zh41S@rG-_u+NSmPFMV|Xj~eI@tr!&lYAG&;Jg6>f<-QRv8ap2b{6628=%oE&2%!fQ pdPH~B^igMQ6#S}&7%PqNp!B-F!bDH#kMHY{&O7@8HA~jG{{gLh-@pI> literal 0 HcmV?d00001 diff --git a/submission/tests/__pycache__/test_cdc.cpython-314-pytest-9.0.2.pyc b/submission/tests/__pycache__/test_cdc.cpython-314-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a287f0ae237daa394be983dd772fcc558b8a93f6 GIT binary patch literal 20869 zcmeHPTWlQHd7jywy$B$pH^k-E^fEMI6Dk=2R3a%)E0^>RdQyxd*R zEG?6Q3KmIPDsf;H2oM@55C%wVxi9sj(U&GK?d$R)(p{uL+6E~4)<`Xk`XT7|pEGl2 zW;x`NvYhxK!{NW@@}Dzj&N=hnXP#+kj!QU>eD9y~C-z8EjxzQUuoCx{keHV=sUlsL zH2GDz5@5fLWgk~HOE zNeewJ71}DzTKHk9(xOEGTeT=)n-&9X*W!R3nhLl_YXaP>H3L4RwE*tZS^*!{+5jKX z+5sQcrKp=`Up#)c^sa80 zClmRisT-BlOnD+((Nhz8L9YOnH|as`yV*j1B2g;qMz)eK6%&U3ZFGWEPu$d}XY^uD zPfV4HrAn!o&*dw#J&D{^J@-zzlrL6#-_Dx)1e%l!SyW8SlnePBE|V~`SMa%qEBPtC79}`WD(WbWW8`P~C~7gJH66pB$jP`p`lo}ICW!xghXCfKv~)Qz@?A-q zK-qrjkL0VeCXb_rA1Ruiz<*JjTG-GrA;qL@P*+L07QRw4rm_{2vIlAxQ*U5O%v43s zUBw*bE2Y#sGliKHZlh#!i1y_a#+~iWnr42oIHeaWX3Cs-dn%87sfe~%0R}U9Pd@cx_Z=N^i^8Axgi&{B zc*wK>{)3v53Jh5oWjQTh5O?M@DF?LB^wD&1WK5E#UwqoCnEYqj3FG4aFGQUdcB)&_ z+SAIQtaV&EI2ahR`qsi)WSkKN!n@F9V>`dy;to!raI&-RN-Ykio$8j-vOVwi2qzw{ zYe#cjcevYqL)Ic%)OAPx_yPMmTC5>%(8f&#d&EphQaPl>H8mYd$)3{iIBi&VxO(1e zmpQ)}b=dCN{>2m7bV@Db; zaql12@prtAk2&jSr?g;?tF#=^l<8N}3Zt?MHVvDct)pREQQ0V{>_e9-E-KsXq9Oj2 zEMn6Rt)(HZV&kR)o|%hjty)_;R!7O=yP#xoyJ!2?qGS)obNCHra`SK3m_cgPIK_P5bIYxO$60Y~274lxi_C!dm?K z@+`?unXFN3&*ZX&LI$Fjkt6Ks(oA83p=Krv zrMI(%44d{?hNMmEq1JMSO)mtrmkgt1)b?b!;Fr6cV@(m*6+?7b)y|~XNFA@$P&)MWwQY`SA0`t za)u58)5I*0xSY9`uUySoa(_n5u<*@I2?mzc0~?BUt||u>o)bBdpletdE5NXF>kjf7^(JuK3EQftvb*?H;Rn^CN_KqvPM%V(&oj}btQg{^N{cff3l+;-IzI^}OKE|Z) zxQE*ZwBXlN`i3^rkPkW1cc>xm*3vgLkOwY(M{Vg_i-KpVMh9qsK?1!&G8`jp2Jw62 zaRU1ZK(Ud|8wUWvEc&;h{kAc{L-=tGhr~y16yq@UoFs6Bz%vAT2=o%@Baj04IAV~R z$j5bzQpX7l6Br>ts+So7uxKJDh&^tC$n|qGV_!RCRZ915e?Ps}*^zIODx$dy^ZpFc5I=4$Rv>R-za|;>pg3zMvc9OJ>&Mh?U z3%(k(Z8dP|+|HH!CCIIGZl@hR1$Bpy&fN&#dFo^1jc~Vq>iuL}rac1fBv&Hao=Ac!f$y<=QFnG>#JT zAb`Kv*w3gWAFHp}XjM5<;~4rnO#S^9z@d~%=q>- z`Q%jf@N=v2=YL;fB#BsKcht_R(z66P3t(C8*-)%=Rq3g!ojiNTm0lx6f#pu1W*gG{ z+xsKUvO{nikY*R=Hl#VYrTN8q#QYy(l?h}`9)V2JLgQag(j3ylE~4m9U>wpMX^6Wu zX^v_!AAxM=1jf%C8n*>9m_vO?^R{NRovmZ@AJ)-h3<_yq={jn02-YnQqcF%cUJG-# zS|gU=kpCyq%0vFophsX_x8VwZiwT6l4#2A#H>{zTaymY27!4jFOKph$1Mr(le`)w4YIlC6B30T)(aYfa3B6$Fd$(ZZ%*@GZTV4vk=u zr*powGwBGc{R?|TOh6RaFQub){v~;Gj#>2^F`3p&R|f-H+ohM%k-?zwo|2X1>nSdf zAmiMVzqlZD4CV!X@b@9a5qq{RFL0xAiFTG3xDkHH_Eu;yE&dg(hY*!^c^{%3?gRR6 z!}SpQVJhrlEYxeQ$>|7-C)A?E;k}3EP;~(}iIs`68b-h$AEebKlsRVePc-7~(D-c!(?) z4}3X&wuLKxWA1h84z=5(C3trO@DFiBN?9F_csfir)oY^9W86E{L}3JEX1q73O`5=) z1dai?9UsS;Vb5K1g4z4H!-xA&1_nI2L4VcuDaJ(X`7q%}(qZ;9^0`W3Hj^)skHmxp z(8yv0FXP|zN;R>j4<5Vu*sa%=_AVK#u_N;@e72JkwfTcnH&0cSqd$54Lui>t7olYy z#qUSge|jB1OV^RzP@Hp3Il6Fa>AJg!rxzTY72sI8bwaO0M%iMAC!jdnm>-G!! z*{mCb;el)*SV-u^{y{%l6Vr~pS{1Q7VXnO3YlieOTsl|sXG4-rzU#=V(~fkDx&tOa z(GKL5*j*!h!B!Z%d+_(Ml>xTVxI{aP-8I4&Y$c_5*hGlfpO5BFZfAD5X2?ee-@>S> z+M2tj)@ghf(J9yT7}aC`6teTxltLC%s2fR-A0MRFxP(ICWqD&8W>6sSZbf)A9A^un z0XPvy!lL1DvAdXpN1c3y#?xBYnejYIcTChR-%O6$)RXdK+($D?S7E+!PQ6cTRz?8z}NhEU`=`QGnY=v^6;^qym|80J4-{iJ66@6 z`4{1J-B6!cR-dRUsgHVp+KZp1-eonlp;+gdlB%jth^$DE|HeXDQBrnp-47duOYn|6 z*mjO8dPRjhfvavqX)nT6bN`CSE+{P&p8A8)R)0DjbWvbZV9j@m0?|61W$5aSI!9lR{L9bD5NUT}r9*?S3Uv|s7Z(sA#; zoc!0iwcc)5?!9!*wJXC7XDDWmA0fVtR?3G$Hpa-tYLExadkK(ce| zPLJq(@r=N6CxHK%y64uJA3`hb+fcif)h;AHI`vb^FP&Oe`!-O{&Q+zas&?`89i@*n zO{!UQ74s6nbu@-%X|-H&a#Ue86xBR|n$5CJJ643GihKV6_&bqpwg`7|WEdtV`50Nt z@?~kd8=8QiZd=MSL0?%Wvm>t2%48WBQ(atRx4Px;PnLny1z+rg=R`uE-6bay?ZAn= zuk-lJCqm!c0rwUA058}y;?zRYg%P$9uEW*UhPaMA74hi15oQ*B(DD?SMP;t@mV*`t z)$2{8jDvFBmvIpm<3hZ_m?FR!CGkk3MBpueIgfP|8Y{Do4hdb@Am#@y%5_^L>KLXE zyW4_Q+MIzgx%dsnZYcyle#6Ca$v*2*l+V#XrwQzaEhb~!;>ma)V|&Es+@{SD&hM-M za?7e|<@$|3C2SVwZkaugF=v%vh@2TdKycHzjxLD{n-A@Z2i9xT zHruNz$)$FHDmY;KYU@EH-yf{D9$ZxqA$eP7U|CIWDAu_KT#6ZYN_d*so>jI&eTbY$ zpsNK_mzl10MPE9Lh(po^JEESbF(P zpEP~edgbN~n*u^RKEpVQ;{p2DQL0w$IxVXt0N)}a444EOcjt`JjZu|)WVg`ch&l1)5A?;hd+_VL0=R5;6-r^Zx z41IN$MlbP)`^)3$VsxKWq@Aim$tb^7vWIeYhv_$$DSM3o1g1MBJWWPVt87IXwsY&OKT*RIC|vxeCKa_Is&vr|n>D6*jn;pbla!kAi?L^aT5w5{8@q~FWBUqZc8K`Q+u*F~Hw`O4jnoNLn+!9gCc z8>uf{+T0$TB+5b_b~D!cS@9jKGrMApSToOT`i{|Qhp|z}gEJ@o_=1XSO$~7!J;3Yt zifFJ5rZ?*W*f;!e_3UH&gcRauS8Wxtf9pZ)*)IH-19|%L$S8WHfU#_u0^!Ysj&Eno z5-e8Or$`VRZ+-uRJ|l9Kp_z$^(zPPIVtRI}&t-{w5q%ji5g`A3Eu1Tq@F9^p&7u}B zzhfJGxPpNVCm9n2E&*_MVw92e7`6&a9^VA8G^nI*cesX&+ zr|EblID;F=Ws4b_i>#5En!!7*<$|7}@1U57RLB%clg4+@KQRw;k4#kl*VfCLF3H>p zyZ02nRXuQGO*!fIhzZPJwX|=vbS<}ZEy=4b2j*Y-m(D}0?azLAX;J?B=8u~{-Mf5v z^vB(|x<9-$f3Dj8>@Q*6n)?A>jqF;?G5}N3hGLy-N*BEv=@jrZiSkz2iqhrEiDLA$ zFeh?AvUBTBkLY|cD{$Ni)coJwpw#i4Ne|O6x;kDysr=%27%+#tILMF)x+&c|4(reT zTbWp-y&`%Q3D90a;nM*}h8nVBH>U?}N&%Vub)prV^Egz?KhD61jyABYIIkLRK}VhT zzLa}&d(bLw(-FGG4#vvqsB z=3GeQ_c`aA+T*?`Sg+AM2%AzW z@1`T?WM`*aJ~HR}zT5>8MvzGulP=vf2n*uZb+vX!2nqwSF1y7*tXK;5}R{^q}Lq&Dd z_3tSC;*$-%8*2Bm+Fez8Z|~(mMU;(ot}4A%wOeFGg49fm=vS@-$7ZHe-jAF%I7WkOx<^-4e9CskedHf z>igd?zFK#;B`iOEw<9Qb-t7p}qJ-L{ZCNTC1jzPOya`&dXmat$(0%A_774NJphO$!;#K+;6y%&>tG zP1Yu%y{@&io0RymAKD~Uapi0M=l$I;SCUHUDZ#U4u z3^5Y5vUj~z0y%dYKzF0jm_GO3)3@I4YU}`W+mRih<{ak&qdF9?t|uBO?(rG7>eTbYC2enK3vX9F332rc94Xqlu9^ zv+j984Q&zB@D^dZ(X3Y^TLiN~je=}cV<4NknL&%$SrCk z$PTp$WT)B;vP*3Nxm9fixlL^Y`P^HviKEW#xZe}6x@jk+w)Y8Y=eRf+U`Ny~(7q0A z-=LkvR8(=N(9>0nL3hM2=!R)1|F-y3C6h8!%4{(;oi*o`R4${;rHFnlUo>pPu`r`3Z`5QVh4lCK5p%YUnkbEI^HDzY=xn8AMfNClV^eRUFiuqDTpVm#iS0Q>~ zIyJ8tsVllU-y5Ges}ytD*`khh_bR!(`BJVpJ*}j#>FMim;BwK(;wgm`r*$P|;1%IQ zaArDn9j`M}z89GRZi(wVdd-I6C6BMn_C;0HJC`5=5A z?gFtWB!x?XL+=Sf299kLUKg*4syGE_*w&BFBJe4evBE?*izL@0608a?6uy!tGbz)+ z-5u6j{TJc(jeb*4UxO!*HS_(~i_^t^=+eBwB$O}o2lFqu(fAp2@;~ zJ_lvtaZ{;bHqCzCeT8`|p=qfcJOJneO(Qs*n&4x!fLIj16hv{)XDI&s59rrJI9xf* z@RsRqpP?n(Tb!_eide$gwhL2|lb#4WDffHO?o8+$ajZw89byG@`8h&+Dp)kr9|}Z< z0we~)3Tmc78X-m7+y;q7;n#A@?V~?F`q$CFN&GBPl6NmlyJ-pfupkHBA|Pq2??c7- zu;$>KG$+Lg=W*H@@kWJs?8Z-g`g_Vo=Qq*#Mz78cAl%hz=!|nd)_XniDc-2 zsMbvm9taHD{i6od;8fJP)C7e&XODev7o01<0tc=>S9PUCWp3QtE+m7_oj5JbY^f?& z|LAh(MK!2Oo;&i-FFDug5A=GTb0MsT)NnG~FIF6lOyTpRo2TYx=a~BqcMx%EcD`|7 z9TCZeWbl&E+vVmhMAgXbon-9Lh#<@kIrn>YzUqEGt#iu46RA&1@I)FWZ+V_b)YCTq zyN)_-Qe%(J8*}nj^-x@mt8y}4)kE?V^pNb-?0mBxDny}|MyV2LTsZi~@VVhr7nEFT zMpsU&cCg^44akR zdkga~_ZWEaWh-=YI-Axpy&uJ3MXrM^*TE0Pufkaaq5ng1p&iTb>oF8|gjRQ;$ACP6 zW5?ipGoIfM=Z_tu=LX?i+=}RL>*=BixD8q*eGPEiik~RVBiLyvVl`>$)buo9b%CJ5 z0JUZ%;IwwRn4LC3i&^36JmfH}4$T0CH=UtbwX4(l%c*G%d&Y`ssIjmHtNx^;g@y^q z6RSmID6Q#lgKEoBMc9+D!iKJA^o%8?^SPWQY6OAQlJUB%s|LI~``HewY0$fK(3@?a zcjtgN+soda?;1Gh_bBehP*R~i!t}O=YArTnBuN5{N}%y}tFG*A|D? z)*Nd!#}AH@S*=c^+>9&OAZ^ zSOEB6j{>NIpWvkcssjo?PpS|iR3Stw6oRM*zNHEQRh6x_1YyD-8fz;ANezAN3L%UN zK@Ec$5nGtDq3+c9#jy)}U+7_Qhc+EG<@6h9EZK+$*5kBJhdu>72M@cIlVd|lzMvC2 z%<&u>P2B{Qlod+ni#e0DLYklvLfS!$5%5R^2fPs6Z14hLBRk+!Pn5ibnfp-mqd1IW zABz1b22dOTVW1)4jB7Tgx7HZbcwYpahV5AHCm`J6oM^m#?$){c0Wr4gVK7)1{xU2? z8gNK2HGL5Y*0nAU-;WAy?aNZz(#czg%TgQrFG+1TU*>x};W>8C&G9QN%=35?km(V_ zun&Gfzr|s=IV&JahheO87~HD$ z*-4=zVj)OvC4W0N>{twsvU4{DbIsWo|ISvr6BBo!<2`LbmHm!$K{@(9~q^Q2cX zH_zow+>_4ps~j)Mua@PJ6?vp2pSin#O+LRYsmrgcOs>dkS+f62a2zj-yKA2GD$UQc zc@uDIqpCuhfl>xTAYD~8#Rt0Mju(Q-fZqUDfqewGo;uLz2CES@s>YH*HJ%LihX-wH z;duc{JKQB)(%ImskE?n~?DbFs4ty7+DWpf}HreWMf>5(K!4-&e9_S73FizCDN1YVE zt57mj1^r=Fn!S;XPz(2Fm4nT#zzOWPstp|MreqWx?B>a<9tS%y<(Bey{<(ru>mHkz zgKd?A9aHPohGeYD!H!R%;YK%4&1lC}ZJ^-ePCq-};9xg`YDKH_edr~zf`$mI(PzDc zo#^c)wGjZk#fmu;gJA{I;0!yvz&KbEdf*E(rKaVcfCBN=C>~Xi zpnRkiW>>W8Y^qoTLK`@d2E=TjCk!3Q@3cKFilkEoV2pJuU>cU3)8`yM znvxBqrdq+V?|2cw5im!BX%^O64JrOPpqO>TB?mMhdx#1}v0h^@(=4rB@0?#8`gJVv zleWe0txNLzWABWuM_bmTiBA%pcLvuJ%}eLk6D`Yad)E`KOV`&E9d~w6albASi+>)G zL*e@!LQ7Ywnzi9WQgjID_nmt%G}ii{`#Xe83z5)HaYCkAAsvqTyIRYR^2Y_ z`cmfZmHg43yJ$`eZOYs&hc;%_23mrb1IoNXZS-gr|NJ4R2dnf|L~T-=laVTY6`ev; zmTsP!w>ih$Z)gdprJXAP61RY^($y9~;dU!xo4rX`nz$&S3_MqUSL+hGHj*T_{{*9G@Q9i=qcbm2Dn#;gyWz`3V#? z`53gWIFd3PJwS2Y#?ee35EQ6wjp-mQQ!IeW!YMY+xl&kt6Y9Zc8~+Ugh#G%bZKANc zAV%!gIm9loK%oF-fx^El7AObc=$h65P~5nN{;@a5jP4gbP=}yOH-I`0 zd(JTdP3lb>&lOO|&1knSmCvF18PljFaf8`Fsm5#|=WG0_QZP<&Hn6dRbpm%`iUZ*U z2RU{vl)ljBUda~>Yt9C`<@_O$_W(c}xG__Y%g>>(3fjOl1_x;4Y#{pUbo12gnb~vB z0<>`>-vezDWwM9;0kHGeRqI z`+}dci9Op08uGM!=C&*@5%f*P*D#|U|?Xxn^VI`V<^!JmATS>AJGRXR!y z;MxH0n8ibWX7O)jX0CJT!mCb3{oU_E zXo^#_lWi3is;`CYXnQ{z+PSN$`wma054{JGQgi4i;db>?8Y#tXq8dbQOELo1QR`&d z`*XD?G^4DqMCLM8J4y;wMr@j%AfLp{**fw4cNrEctII1=xO89+To8^n}TSUF|Yr1OkF{NQ5fC;s{;hBK?8@agU=3)x`QG#?L5R|8W5u8 za}X~Kye|;ZhHj8)DEBT_u%*KMt82PFb!%$5>qM#UWV!ClO5K@K-MPE3uhoq&p8k(V zF|6Y*!|J60Hp{R!FAc6ryKOFDx%1Uh;zT(yx{??zCC&m_kQiGWUY8qhRoZtD2x41% zJb{5A_OW|l_PKlfFdB-sVF_l8x2AZ`M(0k-X#Z~mNVV#yqhoG5P_#i zH=r9f5A6ZJxw;B4=OFLYUokt^;2w2S4w!?Ak{X4Jp<|8SZ3RE-e&fGEP^)WPq+D2#Y+9z*#0Kd*-fVMhqbHBZgIt5^E3QqWP zq0U`l`|QAGw^jZ`00;Rh0Cvmyw~Lqa0pJ(~;Egkd5Ngdf&t0P%TeZPm`^^+Kg9-Y! z+VZUbkz7gsuKK_AS^xh|`yWgYTzypx6hc{{XN#|y5oKn^je$p77z;BtW#tBF(unpd zOJ*}6o&}Nx3TIxsuFu;M@5;D3G^{G$`$NTbvUogCr6zAX|`6CdN z*n*WkZUB2W=LEO+GrPz$I+WSRoPi?IZhb(mjgR1RORS~kv#FC`kI^?Ji(%zR&dVIR{2c> zMZwCAf}<;w$X!MN>qhqiD$`f&9uMDUNAH~aAArqEIPLr#cj%Kn3EFMam zBn!LDXRQ_J4^Ar$^#5=}p#KdlJBI>YOmZCsbuBkif4eFEy3@u^gCPX;x|TIGwvLu8 zKo!4)kHh%cD|i@vx7>NOlsHySoL)(sE+s|)ClhDExoc>-{exRSSdSiuNrc9VS>VRj z#gY4AA<BR4I`_ep#K!cieF%W&md$9+S3w^H39r20zOgR$kL#=MX2$g;bY8zaEGBY zkzcXImPTU9t&68X2e-5?jxTM$b?E(Z{QKef%>yjOq_g*~ef&p|*L(i`@ps1`p;MoY z9C#S8J%uj(b}xS2U!jtydx8bEVa6PL#Xvm+zZq4yvt3JD8 zHWqUEoR-t^oiPw3iW4DatRV-YdOR#|Ia7+CEypjc#4nWM$@OUL3tWNzY(VhWgFIUe zzIjj3t4hXe4G0f_JFCGnS$zEwygx@78^558jwyS)VJa_^HM@J2?u>pJ5eF0uH<}tLv}CmK#W3g z3Y|H+O}13beffm#sxH5SZeZ1h6eez!^IQc9-n&q1Rl#HU5DYkiV1`)5ssaRf&V~4g z>^d#+%vZTr;xkaJwJ2fUqZW_imh=C8^K#4$^DsRA`D?E#fRGiRRR!)UIJYHywyxq1 zcz~DAEYi7dtF&Jca;&_&KfQrc~u?OZ;)HizW;%b@mC;t#MLLw zo%RYRvDEy6XIaxDd70L>Z}=4qRycpY`ak^2YhyxNdOVAA&&7VkebbA^qE2)UQ=k2+fdCH@Uh|H6SmN2y}qqnzjLL2=Le@t z^)D{IX0LkO#ba`OY1?lc9(sc0N6^+!5gF(Z@7ZoeG%b@)Ynl~xUY|uF+3M8T3%B~% z+pWM@>LWT`LTbhMAuD1ZW0RP)UL5RJm=(6_3fY1_4X=FcqwlZ5A#64CW1ReED*#ZT z(l=Jo_uP;ZxT~VT)l@Xbp2ok@a4R|?Ny%}7JQ!I-@h*z@P#|hs5=QM?GCWV1!~|Y{ zL8p(4G++*#6J<|5dUPhAf!9$UCkXL)55`YH+z*PP_*oz5zA+@ayQv15Ztq)F! zcJ}8UbcDp#2aQyOA=MEP+wZpuf#~~N-`QFgJ66PwI~{-fN=ba_bGUx{!=Mo8dw?n0 G>i-8gdBgbt literal 0 HcmV?d00001 diff --git a/submission/tests/__pycache__/test_schema_contracts.cpython-314-pytest-9.0.2.pyc b/submission/tests/__pycache__/test_schema_contracts.cpython-314-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..76d6304a74b3c8e334e3712babf2f1e92ce3a6e0 GIT binary patch literal 16244 zcmeHOeQX;?cAq7e-_&PhTef9sCEKx?$ojHu%W*6pEz4&~wyfJIG!p1S(BxX;g(B%) zGPZQdo$doTr3o%>dq7>lZ5132RnUL(M^W6NY0&`JqJP{UlqpNPt&66`0l@)9p;%7S zq$tq$W_D+n6lK|MfoqEl$l3XxH?uRp_vX#pXRE9H1l^l|lWb`uBu*oS=Q5rfIt1p3 zOmgHEA`3%8&Ox8f3u4~ULN2&+?iQBg$$4mwH|K??W5{>GpYvY`*N6V^>PsWr{pT|8{}&6cYNJD)@~<~g~vKK zr^z)PM6SOq^g39cTnoL{VXwVL;asCE>?fh8@?L=u|G1*%H0ht_ek5t}DP=k)#WU%g z8jI(ogpyO@xnw3S#nK5$i(ONWXt_*Qx~fdYUQ1@w4!?gO^O~Y+&q>*srX|ypysp+R zO()HyRDL>bHP$XA)A7u7HkM0XO)0$7saSea@mpOUPN`MC zFDk9OC{emwRdRVX9TIhSA|HP>aaFHB2hVe7$7Tkg7<-SpQ_HE~DY=lKyRT)`=~zy~ zk+kWt&KEU!lb!I!r{I+)bD7Rp^QnBNs$?@7^Pz)mXCju19l@79nVweCIjvL6U!`w4 zlZLvmlBgeLODSqDC9NYnqX(nWSUR1_;eF{khpy|ansWG@H%ZSLdMKDghdS}lZX%`J&s-@Zwzac zIdhZLk{2C$M;FOE&Ibs&-u4QyzlT>HBUOa>NHgST9s9v=p`j!R3A4U)gD(uoLuUsi zSW_P5rV`KR6y1AG$;GE)sZ_|RBDE^MYTc=3ZfHox(@#q(G)eTDDD1s_ECrK?3tv+k zAeGM4NeIl5wdS@zO8tIn@um0k?@tzrQH(VIX{0M zf=hvsf;jT|-N1;3tL!IrC&iNG%ct#S+V+2M`C5zDK2@DCJuqbKo{bx~Oh{C{nBW6X z7gI@Cmk7uqK@Gx#tw}l^be&Ld2WA`q4;M>N*!c;1caE>_?Ood2yN;0$8a{~s#crCe z)<8izQg~8_-N8JI8$o#abEJc0x7nDj#99jHVd`(da#Q#3B@-rpAXgkF#%9HFXDvD3 zMo8BEQsN}^FbRD%kr7xrPC56Hq!4oD&jE2m=~ObUSQrVs zApR<43)4we&8Qk6wVKSr2B$fdc6i6E<2h+m7dmEZpEF?FnSgN5#Jq3!HS?ZAc$3Vu>WTKm@PNpNRD z-0^P@t4Qtchs5FAu@>C98r-)O+_$h}!Ly()?qA$dXzpANcKv;@;osL?P~r=17ZCM_ zf$%QJhr0r2ovshtc82?1ANIQ-9EU9KU_mjQ`v8{X9Cc*H5Wk%@}`t(PfM+G@>rT(a+S7e#8 z0DKO&Hg#hN{$v>) zkydMVIgRknOU77{MynpsCdw|`>{l)?m~$b!w=K&#VW-&6B@X~GQTE8*h}bEVq`D?> z|4?UZd8e6XJzGh{Wl~{2K@#$v60*W6LG6)!z!HLAc%PeArZeiybGkD-(?vHZhPoLD zKEPk)OkYUD#JT5`hTnc_A|x6!MSty#m3tW%VP8Wz~0;Eih%S6>@zbBH}^e`imwU0K5Xv z2SC^cCgniH1@IcY(pv_v9CpSk0)r{d*x=Rb31!SO2k^?_-Q2_`tp>bW^(%tF=K367 zpM+KhuU20bX|-mT)5ze}s#lRltDXU`+m+{16~OCuWw~5e2Cw#-pnWKJ0H1J7kv-Ny zMfSGa3dz2Ry9}B=6TtA8vr*M#rddyD+he{>^9j)G2WWNzG>;DoBi~?>r|buiu2T0y zUG)HXx-+Gu)q@yDyB*4j3M-Cv$g*IQp{ElL%C5nbqf6e*`1FE z+5Hs95wUC7Y2jJyk)kyUlk{Mu!k!^2&W?Hny$w{AvV z#6ya@U%=aXGSr3RJ3nPXD~6M~XtR~rDm!+;-l^p$H^5(x%GOS|MB2$eLBvIY4+JZY zc4K^Bn?0kjSph$}7(2%C%wEG1lFXgPv5ASNR{e^Iy}3SMlnZcYn|jtrWzV){Ii7Vo zON2eqChS@LRmM<`bLKb!QM`aA`(-a+h>v2(KT>6*xqk-a!SMK?G#);CaZrlqK~Xmi z+9&C--z6uVGWb$^NnsSY%*tlr(aYn~=!+LGmX_2& z1C`fU*+-XT+P_2;bc@pX(51l_#>1D!q{^-4v!EJNwkpd>4bf1EW&-yJLeU1ISQ}}9 zy)sVD!)nY5sIWqQeUr?&KsNyE5_JQpy*=iHlez;AGX72XEj!G^-)&{WZ{6sP20A z#_iT0zW(EvKi?1`RXR%}J=f@&;h8N!=tLe?W-x#@{D2;0N*7A3goyCrbLHOYSg zxB<2S4*zcexBD@`n9q~!{fz-PoYDcf@r3TDz&*Rp!D}njesZARgu4<2G7j^VG1frt z&6RCH{RB9@xhO;DSQiK^l?QzT&dLg0mz6RqmWpgj;v?9SQB|6(%EOK4RaHsHXAqk< zjhWMjx&NP7ng`S;v2LayO*`PPHP<*R zOZ@?}5la@{Lh1*Zin+&DN^@DivYy3aN2WAfCiPqOaS4{WN?I2~>fffGHBw+S+m&^i zv~805@gjmcTN{;i6pthI&psQzI6f#Ff5a?x48^L5`st|c%LAfG?9eJX`s!^aJhQs?2 zcw1>U3>mmk*S{>D`BhyjmwsU;iq%5}aR@SJEcq2NpQqimVk_bhRAV{I z5Z+W1vNnSjUa`Z0OWgP~@aLHM4!Iy?VewcN7h|XmDsynJ#ig?>EVD|Nx-#EZ(3%B7 zTy2qcM2MwG$T}kqzJTE+HjzuaBF-_Su*{))LUzL+Ze=D@5<1*J1^HBdTp1;kiIP?8%W9#A^(xDv4`oqe)@f-3~ZDuP6> zK?fdUjXOQ7C!1Z1Q4D6sf*lj0Oj-@4vsJ&MdUJDqfKR_1kb_S|D^ogKeO072!rwQi zktv<6dKGE3>H%#*xym;CmCN(DC|7S=mfyU|HQSbTn{(XhDpCG;Z8e+CcVj(aFVvbf zd*VbyAnPxpfHSIH7HJ_kJdJ0t2BK%!&E_qFU3eF`vXW#N) z+18}19Tn08=LRHIiD%RV*e}pqIC;76T=RVF1q^w|-EcFi zi*Pa8HL8Z7Lr};Mp+f$csjh)QnYyOJ^(tJecBC?s{x@uwt8h6Rs@H?rWLClX>xd`f zD)I*#v*LV8cf%bj-0H%$LT2(YwD@X2Hq?V2)!X}7%c#3ianq=i=po)vy}e!CW8981 z&D3$K%x71*>{_6^TSE;39HR`!)Idy}H^&mim7zt^l91n(8u~INp=P1}H2k$+2*_1F z^%K|Ohi($Aw_TlXqYG+$D>e7kk3DrCd4nH$Yu>4Tv-;LI7tSr7{^^0ga{sNT@XXk< z_iO8JQq}Oz$eSbUPKWEPaNir4A1QhpS!}IopAqmjE_wHTMvHt(u;9mC;iazdLl=hE zyp1etgJqox|Lpi@H0NJgLMXfx3WHS&>wuO0jAiwOf95O80RtE8;8Crt1)tiY6`o_z zVzs{(+{NmAQrQQIg9@+AKhL;58!y z-uPz`U$soG+l+9K&0ViEnZe{=Qr3#{8Fgb&quc?M`XU0bR)56Sj*ZC*6IT6-Vu^Kv zLemxN&Y-e!P6dW+wa?E&7L~w|dBg-|b9@Fvw(3`;Z*zTMdd^Nq3Bz%qh!b&?u@2`1 zn(Di2EQAl z)oD!?3;~D}kGIcvgFEdRCZjPTYP$Q^Ul^Np#U%d%6e;fW< z8|CP00kG)|-RWEkgjPl4R}@3@r|xvJ_&tp9EEZV78a(Z8NsK2$0CLs>4Pf|mr^aqY zRTK}+pS)AUtefs(#BS&WVX%lYvmpkiQnr}ADCDdKb{E9;`>JUCiemeG|D7m{ z-@^#cVu2N`!PD-R#CS3Umc{l*@X1^Bpe++HK!y2aP8RGI;@4}L*5-Pf$yp|69*c7Z zA`WT^CD{0-!^SBp@*^9+bZ%RgAKlCN6}1?MILr8zXjvQBTISbN=IANE;zzntew8J_ zvsz_O$!vMQoGo*f%qQdS#%%dYX3PKOY?-rUK1*f`vAjx6z-qJsI^J!RU^$pwQRA*r z-N&!DR253;)ixa}oMkD*s~Y4~KzO+M2dmaf2ya2UOU)xKGJ5p<_5y~`?yCHTrJJfJ z%K_X_$ZBA4m7I>LGf{+b^W&M0b%}Ivc+tq@X1&M(^3e%d{jfW)DCXh3Huy z{amaariW_V@y(z^MoCXK$Zog$j15fru?}_iuX_+~RS7)!j0ZvQQD5`?G30BgTEee+ zoWclv`i1@n``797=L0Ox_zFK^KzR@RDWxA##%xg%DfvJv>$p-+PhO zb=iO*3MW%BLck8yMXgcYJ(gn8Q=;;#^q~M(K->DLuhhp@9}1Ws38*=!#WZ9RhMop* zcK6vld~K>|+7b99DVGE*eDiZ76}8Z_d+@tX_|g#N%(729%S=-N}6zbPmCWdui0yMhxE;6)R_e7(KOgtLZz2@f~^a!Hwj#tsNRph5b^ F{u2THA94Ty literal 0 HcmV?d00001 diff --git a/submission/tests/conftest.py b/submission/tests/conftest.py new file mode 100644 index 0000000..2261284 --- /dev/null +++ b/submission/tests/conftest.py @@ -0,0 +1,125 @@ +"""Shared pytest fixtures for the payments CDC pipeline tests.""" + +from datetime import datetime, timezone + +import duckdb +import pytest + +from pipeline.cdc import CDCCapture +from pipeline.lake import append_to_lake, create_lake_table +from pipeline.warehouse import apply_cdc_records, create_warehouse_tables +from source.models import create_source_tables + + +def _ts() -> datetime: + return datetime.now(timezone.utc) + + +@pytest.fixture() +def conn() -> duckdb.DuckDBPyConnection: + """Fresh in-memory DuckDB with source + lake + warehouse tables.""" + c = duckdb.connect(":memory:") + create_source_tables(c) + create_lake_table(c) + create_warehouse_tables(c) + return c + + +@pytest.fixture() +def capture() -> CDCCapture: + """Empty CDC capture log.""" + return CDCCapture() + + +@pytest.fixture() +def seeded(conn: duckdb.DuckDBPyConnection, capture: CDCCapture): + """ + DuckDB connection pre-loaded with two customers, two wallets, + and two transactions — all flushed through lake and warehouse. + Returns (conn, capture). + """ + ts = _ts() + + capture.insert( + "customers", + "c1", + { + "customer_id": "c1", + "name": "Alice", + "email": "alice@example.com", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + capture.insert( + "customers", + "c2", + { + "customer_id": "c2", + "name": "Bob", + "email": "bob@example.com", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + capture.insert( + "wallets", + "w1", + { + "wallet_id": "w1", + "customer_id": "c1", + "balance": 100.00, + "currency": "USD", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + capture.insert( + "wallets", + "w2", + { + "wallet_id": "w2", + "customer_id": "c2", + "balance": 50.00, + "currency": "USD", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + capture.insert( + "transactions", + "t1", + { + "transaction_id": "t1", + "wallet_id": "w1", + "amount": 25.00, + "direction": "credit", + "status": "settled", + "reference": "ref-001", + "created_at": ts, + "settled_at": ts, + }, + ) + capture.insert( + "transactions", + "t2", + { + "transaction_id": "t2", + "wallet_id": "w2", + "amount": 10.00, + "direction": "debit", + "status": "settled", + "reference": "ref-002", + "created_at": ts, + "settled_at": ts, + }, + ) + + records = capture.records_since(0) + append_to_lake(conn, records) + apply_cdc_records(conn, records) + return conn, capture diff --git a/submission/tests/test_catalog.py b/submission/tests/test_catalog.py new file mode 100644 index 0000000..9f7d056 --- /dev/null +++ b/submission/tests/test_catalog.py @@ -0,0 +1,134 @@ +""" +Tests — catalog metadata completeness and correctness. + +Covers: file presence, all required datasets, required fields, layer labels, +schema presence, and that lake/warehouse datasets are correctly distinguished. +""" + +import json +import os + +import pytest + +CATALOG_PATH = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "catalog", + "catalog.json", +) + +REQUIRED_DATASETS = [ + "lake_cdc_events", + "wh_customers", + "wh_wallets", + "wh_transactions", +] + +REQUIRED_FIELDS = ["name", "layer", "description", "owner", "schema", "update_cadence"] + + +@pytest.fixture(scope="module") +def catalog() -> dict: + with open(CATALOG_PATH) as f: + return json.load(f) + + +@pytest.fixture(scope="module") +def datasets(catalog: dict) -> dict[str, dict]: + return {d["name"]: d for d in catalog.get("datasets", []) if "name" in d} + + +# ── file presence ───────────────────────────────────────────────────────────── + + +def test_catalog_file_exists(): + assert os.path.exists(CATALOG_PATH), f"catalog.json not found at {CATALOG_PATH}" + + +def test_catalog_file_is_valid_json(): + with open(CATALOG_PATH) as f: + data = json.load(f) + assert isinstance(data, dict) + + +def test_catalog_has_datasets_key(catalog: dict): + assert "datasets" in catalog + assert isinstance(catalog["datasets"], list) + + +# ── required datasets ───────────────────────────────────────────────────────── + + +@pytest.mark.parametrize("name", REQUIRED_DATASETS) +def test_required_dataset_is_present(name: str, datasets: dict): + assert name in datasets, f"Dataset '{name}' is missing from catalog" + + +def test_catalog_has_at_least_four_datasets(datasets: dict): + assert len(datasets) >= 4 + + +# ── required fields ─────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize("name", REQUIRED_DATASETS) +@pytest.mark.parametrize("field", REQUIRED_FIELDS) +def test_required_field_is_present_and_non_empty(name: str, field: str, datasets: dict): + entry = datasets.get(name, {}) + assert field in entry, f"Dataset '{name}' is missing field '{field}'" + assert entry[field], f"Dataset '{name}' field '{field}' is empty" + + +# ── layer labels ────────────────────────────────────────────────────────────── + + +def test_lake_cdc_events_is_labeled_as_lake(datasets: dict): + assert datasets["lake_cdc_events"]["layer"] == "lake" + + +@pytest.mark.parametrize( + "name", + ["wh_customers", "wh_wallets", "wh_transactions"], +) +def test_warehouse_datasets_are_labeled_as_warehouse(name: str, datasets: dict): + assert datasets[name]["layer"] == "warehouse" + + +# ── schema presence ─────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize("name", REQUIRED_DATASETS) +def test_schema_field_is_a_non_empty_dict(name: str, datasets: dict): + schema = datasets[name].get("schema", {}) + assert isinstance(schema, dict) + assert len(schema) > 0, f"Dataset '{name}' has an empty schema" + + +def test_lake_schema_includes_operation_column(datasets: dict): + assert "operation" in datasets["lake_cdc_events"]["schema"] + + +def test_lake_schema_includes_sequence_column(datasets: dict): + assert "sequence" in datasets["lake_cdc_events"]["schema"] + + +def test_wh_customers_schema_includes_pk(datasets: dict): + assert "customer_id" in datasets["wh_customers"]["schema"] + + +def test_wh_transactions_schema_includes_amount(datasets: dict): + assert "amount" in datasets["wh_transactions"]["schema"] + + +# ── update cadence ──────────────────────────────────────────────────────────── + + +def test_lake_update_cadence_is_real_time(datasets: dict): + assert datasets["lake_cdc_events"]["update_cadence"] == "real-time" + + +@pytest.mark.parametrize( + "name", + ["wh_customers", "wh_wallets", "wh_transactions"], +) +def test_warehouse_update_cadence_is_near_real_time(name: str, datasets: dict): + assert datasets[name]["update_cadence"] == "near-real-time" diff --git a/submission/tests/test_cdc.py b/submission/tests/test_cdc.py new file mode 100644 index 0000000..704902e --- /dev/null +++ b/submission/tests/test_cdc.py @@ -0,0 +1,135 @@ +""" +Tests — CDC capture correctness. + +Covers: insert/update/delete capture, invalid operation rejection, +sequence monotonicity, checkpoint-based replay, duplicate safety. +""" + +from datetime import datetime, timezone + +import pytest + +from pipeline.cdc import CDCCapture, CDCRecord + + +def _ts() -> datetime: + return datetime.now(timezone.utc) + + +# ── insert ──────────────────────────────────────────────────────────────────── + + +def test_insert_creates_record_with_correct_operation(): + cap = CDCCapture() + rec = cap.insert("customers", "c1", {"customer_id": "c1", "name": "Alice"}) + assert rec.operation == "insert" + assert rec.table == "customers" + assert rec.primary_key == "c1" + assert rec.data["name"] == "Alice" + + +def test_insert_assigns_sequence_starting_at_one(): + cap = CDCCapture() + rec = cap.insert("customers", "c1", {}) + assert rec.sequence == 1 + + +# ── update ──────────────────────────────────────────────────────────────────── + + +def test_update_creates_record_with_update_operation(): + cap = CDCCapture() + cap.insert("customers", "c1", {"status": "active"}) + rec = cap.update("customers", "c1", {"status": "suspended"}) + assert rec.operation == "update" + assert rec.data["status"] == "suspended" + + +# ── delete ──────────────────────────────────────────────────────────────────── + + +def test_delete_creates_record_with_delete_operation(): + cap = CDCCapture() + cap.insert("customers", "c1", {"customer_id": "c1"}) + rec = cap.delete("customers", "c1", {"customer_id": "c1"}) + assert rec.operation == "delete" + assert rec.primary_key == "c1" + + +# ── invalid operation ───────────────────────────────────────────────────────── + + +def test_invalid_operation_raises_value_error(): + with pytest.raises(ValueError, match="Invalid CDC operation"): + CDCRecord(operation="upsert", table="customers", primary_key="c1", data={}) + + +# ── sequence monotonicity ───────────────────────────────────────────────────── + + +def test_sequences_are_strictly_increasing(): + cap = CDCCapture() + r1 = cap.insert("customers", "c1", {}) + r2 = cap.insert("customers", "c2", {}) + r3 = cap.update("customers", "c1", {}) + assert r1.sequence < r2.sequence < r3.sequence + + +def test_latest_sequence_reflects_last_record(): + cap = CDCCapture() + cap.insert("customers", "c1", {}) + cap.insert("customers", "c2", {}) + last = cap.update("customers", "c1", {}) + assert cap.latest_sequence == last.sequence + + +# ── checkpoint-based replay ─────────────────────────────────────────────────── + + +def test_records_since_returns_only_records_after_offset(): + cap = CDCCapture() + cap.insert("customers", "c1", {}) + cap.insert("customers", "c2", {}) + checkpoint = cap.latest_sequence + r3 = cap.insert("customers", "c3", {}) + + replayed = cap.records_since(checkpoint) + + assert len(replayed) == 1 + assert replayed[0].sequence == r3.sequence + + +def test_records_since_zero_returns_all_records(): + cap = CDCCapture() + cap.insert("customers", "c1", {}) + cap.insert("customers", "c2", {}) + cap.delete("customers", "c1", {}) + assert len(cap.records_since(0)) == 3 + + +def test_records_since_latest_sequence_returns_empty(): + cap = CDCCapture() + cap.insert("customers", "c1", {}) + assert cap.records_since(cap.latest_sequence) == [] + + +# ── duplicate / replay safety ───────────────────────────────────────────────── + + +def test_same_pk_can_appear_multiple_times_in_log(): + """CDC appends every event — deduplication happens downstream.""" + cap = CDCCapture() + cap.insert("customers", "c1", {"status": "active"}) + cap.update("customers", "c1", {"status": "suspended"}) + cap.update("customers", "c1", {"status": "closed"}) + + records = cap.records_since(0) + pk_records = [r for r in records if r.primary_key == "c1"] + assert len(pk_records) == 3 + + +def test_captured_at_is_utc_datetime(): + cap = CDCCapture() + rec = cap.insert("customers", "c1", {}) + assert isinstance(rec.captured_at, datetime) + assert rec.captured_at.tzinfo is not None diff --git a/submission/tests/test_data_quality.py b/submission/tests/test_data_quality.py new file mode 100644 index 0000000..f33fc86 --- /dev/null +++ b/submission/tests/test_data_quality.py @@ -0,0 +1,252 @@ +""" +Tests — data quality and warehouse correctness. + +Covers: insert propagation, update correctness, soft-delete, replay safety, +PK uniqueness, not-null checks, business rule assertions, lake completeness, +and lake immutability (no deletes or updates to lake rows). +""" + +from datetime import datetime, timezone + +import pytest + +from pipeline.lake import append_to_lake +from pipeline.warehouse import apply_cdc_records + + +def _ts() -> datetime: + return datetime.now(timezone.utc) + + +# ── insert propagation ──────────────────────────────────────────────────────── + + +def test_insert_appears_in_warehouse(seeded): + conn, _ = seeded + row = conn.execute( + "SELECT name FROM wh_customers WHERE customer_id = 'c1'" + ).fetchone() + assert row is not None + assert row[0] == "Alice" + + +def test_insert_appears_in_lake(seeded): + conn, _ = seeded + count = conn.execute( + "SELECT COUNT(*) FROM lake_cdc_events WHERE table_name = 'customers'" + " AND operation = 'insert'" + ).fetchone()[0] + assert count == 2 + + +def test_all_tables_populated_after_seed(seeded): + conn, _ = seeded + assert conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] == 2 + assert conn.execute("SELECT COUNT(*) FROM wh_wallets").fetchone()[0] == 2 + assert conn.execute("SELECT COUNT(*) FROM wh_transactions").fetchone()[0] == 2 + + +# ── update correctness ──────────────────────────────────────────────────────── + + +def test_update_overwrites_warehouse_row(seeded): + conn, capture = seeded + ts = _ts() + capture.update( + "customers", + "c1", + { + "customer_id": "c1", + "name": "Alice Smith", + "email": "alice@example.com", + "status": "suspended", + "created_at": ts, + "updated_at": ts, + }, + ) + new_records = capture.records_since(capture.latest_sequence - 1) + apply_cdc_records(conn, new_records) + + row = conn.execute( + "SELECT name, status FROM wh_customers WHERE customer_id = 'c1'" + ).fetchone() + assert row[0] == "Alice Smith" + assert row[1] == "suspended" + + +def test_update_does_not_create_duplicate_warehouse_row(seeded): + conn, capture = seeded + ts = _ts() + capture.update( + "wallets", + "w1", + { + "wallet_id": "w1", + "customer_id": "c1", + "balance": 200.00, + "currency": "USD", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + count = conn.execute( + "SELECT COUNT(*) FROM wh_wallets WHERE wallet_id = 'w1'" + ).fetchone()[0] + assert count == 1 + + +# ── soft delete ─────────────────────────────────────────────────────────────── + + +def test_delete_marks_warehouse_row_as_deleted(seeded): + conn, capture = seeded + capture.delete("customers", "c2", {"customer_id": "c2"}) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + row = conn.execute( + "SELECT _deleted FROM wh_customers WHERE customer_id = 'c2'" + ).fetchone() + assert row is not None + assert row[0] is True + + +def test_delete_does_not_remove_row_from_warehouse(seeded): + conn, capture = seeded + capture.delete("wallets", "w2", {"wallet_id": "w2"}) + apply_cdc_records(conn, capture.records_since(capture.latest_sequence - 1)) + + count = conn.execute( + "SELECT COUNT(*) FROM wh_wallets WHERE wallet_id = 'w2'" + ).fetchone()[0] + assert count == 1 + + +# ── lake immutability ───────────────────────────────────────────────────────── + + +def test_lake_row_count_only_increases(seeded): + conn, capture = seeded + before = conn.execute("SELECT COUNT(*) FROM lake_cdc_events").fetchone()[0] + + ts = _ts() + capture.update( + "customers", + "c1", + { + "customer_id": "c1", + "name": "Alice Updated", + "email": "alice@example.com", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + new_records = capture.records_since(capture.latest_sequence - 1) + append_to_lake(conn, new_records) + + after = conn.execute("SELECT COUNT(*) FROM lake_cdc_events").fetchone()[0] + assert after > before + + +def test_lake_retains_all_operations_for_same_pk(seeded): + conn, capture = seeded + ts = _ts() + capture.update( + "customers", + "c1", + { + "customer_id": "c1", + "name": "Alice v2", + "email": "alice@example.com", + "status": "active", + "created_at": ts, + "updated_at": ts, + }, + ) + capture.delete("customers", "c1", {"customer_id": "c1"}) + append_to_lake(conn, capture.records_since(capture.latest_sequence - 2)) + + events = conn.execute( + "SELECT operation FROM lake_cdc_events" + " WHERE table_name = 'customers' AND primary_key = 'c1'" + " ORDER BY sequence" + ).fetchall() + ops = [e[0] for e in events] + assert "insert" in ops + assert "update" in ops + assert "delete" in ops + + +# ── PK uniqueness ───────────────────────────────────────────────────────────── + + +def test_warehouse_customers_pk_is_unique(seeded): + conn, _ = seeded + total = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] + distinct = conn.execute( + "SELECT COUNT(DISTINCT customer_id) FROM wh_customers" + ).fetchone()[0] + assert total == distinct + + +def test_warehouse_wallets_pk_is_unique(seeded): + conn, _ = seeded + total = conn.execute("SELECT COUNT(*) FROM wh_wallets").fetchone()[0] + distinct = conn.execute( + "SELECT COUNT(DISTINCT wallet_id) FROM wh_wallets" + ).fetchone()[0] + assert total == distinct + + +# ── business rules ──────────────────────────────────────────────────────────── + + +def test_transaction_amounts_are_positive(seeded): + conn, _ = seeded + bad = conn.execute( + "SELECT COUNT(*) FROM wh_transactions WHERE amount <= 0" + ).fetchone()[0] + assert bad == 0 + + +def test_wallet_balances_are_non_negative(seeded): + conn, _ = seeded + bad = conn.execute("SELECT COUNT(*) FROM wh_wallets WHERE balance < 0").fetchone()[ + 0 + ] + assert bad == 0 + + +def test_transaction_directions_are_valid(seeded): + conn, _ = seeded + bad = conn.execute( + "SELECT COUNT(*) FROM wh_transactions" + " WHERE direction NOT IN ('credit', 'debit')" + ).fetchone()[0] + assert bad == 0 + + +# ── replay safety ───────────────────────────────────────────────────────────── + + +def test_replaying_same_records_does_not_create_duplicates(seeded): + conn, capture = seeded + # Replay all records from the beginning + all_records = capture.records_since(0) + apply_cdc_records(conn, all_records) + + count = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] + assert count == 2 # Must not double to 4 + + +@pytest.mark.parametrize("offset", [0, 1, 2]) +def test_partial_replay_from_checkpoint_is_idempotent(seeded, offset: int): + conn, capture = seeded + partial = capture.records_since(offset) + apply_cdc_records(conn, partial) + + count = conn.execute("SELECT COUNT(*) FROM wh_customers").fetchone()[0] + assert count == 2 diff --git a/submission/tests/test_schema_contracts.py b/submission/tests/test_schema_contracts.py new file mode 100644 index 0000000..69d30f7 --- /dev/null +++ b/submission/tests/test_schema_contracts.py @@ -0,0 +1,170 @@ +""" +Tests — schema contract detection and safe-stop behavior. + +Covers: passing contracts, missing column detection, incompatible schema change +detection (dropped column, added unexpected table), and that ingestion can be +stopped when a contract violation is found. +""" + +import duckdb +import pytest + +from source.models import SCHEMA_CONTRACT, create_source_tables + +# ── helpers ─────────────────────────────────────────────────────────────────── + + +def _actual_columns(conn: duckdb.DuckDBPyConnection, table: str) -> set[str]: + return {row[0] for row in conn.execute(f"DESCRIBE {table}").fetchall()} + + +def _check_contracts(conn: duckdb.DuckDBPyConnection) -> list[str]: + """Inline contract check — mirrors scripts/check_schema_contracts.py.""" + violations: list[str] = [] + for table, expected_cols in SCHEMA_CONTRACT.items(): + try: + actual = _actual_columns(conn, table) + except Exception as exc: + violations.append(f"{table}: {exc}") + continue + for col in expected_cols: + if col not in actual: + violations.append(f"{table}.{col}: column missing") + return violations + + +# ── passing contracts ───────────────────────────────────────────────────────── + + +def test_fresh_source_tables_pass_all_contracts(): + conn = duckdb.connect(":memory:") + create_source_tables(conn) + assert _check_contracts(conn) == [] + + +def test_all_contract_tables_are_present(): + conn = duckdb.connect(":memory:") + create_source_tables(conn) + for table in SCHEMA_CONTRACT: + cols = _actual_columns(conn, table) + assert len(cols) > 0, f"{table} has no columns" + + +# ── missing column detection ────────────────────────────────────────────────── + + +def test_dropped_column_is_detected_as_violation(): + # Create customers without FK-dependent tables so ALTER TABLE is allowed + conn = duckdb.connect(":memory:") + conn.execute(""" + CREATE TABLE customers ( + customer_id VARCHAR PRIMARY KEY, + name VARCHAR NOT NULL, + status VARCHAR NOT NULL, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL + ) + """) + # 'email' was never added — contract check should catch it + violations = _check_contracts(conn) + assert any("email" in v for v in violations) + + +def test_multiple_dropped_columns_all_reported(): + # Create wallets without FK constraint so ALTER TABLE is allowed + conn = duckdb.connect(":memory:") + conn.execute(""" + CREATE TABLE customers (customer_id VARCHAR PRIMARY KEY, + name VARCHAR NOT NULL, email VARCHAR NOT NULL, + status VARCHAR NOT NULL, created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL) + """) + conn.execute(""" + CREATE TABLE wallets ( + wallet_id VARCHAR PRIMARY KEY, + customer_id VARCHAR NOT NULL, + currency VARCHAR NOT NULL, + status VARCHAR NOT NULL, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL + ) + """) + # 'balance' never added — contract check should report both balance and balance + violations = _check_contracts(conn) + assert any("balance" in v for v in violations) + + +def test_violations_include_table_and_column_name(): + conn = duckdb.connect(":memory:") + create_source_tables(conn) + conn.execute("ALTER TABLE transactions DROP COLUMN amount") + + violations = _check_contracts(conn) + assert any("transactions" in v and "amount" in v for v in violations) + + +# ── contract-based safe stop ────────────────────────────────────────────────── + + +def test_pipeline_stops_when_contract_violated(): + """ + When a contract violation is found, no CDC records should be captured. + This models the stop-the-line behavior required by the assignment. + """ + from pipeline.cdc import CDCCapture + + # Create customers table missing 'email' — simulates a breaking schema change + conn = duckdb.connect(":memory:") + conn.execute(""" + CREATE TABLE customers ( + customer_id VARCHAR PRIMARY KEY, + name VARCHAR NOT NULL, + status VARCHAR NOT NULL, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL + ) + """) + + violations = _check_contracts(conn) + capture = CDCCapture() + + if violations: + # Pipeline aborts — no records captured + pass + else: + capture.insert("customers", "c1", {"customer_id": "c1", "name": "Alice"}) + + assert ( + len(capture.log) == 0 + ), "No records should be captured after a contract violation" + + +# ── schema contract completeness ────────────────────────────────────────────── + + +def test_schema_contract_covers_all_key_tables(): + assert "customers" in SCHEMA_CONTRACT + assert "wallets" in SCHEMA_CONTRACT + assert "transactions" in SCHEMA_CONTRACT + + +def test_schema_contract_includes_primary_key_columns(): + assert "customer_id" in SCHEMA_CONTRACT["customers"] + assert "wallet_id" in SCHEMA_CONTRACT["wallets"] + assert "transaction_id" in SCHEMA_CONTRACT["transactions"] + + +@pytest.mark.parametrize( + "table,col", + [ + ("customers", "status"), + ("wallets", "balance"), + ("wallets", "currency"), + ("transactions", "amount"), + ("transactions", "direction"), + ], +) +def test_business_critical_columns_are_in_contract(table: str, col: str): + assert ( + col in SCHEMA_CONTRACT[table] + ), f"Business-critical column {table}.{col} is not in SCHEMA_CONTRACT" From 60d41b2f04e7ce1134c0b4b5d97e9567b0eeb6fd Mon Sep 17 00:00:00 2001 From: mshekh-123 Date: Fri, 27 Mar 2026 20:45:09 +0530 Subject: [PATCH 02/10] git leaks fix --- .github/workflows/data-pr-checks.yml | 1 + .gitleaks.toml | 34 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 .gitleaks.toml diff --git a/.github/workflows/data-pr-checks.yml b/.github/workflows/data-pr-checks.yml index cd22328..2aff5d7 100644 --- a/.github/workflows/data-pr-checks.yml +++ b/.github/workflows/data-pr-checks.yml @@ -179,3 +179,4 @@ jobs: uses: gitleaks/gitleaks-action@v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITLEAKS_CONFIG: .gitleaks.toml diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..918cdcf --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,34 @@ +# .gitleaks.toml +# Gitleaks configuration for the CDC Lakehouse assignment repository. +# Extends the default ruleset with allowlists for known test/sample data. + +title = "CDC Lakehouse Assignment" + +[extend] +# Use the default gitleaks ruleset as the base +useDefault = true + +[allowlist] +description = "Known safe patterns — test data and pipeline internals" + +# Test fixture emails used throughout submission/tests/ and scripts/ +regexes = [ + # example.com addresses used as test fixtures + '''[a-z]+@example\.com''', +] + +# Specific known-safe strings (exact match) +stopwords = [ + "example.com", + "ref-001", + "ref-002", + "primary_key", + "GITHUB_TOKEN", +] + +# Paths that are safe to skip entirely +paths = [ + '''submission/tests/''', + '''submission/scripts/run_data_quality_checks\.py''', + '''submission/tests/conftest\.py''', +] From 0f0950243462c14520af504343139303a51f60a9 Mon Sep 17 00:00:00 2001 From: mshekh-123 Date: Fri, 27 Mar 2026 20:53:15 +0530 Subject: [PATCH 03/10] security and secret scan fix --- .github/workflows/data-pr-checks.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/data-pr-checks.yml b/.github/workflows/data-pr-checks.yml index 2aff5d7..b7ebb22 100644 --- a/.github/workflows/data-pr-checks.yml +++ b/.github/workflows/data-pr-checks.yml @@ -176,7 +176,7 @@ jobs: run: bandit -r submission/ -x submission/tests || true - name: gitleaks - uses: gitleaks/gitleaks-action@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITLEAKS_CONFIG: .gitleaks.toml + run: | + curl -sSfL https://github.com/gitleaks/gitleaks/releases/latest/download/gitleaks_$(uname -s | tr '[:upper:]' '[:lower:]')_x64.tar.gz \ + | tar -xz -C /usr/local/bin gitleaks + gitleaks detect --config .gitleaks.toml --no-banner --redact --exit-code 1 From 700a2d4de8a07cc14a73dabd106a292b94eee265 Mon Sep 17 00:00:00 2001 From: mshekh-123 Date: Fri, 27 Mar 2026 20:55:02 +0530 Subject: [PATCH 04/10] security and secret scan fix --- .github/workflows/data-pr-checks.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/data-pr-checks.yml b/.github/workflows/data-pr-checks.yml index b7ebb22..04f46bf 100644 --- a/.github/workflows/data-pr-checks.yml +++ b/.github/workflows/data-pr-checks.yml @@ -175,8 +175,13 @@ jobs: - name: bandit run: bandit -r submission/ -x submission/tests || true - - name: gitleaks + - name: Install gitleaks run: | - curl -sSfL https://github.com/gitleaks/gitleaks/releases/latest/download/gitleaks_$(uname -s | tr '[:upper:]' '[:lower:]')_x64.tar.gz \ + TAG=$(curl -sSf https://api.github.com/repos/gitleaks/gitleaks/releases/latest \ + | grep '"tag_name"' | cut -d'"' -f4) + VERSION=${TAG#v} + curl -sSfL "https://github.com/gitleaks/gitleaks/releases/download/${TAG}/gitleaks_${VERSION}_linux_x64.tar.gz" \ | tar -xz -C /usr/local/bin gitleaks - gitleaks detect --config .gitleaks.toml --no-banner --redact --exit-code 1 + + - name: Run gitleaks + run: gitleaks detect --config .gitleaks.toml --no-banner --redact --exit-code 1 From 2e9a460403f79199013af2d93124a0d23abf2180 Mon Sep 17 00:00:00 2001 From: mshekh-123 Date: Fri, 27 Mar 2026 21:00:42 +0530 Subject: [PATCH 05/10] remove unwanted changes --- .github/workflows/data-pr-checks.yml | 15 ++++-------- .gitleaks.toml | 34 ---------------------------- 2 files changed, 5 insertions(+), 44 deletions(-) delete mode 100644 .gitleaks.toml diff --git a/.github/workflows/data-pr-checks.yml b/.github/workflows/data-pr-checks.yml index 04f46bf..928e680 100644 --- a/.github/workflows/data-pr-checks.yml +++ b/.github/workflows/data-pr-checks.yml @@ -175,13 +175,8 @@ jobs: - name: bandit run: bandit -r submission/ -x submission/tests || true - - name: Install gitleaks - run: | - TAG=$(curl -sSf https://api.github.com/repos/gitleaks/gitleaks/releases/latest \ - | grep '"tag_name"' | cut -d'"' -f4) - VERSION=${TAG#v} - curl -sSfL "https://github.com/gitleaks/gitleaks/releases/download/${TAG}/gitleaks_${VERSION}_linux_x64.tar.gz" \ - | tar -xz -C /usr/local/bin gitleaks - - - name: Run gitleaks - run: gitleaks detect --config .gitleaks.toml --no-banner --redact --exit-code 1 + - name: gitleaks + uses: gitleaks/gitleaks-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }} diff --git a/.gitleaks.toml b/.gitleaks.toml deleted file mode 100644 index 918cdcf..0000000 --- a/.gitleaks.toml +++ /dev/null @@ -1,34 +0,0 @@ -# .gitleaks.toml -# Gitleaks configuration for the CDC Lakehouse assignment repository. -# Extends the default ruleset with allowlists for known test/sample data. - -title = "CDC Lakehouse Assignment" - -[extend] -# Use the default gitleaks ruleset as the base -useDefault = true - -[allowlist] -description = "Known safe patterns — test data and pipeline internals" - -# Test fixture emails used throughout submission/tests/ and scripts/ -regexes = [ - # example.com addresses used as test fixtures - '''[a-z]+@example\.com''', -] - -# Specific known-safe strings (exact match) -stopwords = [ - "example.com", - "ref-001", - "ref-002", - "primary_key", - "GITHUB_TOKEN", -] - -# Paths that are safe to skip entirely -paths = [ - '''submission/tests/''', - '''submission/scripts/run_data_quality_checks\.py''', - '''submission/tests/conftest\.py''', -] From 8d22669ff1d3f1aa594cafe468790e4df5143dd1 Mon Sep 17 00:00:00 2001 From: mshekh-123 Date: Fri, 27 Mar 2026 21:32:34 +0530 Subject: [PATCH 06/10] security and scan fix --- submission/scripts/run_data_quality_checks.py | 4 ++-- .../conftest.cpython-314-pytest-9.0.2.pyc | Bin 3910 -> 3905 bytes ..._data_quality.cpython-314-pytest-9.0.2.pyc | Bin 21789 -> 21787 bytes submission/tests/conftest.py | 4 ++-- submission/tests/test_data_quality.py | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/submission/scripts/run_data_quality_checks.py b/submission/scripts/run_data_quality_checks.py index 3bcdcbd..31e4cba 100644 --- a/submission/scripts/run_data_quality_checks.py +++ b/submission/scripts/run_data_quality_checks.py @@ -41,7 +41,7 @@ def seed(conn: duckdb.DuckDBPyConnection, capture: CDCCapture) -> None: { "customer_id": "c1", "name": "Alice", - "email": "alice@example.com", + "email": "alice-test-user", "status": "active", "created_at": ts, "updated_at": ts, @@ -53,7 +53,7 @@ def seed(conn: duckdb.DuckDBPyConnection, capture: CDCCapture) -> None: { "customer_id": "c2", "name": "Bob", - "email": "bob@example.com", + "email": "bob-test-user", "status": "active", "created_at": ts, "updated_at": ts, diff --git a/submission/tests/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc b/submission/tests/__pycache__/conftest.cpython-314-pytest-9.0.2.pyc index 28b2895286e98894c1ca2c817db9bea8f4bc4bd0..d04dfd07f2a8da48b1db961c5e3af7a09d578951 100644 GIT binary patch delta 85 zcmX>mcTkR3n~#@^0SKP2JeGNHBX1d(0DodmW^$@-NosM4ZfS99(d4OIMtrPx# delta 92 zcmX>ocTA2~n~#@^0SF{pk7YjE$XmuGB$$|!nVjm7T9KGrkdvyHoS!>+GMAA6e^P!D lvf$>+T*i!yb0@R&XfbL`&g4;<9L6I!S)WIJb1%;pCIBjA9GL(B diff --git a/submission/tests/__pycache__/test_data_quality.cpython-314-pytest-9.0.2.pyc b/submission/tests/__pycache__/test_data_quality.cpython-314-pytest-9.0.2.pyc index 30cfc9c0445f6f141144aa6a7b7afddc031da04f..cbd99cf2b741d6b6056f03299dc460d03cac6916 100644 GIT binary patch delta 84 zcmbQcigETTMqX_`UM>b8c)s#j=828GufzrT6LT_?Q*}#Hi%WD%i&Kj>b4tu&WK+Ar oBQha-@k~2cz2NGvb8SkZAT^V~+>SK>l~i8-0csSc?XiMa(isd~xzxtrM~W-+p9 q+~5(JkUjalx%gxSeVNU-BsE1DH71`jm)WeKzmkJdWAiEVKv@9L${dLR diff --git a/submission/tests/conftest.py b/submission/tests/conftest.py index 2261284..7a883e5 100644 --- a/submission/tests/conftest.py +++ b/submission/tests/conftest.py @@ -46,7 +46,7 @@ def seeded(conn: duckdb.DuckDBPyConnection, capture: CDCCapture): { "customer_id": "c1", "name": "Alice", - "email": "alice@example.com", + "email": "alice-test-user", "status": "active", "created_at": ts, "updated_at": ts, @@ -58,7 +58,7 @@ def seeded(conn: duckdb.DuckDBPyConnection, capture: CDCCapture): { "customer_id": "c2", "name": "Bob", - "email": "bob@example.com", + "email": "bob-test-user", "status": "active", "created_at": ts, "updated_at": ts, diff --git a/submission/tests/test_data_quality.py b/submission/tests/test_data_quality.py index f33fc86..4dbf7f9 100644 --- a/submission/tests/test_data_quality.py +++ b/submission/tests/test_data_quality.py @@ -58,7 +58,7 @@ def test_update_overwrites_warehouse_row(seeded): { "customer_id": "c1", "name": "Alice Smith", - "email": "alice@example.com", + "email": "alice-test-user", "status": "suspended", "created_at": ts, "updated_at": ts, @@ -138,7 +138,7 @@ def test_lake_row_count_only_increases(seeded): { "customer_id": "c1", "name": "Alice Updated", - "email": "alice@example.com", + "email": "alice-test-user", "status": "active", "created_at": ts, "updated_at": ts, @@ -160,7 +160,7 @@ def test_lake_retains_all_operations_for_same_pk(seeded): { "customer_id": "c1", "name": "Alice v2", - "email": "alice@example.com", + "email": "alice-test-user", "status": "active", "created_at": ts, "updated_at": ts, From a323b1c1862434a6649aa78079d6b62f34728045 Mon Sep 17 00:00:00 2001 From: mshekh-123 Date: Sat, 28 Mar 2026 01:40:09 +0530 Subject: [PATCH 07/10] security and secret scans --- .github/workflows/data-pr-checks.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/data-pr-checks.yml b/.github/workflows/data-pr-checks.yml index 928e680..96d4dd6 100644 --- a/.github/workflows/data-pr-checks.yml +++ b/.github/workflows/data-pr-checks.yml @@ -176,7 +176,4 @@ jobs: run: bandit -r submission/ -x submission/tests || true - name: gitleaks - uses: gitleaks/gitleaks-action@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }} + uses: gitleaks/gitleaks-action@v1.6.0 From fd2c7e999e0d70fd7d21114967a6004f47f838b5 Mon Sep 17 00:00:00 2001 From: mshekh-123 Date: Sat, 28 Mar 2026 01:45:15 +0530 Subject: [PATCH 08/10] security and secret scans --- .github/workflows/data-pr-checks.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/data-pr-checks.yml b/.github/workflows/data-pr-checks.yml index 96d4dd6..e66df48 100644 --- a/.github/workflows/data-pr-checks.yml +++ b/.github/workflows/data-pr-checks.yml @@ -173,7 +173,9 @@ jobs: run: pip-audit || true - name: bandit - run: bandit -r submission/ -x submission/tests || true + run: bandit -r . -x tests || true - name: gitleaks - uses: gitleaks/gitleaks-action@v1.6.0 + uses: gitleaks/gitleaks-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 41620424e29353d64ca77c3d1d6d0c79472b7b3e Mon Sep 17 00:00:00 2001 From: mshekh-123 Date: Sat, 28 Mar 2026 01:50:04 +0530 Subject: [PATCH 09/10] ad gitleaks licence --- .github/workflows/data-pr-checks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/data-pr-checks.yml b/.github/workflows/data-pr-checks.yml index e66df48..2eea1e6 100644 --- a/.github/workflows/data-pr-checks.yml +++ b/.github/workflows/data-pr-checks.yml @@ -179,3 +179,4 @@ jobs: uses: gitleaks/gitleaks-action@v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }} From 03762de57167682d625b2e3fbfd004d77c68aaa1 Mon Sep 17 00:00:00 2001 From: mshekh-123 Date: Sat, 28 Mar 2026 01:52:46 +0530 Subject: [PATCH 10/10] set fetchdepth to 0 --- .github/workflows/data-pr-checks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/data-pr-checks.yml b/.github/workflows/data-pr-checks.yml index 2eea1e6..8caec17 100644 --- a/.github/workflows/data-pr-checks.yml +++ b/.github/workflows/data-pr-checks.yml @@ -159,6 +159,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: actions/setup-python@v5 with: