diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 3b1adfd..0000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd5d9a9..66be289 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: push: pull_request: branches: [dev] - types: [opened, reopened] + types: [opened, reopened, closed] permissions: contents: read @@ -32,18 +32,6 @@ jobs: - name: Build and run services run: docker compose up db backend nginx -d --build --wait --wait-timeout 60 - - name: Install goose - run: docker exec backend go install github.com/pressly/goose/v3/cmd/goose@latest - - - name: Migration db - run: docker exec backend sh ./src/scripts/migrations.test.sh --up - - - name: Check migrations - run: docker exec backend sh ./src/scripts/migrations.test.sh --status - - - name: Testing - run: docker exec backend go test -v -short ./src/... - - name: Shutdown Docker Compose if: always() run: docker compose down -v --rmi all --remove-orphans diff --git a/go.mod b/go.mod index 1a9a732..36482f1 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,9 @@ module backend go 1.26.1 require ( - github.com/PuerkitoBio/goquery v1.12.0 github.com/go-playground/validator/v10 v10.30.2 github.com/gofiber/contrib/v3/swaggo v1.0.2 github.com/gofiber/fiber/v3 v3.1.0 - github.com/jinzhu/copier v0.4.0 - github.com/joho/godotenv v1.5.1 - golang.org/x/sync v0.20.0 - golang.org/x/text v0.36.0 gorm.io/driver/postgres v1.6.0 gorm.io/gorm v1.31.1 ) @@ -18,7 +13,6 @@ require ( require ( github.com/KyleBanks/depth v1.2.1 // indirect github.com/andybalholm/brotli v1.2.1 // indirect - github.com/andybalholm/cascadia v1.3.3 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/go-openapi/jsonpointer v0.23.0 // indirect github.com/go-openapi/jsonreference v0.21.5 // indirect @@ -32,6 +26,7 @@ require ( github.com/go-openapi/swag/yamlutils v0.26.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/gofiber/fiber/v2 v2.52.13 // indirect github.com/gofiber/schema v1.7.0 // indirect github.com/gofiber/utils/v2 v2.0.3 // indirect github.com/google/uuid v1.6.0 // indirect @@ -39,14 +34,18 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgx/v5 v5.6.0 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jinzhu/copier v0.4.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/joho/godotenv v1.5.1 // indirect github.com/klauspost/compress v1.18.5 // indirect github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.21 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect github.com/philhofer/fwd v1.2.0 // indirect + github.com/rivo/uniseg v0.2.0 // indirect github.com/swaggo/files/v2 v2.0.2 // indirect github.com/swaggo/swag v1.16.6 // indirect github.com/tinylib/msgp v1.6.4 // indirect @@ -56,6 +55,8 @@ require ( golang.org/x/crypto v0.50.0 // indirect golang.org/x/mod v0.35.0 // indirect golang.org/x/net v0.53.0 // indirect + golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect golang.org/x/tools v0.44.0 // indirect ) diff --git a/go.sum b/go.sum index 5f6e96e..83e21b4 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,7 @@ github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo= -github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= -github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -51,13 +47,14 @@ github.com/go-playground/validator/v10 v10.30.2 h1:JiFIMtSSHb2/XBUbWM4i/MpeQm9ZK github.com/go-playground/validator/v10 v10.30.2/go.mod h1:mAf2pIOVXjTEBrwUMGKkCWKKPs9NheYGabeB04txQSc= github.com/gofiber/contrib/v3/swaggo v1.0.2 h1:we2p7xPry097dahzvYobfAaILw6e7t3mdsrqlipmG1Q= github.com/gofiber/contrib/v3/swaggo v1.0.2/go.mod h1:5e6A42JPRG3BrY3BpGNEtrpr57+PvUmG+puWdXQzF8k= +github.com/gofiber/fiber/v2 v2.52.13 h1:TOKP64iqC9b5P49VrBW5tHhUOvDyrtJ0xePEfzJbCbk= +github.com/gofiber/fiber/v2 v2.52.13/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw= github.com/gofiber/fiber/v3 v3.1.0 h1:1p4I820pIa+FGxfwWuQZ5rAyX0WlGZbGT6Hnuxt6hKY= github.com/gofiber/fiber/v3 v3.1.0/go.mod h1:n2nYQovvL9z3Too/FGOfgtERjW3GQcAUqgfoezGBZdU= github.com/gofiber/schema v1.7.0 h1:yNM+FNRZjyYEli9Ey0AXRBrAY9jTnb+kmGs3lJGPvKg= github.com/gofiber/schema v1.7.0/go.mod h1:A/X5Ffyru4p9eBdp99qu+nzviHzQiZ7odLT+TwxWhbk= github.com/gofiber/utils/v2 v2.0.3 h1:qJyfS/t7s7Z4+/zlU1i1pafYNP2+xLupVPgkW8ce1uI= github.com/gofiber/utils/v2 v2.0.3/go.mod h1:GGERKU3Vhj5z6hS8YKvxL99A54DjOvTFZ0cjZnG4Lj4= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -90,11 +87,15 @@ github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHP github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs= github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/shamaton/msgpack/v3 v3.1.0 h1:jsk0vEAqVvvS9+fTZ5/EcQ9tz860c9pWxJ4Iwecz8gU= @@ -118,86 +119,22 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/migration/20260429224953_create_async_requests_table.sql b/migration/20260429224953_create_async_requests_table.sql deleted file mode 100644 index a285c4b..0000000 --- a/migration/20260429224953_create_async_requests_table.sql +++ /dev/null @@ -1,18 +0,0 @@ --- +goose Up - -CREATE TABLE IF NOT EXISTS async_requests ( - id SERIAL PRIMARY KEY, - hash TEXT UNIQUE, - code INTEGER, - status INTEGER, - request_type INTEGER, - attempts INTEGER, - result TEXT, - error TEXT, - deadline_at TIMESTAMP, - created_at TIMESTAMP, - updated_at TIMESTAMP -); - --- +goose Down -DROP TABLE IF EXISTS async_requests; diff --git a/src/.DS_Store b/src/.DS_Store deleted file mode 100644 index 6c8cd7a..0000000 Binary files a/src/.DS_Store and /dev/null differ diff --git a/src/api/public/openapi.yaml b/src/api/public/openapi.yaml index 7a2836a..8290363 100644 --- a/src/api/public/openapi.yaml +++ b/src/api/public/openapi.yaml @@ -4,7 +4,7 @@ info: description: | С помощью этого API можно выгрузить данные Росстата и прочие рассчитанные данные для дашбордов, - запросить прогноз по численности населения + запросить прогноз по численнсоти населения и AI справку. version: 1.0.0 paths: @@ -25,13 +25,6 @@ paths: type: array items: $ref: '#/components/schemas/FederalSubject' - example: - - name: "Алтайский край" - code: 1 - upper_municipalities: [] - - name: "Краснодарский край" - code: 3 - upper_municipalities: [] /api/v1/rosstat: get: @@ -47,7 +40,7 @@ paths: type: string enum: [year, population, birth, death, arrival, departure, age, male, female, land_area, avg_salary, - medical_facilities, schools, housing_commissioned] + medicial_facilities, schools, housing_commissioned] style: form explode: true example: ["population", "birth", "death"] @@ -63,7 +56,7 @@ paths: type: integer style: form explode: true - example: [1, 3, 4] + example: [1, 2, 3] responses: '200': description: OK @@ -71,23 +64,6 @@ paths: application/json: schema: $ref: '#/components/schemas/RosstatResponse' - example: - - code: 1 - by_year: - - year: 2020 - population: 2300000 - birth: 20000 - death: 25000 - - year: 2021 - population: 2310000 - birth: 19000 - death: 24000 - - code: 2 - by_year: - - year: 2020 - population: 146000000 - birth: 1400000 - death: 1800000 '400': description: | Отсутствует обязательный параметр @@ -96,8 +72,6 @@ paths: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - example: - error: "field 'fields' is required" components: schemas: @@ -107,8 +81,6 @@ components: error: type: string description: Описание ошибки - example: - error: "field 'fields' is required" LowerMunicipality: type: object @@ -123,9 +95,6 @@ components: required: - name - code - example: - name: "Центральный район" - code: 1000 UpperMunicipality: type: object @@ -146,14 +115,6 @@ components: - name - code - lower_municipalities - example: - name: "г. Барнаул" - code: 100 - lower_municipalities: - - name: "Центральный район" - code: 1000 - - name: "Ленинский район" - code: 1001 FederalSubject: type: object @@ -174,13 +135,6 @@ components: - name - code - upper_municipalities - example: - name: "Алтайский край" - code: 1 - upper_municipalities: - - name: "г. Барнаул" - code: 100 - lower_municipalities: [] RosstatByAge: type: object @@ -198,10 +152,6 @@ components: - age - male - female - example: - age: 25 - male: 50000 - female: 48000 RosstatByYear: type: object @@ -212,74 +162,54 @@ components: population: type: integer description: Численность населения за указанный год. - nullable: true birth: type: integer description: Численность рожденных за указанный год. - nullable: true death: type: integer description: Численность умерших за указанный год. - nullable: true arrival: type: integer description: Численность прибывших за указанный год. - nullable: true departure: type: integer description: Численность выбывших за указанный год. - nullable: true male: type: integer description: Численность мужчин за указанный год. - nullable: true female: type: integer description: Численность женщин за указанный год. - nullable: true land_area: type: number format: float description: Земельная площадь субъекта РФ или муниципалитета за указанный год. - nullable: true avg_salary: type: number format: float description: | Средняя заработная плата по субъекту РФ или муниципалитету за указанный год. - nullable: true - medical_facilities: + medicial_facilities: type: integer description: | Число медицинских учреждений в субъекте РФ или муниципалитете за указанный год. - nullable: true schools: type: integer description: | Число школ в субъекте РФ или муниципалитете за указанный год. - nullable: true housing_commissioned: type: integer description: | Число введенных в эксплуатацию жилых объектов в субъекте РФ или муниципалитете за указанный год. - nullable: true by_age: type: array description: Численность мужчин и женщин по возрасту за год. - nullable: true items: $ref: '#/components/schemas/RosstatByAge' - example: - year: 2020 - population: 2300000 - birth: 20000 - death: 25000 - arrival: 15000 - departure: 12000 Rosstat: type: object @@ -298,32 +228,8 @@ components: required: - code - by_year - example: - code: 1 - by_year: - - year: 2020 - population: 2300000 - birth: 20000 - death: 25000 - - year: 2021 - population: 2310000 - birth: 19000 - death: 24000 RosstatResponse: type: array items: $ref: '#/components/schemas/Rosstat' - example: - - code: 1 - by_year: - - year: 2020 - population: 2300000 - birth: 20000 - death: 25000 - - code: 0 - by_year: - - year: 2020 - population: 146000000 - birth: 1400000 - death: 1800000 \ No newline at end of file diff --git a/src/cmd/async_backend/main.go b/src/cmd/async_backend/main.go index 06def20..fe93b8f 100644 --- a/src/cmd/async_backend/main.go +++ b/src/cmd/async_backend/main.go @@ -1,4 +1,4 @@ -package main +package async_backend import ( "backend/src/internal/async/dispatcher" diff --git a/src/cmd/parser/main.go b/src/cmd/parser/main.go deleted file mode 100644 index ebf78ec..0000000 --- a/src/cmd/parser/main.go +++ /dev/null @@ -1,132 +0,0 @@ -package main - -import ( - "context" - "fmt" - "log" - - "backend/src/internal/config" - "backend/src/internal/db/postgres" - "backend/src/internal/domain" - "backend/src/internal/repository/impl" - "backend/src/pkg/parser/rosstat/rosstat" - - "github.com/joho/godotenv" -) - -func main() { - godotenv.Load(".env") - - config := config.Load() - conn := postgres.NewPostgresConnection(config.GetDBDSN()) - - geoRepo := impl.NewGeoRepository() - rosstatRepo := impl.NewRosstatRepository() - - parser := rosstat.NewRosstatParser() - - ctx := context.Background() - parsedSlice, err := parser.Parse(ctx) - - if err != nil { - panic(err) - } - - fmt.Printf("Parsed count: %d\n", len(parsedSlice)) - - GEOs := make([]*domain.Geo, 0, len(parsedSlice)) - rosstats := make([]*domain.Rosstat, len(parsedSlice)) - - codeMap := make(map[int]bool) - - fmt.Printf("Grouping...\n") - - for index, parsed := range parsedSlice { - if _, exists := codeMap[parsed.Code]; !exists { - codeMap[parsed.Code] = true - GEOs = append(GEOs, &domain.Geo{ - Code: parsed.Code, - ParentCode: &parsed.ParentCode, - Name: parsed.Name, - Level: 2, - }) - } - - rosstats[index] = &domain.Rosstat{ - Code: parsed.Code, - Year: parsed.Year, - PopulationAmount: parsed.Population, - } - } - - fmt.Printf("Grouped\n") - - batchSize := 50 - startIndex := 0 - endIndex := batchSize - - fmt.Printf("Upsert batch geo higher...\n") - for { - if endIndex >= len(GEOs) { - endIndex = len(GEOs) - } - fmt.Printf("Upsert batch geo higher [%d:%d]...\n", startIndex, endIndex) - if err := geoRepo.UpsertBatch(conn, GEOs[startIndex:endIndex]); err != nil { - log.Printf("ERROR [higherGEOs]: %s\n", err.Error()) - return - } - - if endIndex >= len(GEOs) { - break - } - - startIndex = endIndex - endIndex += batchSize - - if endIndex >= len(GEOs) { - endIndex = len(GEOs) - } - } - - startIndex = 0 - endIndex = batchSize - - fmt.Printf("Upsert batch rosstat...\n") - for { - if endIndex >= len(rosstats) { - endIndex = len(rosstats) - } - fmt.Printf("Upsert batch rosstat [%d:%d]...\n", startIndex, endIndex) - if err := rosstatRepo.UpsertBatch(conn, rosstats[startIndex:endIndex]); err != nil { - log.Printf("ERROR [lowerGEOs]: %s\n", err.Error()) - return - } - - if endIndex >= len(rosstats) { - break - } - - startIndex = endIndex - endIndex += batchSize - - if endIndex >= len(rosstats) { - endIndex = len(rosstats) - } - } -} - -func getSubjectCode(code int) int { - extendedSubjectCode := code / 100000 - subjectCode := code / 1000000 - - switch extendedSubjectCode { - case 118: - subjectCode = extendedSubjectCode - case 718: - subjectCode = extendedSubjectCode - case 719: - subjectCode = extendedSubjectCode - } - - return subjectCode -} diff --git a/src/internal/app/app.go b/src/internal/app/app.go index 2aa9e89..5faf136 100644 --- a/src/internal/app/app.go +++ b/src/internal/app/app.go @@ -38,15 +38,15 @@ func Run() { // CORS middleware app.Use(func(c fiber.Ctx) error { - c.Set("Access-Control-Allow-Origin", "https://midray.ru") - c.Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") - c.Set("Access-Control-Allow-Headers", "Accept, Content-Type, Authorization") + c.Set("Access-Control-Allow-Origin", "https://midray.ru") + c.Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") + c.Set("Access-Control-Allow-Headers", "Accept, Content-Type, Authorization") - if c.Method() == fiber.MethodOptions { - return c.SendStatus(fiber.StatusNoContent) - } + if c.Method() == fiber.MethodOptions { + return c.SendStatus(fiber.StatusNoContent) + } - return c.Next() + return c.Next() }) app.Get("/ping", health.PingHandler) @@ -57,9 +57,5 @@ func Run() { app.Get("/openapi.yaml", api.OpenapiYamlHandler) app.Get("/api/*", api.ApiHandler()) - app.Get("/api/v1/report/:code", middleware.Adapt(public.GetReportAsyncHandler, serviceProvider)) - app.Get("/api/v1/report/:code/request/:hash", middleware.Adapt(public.GetRequestStatusHandler, serviceProvider)) - app.Post("/api/v1/report", middleware.Adapt(public.PostReportAsyncHandler, serviceProvider)) - log.Fatal(app.Listen(":80")) } diff --git a/src/internal/async/request/request.go b/src/internal/async/request/request.go index b3650b6..2aa00de 100644 --- a/src/internal/async/request/request.go +++ b/src/internal/async/request/request.go @@ -2,7 +2,4 @@ package request const ( YDISKPARSE = 100 - - AIREPORT = 200 - ROSSTAT_PARSE = 100 ) diff --git a/src/internal/async/worker/yadisk_parse/worker.go b/src/internal/async/worker/yadisk_parse/worker.go new file mode 100644 index 0000000..2a66c76 --- /dev/null +++ b/src/internal/async/worker/yadisk_parse/worker.go @@ -0,0 +1,475 @@ +package yadisk_parse + +import ( + "context" + "fmt" + "log" + + "backend/src/internal/async/semaphore" + "backend/src/internal/async/status" + "backend/src/internal/async/worker/config" + "backend/src/internal/db/abstract" + "backend/src/internal/domain" + + parser "backend/src/pkg/parser/rosstat/domain" + "backend/src/pkg/parser/rosstat/yadisk" +) + +const batchSize = 2000 + +func YadiskParseWorker( + ctx context.Context, + ch chan status.CompletionStatus, + sem semaphore.Semaphore, + conn abstract.IDBConnection, + workerConfig config.WorkerConfig, + requestId int, + param int, +) { + defer sem.Release() + parser := yadisk.NewYadiskRosstatParser() + + log.Printf("Parsing...\n") + rosstatParsedSlice, err := parser.Parse(ctx) + if err != nil { + log.Printf("Request %d failed with error: %v\n", requestId, err) + ch <- status.CompletionStatus{RequestId: requestId, Err: err} + return + } + + log.Printf("Upsert GEO...\n") + if err := batchUpsertGeo(ctx, conn, workerConfig, rosstatParsedSlice); err != nil { + log.Printf("Request %d failed in upsertGeo: %v\n", requestId, err) + ch <- status.CompletionStatus{RequestId: requestId, Err: err} + return + } + + log.Printf("Aggregate by subject...\n") + subjectAggs, err := aggregateBySubject(rosstatParsedSlice) + if err != nil { + log.Printf("Request %d failed in aggregateBySubject: %v\n", requestId, err) + ch <- status.CompletionStatus{RequestId: requestId, Err: err} + return + } + + log.Printf("Aggregate RF...\n") + rfAgg, err := aggregateRF(subjectAggs) + if err != nil { + log.Printf("Request %d failed in aggregateRF: %v\n", requestId, err) + ch <- status.CompletionStatus{RequestId: requestId, Err: err} + return + } + + log.Printf("Upsert Rosstat with age...\n") + + var allRosstat []*domain.Rosstat + var allAge []*domain.RosstatByAge + + for _, p := range rosstatParsedSlice { + rosstat := &domain.Rosstat{ + Code: p.Code, + Year: p.Year, + PopulationAmount: p.Population, + BirthAmount: p.Birth, + DeathAmount: p.Death, + ArrivalAmount: p.Arrival, + DepartureAmount: p.Departure, + MaleAmount: p.Male, + FemaleAmount: p.Female, + LandArea: p.LandArea, + AvgSalary: p.AverageSalary, + MedicalFacilities: p.MedicialFacilities, + SchoolsCount: p.Schools, + HousingCommissioned: p.HousingCommissioned, + } + allRosstat = append(allRosstat, rosstat) + } + + subjectRosstat := make(map[int]*domain.Rosstat) + for subjCode, agg := range subjectAggs { + rosstat := &domain.Rosstat{ + Code: subjCode, + Year: agg.Year, + PopulationAmount: agg.Population, + BirthAmount: agg.Birth, + DeathAmount: agg.Death, + ArrivalAmount: agg.Arrival, + DepartureAmount: agg.Departure, + MaleAmount: agg.Male, + FemaleAmount: agg.Female, + LandArea: agg.LandArea, + AvgSalary: agg.AvgSalary, + MedicalFacilities: agg.MedicalFacilities, + SchoolsCount: agg.Schools, + HousingCommissioned: agg.HousingCommissioned, + } + allRosstat = append(allRosstat, rosstat) + subjectRosstat[subjCode] = rosstat + } + + rfRosstat := &domain.Rosstat{ + Code: 0, + Year: rfAgg.Year, + PopulationAmount: rfAgg.Population, + BirthAmount: rfAgg.Birth, + DeathAmount: rfAgg.Death, + ArrivalAmount: rfAgg.Arrival, + DepartureAmount: rfAgg.Departure, + MaleAmount: rfAgg.Male, + FemaleAmount: rfAgg.Female, + LandArea: rfAgg.LandArea, + AvgSalary: rfAgg.AvgSalary, + MedicalFacilities: rfAgg.MedicalFacilities, + SchoolsCount: rfAgg.Schools, + HousingCommissioned: rfAgg.HousingCommissioned, + } + allRosstat = append(allRosstat, rfRosstat) + + if err := batchUpsertRosstat(ctx, conn, workerConfig, allRosstat); err != nil { + log.Printf("Request %d failed during batch upsert Rosstat: %v\n", requestId, err) + ch <- status.CompletionStatus{RequestId: requestId, Err: err} + return + } + + codesSet := make(map[int]bool) + for _, r := range allRosstat { + codesSet[r.Code] = true + } + codes := make([]int, 0, len(codesSet)) + for c := range codesSet { + codes = append(codes, c) + } + rosstatList, err := workerConfig.RosstatRepository.GetRosstatByCodes(conn, codes) + if err != nil { + log.Printf("Request %d failed to fetch Rosstat IDs: %v\n", requestId, err) + ch <- status.CompletionStatus{RequestId: requestId, Err: err} + return + } + rosstatIDMap := make(map[[2]int]int) + for _, r := range rosstatList { + key := [2]int{r.Code, r.Year} + rosstatIDMap[key] = r.ID + } + + for _, p := range rosstatParsedSlice { + key := [2]int{p.Code, p.Year} + rosstatID, ok := rosstatIDMap[key] + if !ok { + log.Printf("Rosstat ID not found for code=%d year=%d\n", p.Code, p.Year) + continue + } + for _, a := range p.ByAge { + allAge = append(allAge, &domain.RosstatByAge{ + RosstatID: rosstatID, + Age: a.Age, + MaleAmount: a.MaleAmount, + FemaleAmount: a.FemaleAmount, + }) + } + } + for subjCode, agg := range subjectAggs { + key := [2]int{subjCode, agg.Year} + rosstatID, ok := rosstatIDMap[key] + if !ok { + log.Printf("Rosstat ID not found for subject code=%d year=%d\n", subjCode, agg.Year) + continue + } + for _, a := range agg.ByAge { + allAge = append(allAge, &domain.RosstatByAge{ + RosstatID: rosstatID, + Age: a.Age, + MaleAmount: a.MaleAmount, + FemaleAmount: a.FemaleAmount, + }) + } + } + { + key := [2]int{0, rfAgg.Year} + rosstatID, ok := rosstatIDMap[key] + if !ok { + log.Printf("Rosstat ID not found for RF year=%d\n", rfAgg.Year) + } else { + for _, a := range rfAgg.ByAge { + allAge = append(allAge, &domain.RosstatByAge{ + RosstatID: rosstatID, + Age: a.Age, + MaleAmount: a.MaleAmount, + FemaleAmount: a.FemaleAmount, + }) + } + } + } + + if err := batchUpsertRosstatAge(ctx, conn, workerConfig, allAge); err != nil { + log.Printf("Request %d failed during batch upsert RosstatByAge: %v\n", requestId, err) + ch <- status.CompletionStatus{RequestId: requestId, Err: err} + return + } + + ch <- status.CompletionStatus{RequestId: requestId, Err: nil} +} + +func batchUpsertGeo( + ctx context.Context, + conn abstract.IDBConnection, + workerConfig config.WorkerConfig, + parsedSlice parser.RosstatParsedSlice, +) error { + if err := ctx.Err(); err != nil { + return err + } + seen := make(map[int]bool) + var allGeo []*domain.Geo + + for _, p := range parsedSlice { + if seen[p.Code] { + continue + } + seen[p.Code] = true + level, parentCode := municipalLevelAndParent(p.Code, p.SubjectCode) + geo := &domain.Geo{ + Code: p.Code, + ParentCode: parentCode, + Name: "", + Level: level, + } + allGeo = append(allGeo, geo) + } + + for i := 0; i < len(allGeo); i += batchSize { + end := i + batchSize + if end > len(allGeo) { + end = len(allGeo) + } + batch := allGeo[i:end] + if err := workerConfig.GeoRepository.UpsertBatch(conn, batch); err != nil { + return fmt.Errorf("geo upsert batch: %w", err) + } + } + return nil +} + +func aggregateBySubject(parsed parser.RosstatParsedSlice) (map[int]*aggregatedData, error) { + byKey := make(map[int]*aggregatedData) + + for _, p := range parsed { + subj := p.SubjectCode + agg, ok := byKey[subj] + if !ok { + agg = &aggregatedData{ + Year: p.Year, + ageSums: make(map[int]*ageSum), + } + byKey[subj] = agg + } + + addInt(&agg.Population, p.Population) + addInt(&agg.Birth, p.Birth) + addInt(&agg.Death, p.Death) + addInt(&agg.Arrival, p.Arrival) + addInt(&agg.Departure, p.Departure) + addInt(&agg.Male, p.Male) + addInt(&agg.Female, p.Female) + addInt(&agg.LandArea, nil) + addInt(&agg.MedicalFacilities, p.MedicialFacilities) + addInt(&agg.Schools, p.Schools) + addInt(&agg.HousingCommissioned, p.HousingCommissioned) + + if p.AverageSalary != nil && p.Population != nil { + agg.sumSalaryWeighted += *p.AverageSalary * float64(*p.Population) + agg.totalPopForSalary += *p.Population + } + + for _, a := range p.ByAge { + sum := agg.ageSums[a.Age] + if sum == nil { + sum = &ageSum{} + agg.ageSums[a.Age] = sum + } + sum.male += a.MaleAmount + sum.female += a.FemaleAmount + } + } + + for _, agg := range byKey { + if agg.totalPopForSalary > 0 { + avg := agg.sumSalaryWeighted / float64(agg.totalPopForSalary) + agg.AvgSalary = &avg + } + agg.ByAge = sortedAgeSlice(agg.ageSums) + agg.ageSums = nil + } + + return byKey, nil +} + +func aggregateRF(subjectAggs map[int]*aggregatedData) (*aggregatedData, error) { + rf := &aggregatedData{ + ageSums: make(map[int]*ageSum), + } + + for _, agg := range subjectAggs { + if rf.Year == 0 { + rf.Year = agg.Year + } + + addInt(&rf.Population, agg.Population) + addInt(&rf.Birth, agg.Birth) + addInt(&rf.Death, agg.Death) + addInt(&rf.Arrival, agg.Arrival) + addInt(&rf.Departure, agg.Departure) + addInt(&rf.Male, agg.Male) + addInt(&rf.Female, agg.Female) + addInt(&rf.LandArea, nil) + addInt(&rf.MedicalFacilities, agg.MedicalFacilities) + addInt(&rf.Schools, agg.Schools) + addInt(&rf.HousingCommissioned, agg.HousingCommissioned) + + if agg.AvgSalary != nil && agg.Population != nil { + rf.sumSalaryWeighted += *agg.AvgSalary * float64(*agg.Population) + rf.totalPopForSalary += *agg.Population + } + + for _, a := range agg.ByAge { + sum := rf.ageSums[a.Age] + if sum == nil { + sum = &ageSum{} + rf.ageSums[a.Age] = sum + } + sum.male += a.MaleAmount + sum.female += a.FemaleAmount + } + } + + if rf.totalPopForSalary > 0 { + avg := rf.sumSalaryWeighted / float64(rf.totalPopForSalary) + rf.AvgSalary = &avg + } + rf.ByAge = sortedAgeSlice(rf.ageSums) + rf.ageSums = nil + return rf, nil +} + +func batchUpsertRosstat( + ctx context.Context, + conn abstract.IDBConnection, + workerConfig config.WorkerConfig, + rosstatList []*domain.Rosstat, +) error { + if err := ctx.Err(); err != nil { + return err + } + for i := 0; i < len(rosstatList); i += batchSize { + end := i + batchSize + if end > len(rosstatList) { + end = len(rosstatList) + } + batch := rosstatList[i:end] + if err := workerConfig.RosstatRepository.UpsertBatch(conn, batch); err != nil { + return fmt.Errorf("rosstat upsert batch: %w", err) + } + } + return nil +} + +func batchUpsertRosstatAge( + ctx context.Context, + conn abstract.IDBConnection, + workerConfig config.WorkerConfig, + ageList []*domain.RosstatByAge, +) error { + if err := ctx.Err(); err != nil { + return err + } + for i := 0; i < len(ageList); i += batchSize { + end := i + batchSize + if end > len(ageList) { + end = len(ageList) + } + batch := ageList[i:end] + if err := workerConfig.RosstatAgeRepository.UpsertBatch(conn, batch); err != nil { + return fmt.Errorf("rosstat age upsert batch: %w", err) + } + } + return nil +} + +func municipalLevelAndParent(code int, subjectCode int) (int, *int) { + var level, parentCode int + + switch code % 1000 { + case 0: + level = 2 + parentCode = subjectCode + default: + level = 3 + parentCode = (code / 1000) * 1000 + } + + return level, &parentCode +} + +type aggregatedData struct { + Year int + Population *int + Birth *int + Death *int + Arrival *int + Departure *int + Male *int + Female *int + LandArea *int + AvgSalary *float64 + MedicalFacilities *int + Schools *int + HousingCommissioned *int + ByAge []*parser.RosstatAgeParsed + + sumSalaryWeighted float64 + totalPopForSalary int + ageSums map[int]*ageSum +} + +type ageSum struct { + male int + female int +} + +func addInt(dst **int, src *int) { + if src == nil { + return + } + if *dst == nil { + val := 0 + *dst = &val + } + **dst += *src +} + +func sortedAgeSlice(m map[int]*ageSum) []*parser.RosstatAgeParsed { + if len(m) == 0 { + return nil + } + ages := make([]int, 0, len(m)) + for a := range m { + ages = append(ages, a) + } + for i := 0; i < len(ages); i++ { + for j := i + 1; j < len(ages); j++ { + if ages[j] < ages[i] { + ages[i], ages[j] = ages[j], ages[i] + } + } + } + + result := make([]*parser.RosstatAgeParsed, len(ages)) + for i, age := range ages { + sum := m[age] + result[i] = &parser.RosstatAgeParsed{ + Age: age, + MaleAmount: sum.male, + FemaleAmount: sum.female, + } + } + return result +} diff --git a/src/internal/async/worker/yadisk_parse/worker_test.go b/src/internal/async/worker/yadisk_parse/worker_test.go new file mode 100644 index 0000000..c9113f4 --- /dev/null +++ b/src/internal/async/worker/yadisk_parse/worker_test.go @@ -0,0 +1,38 @@ +package yadisk_parse_test + +import ( + "context" + "testing" + + "backend/src/internal/async/semaphore" + "backend/src/internal/async/status" + "backend/src/internal/async/worker/config" + "backend/src/internal/async/worker/yadisk_parse" + "backend/src/internal/db/postgres" + "backend/src/internal/repository/impl" +) + +func TestPopulation(t *testing.T) { + ctx := context.Background() + ch := make(chan status.CompletionStatus) + sem := semaphore.NewSemaphore(3) + conn := postgres.NewPostgresConnection("postgresql://test_user:123@127.0.0.1:5432/test_db?sslmode=disable") + repositories := config.WorkerConfig{ + GeoRepository: impl.NewGeoRepository(), + RosstatRepository: impl.NewRosstatRepository(), + RosstatAgeRepository: impl.NewRosstatAgeRepository(), + AiApiRepository: impl.NewAiApiRepository(), + } + + yadisk_parse.YadiskParseWorker( + ctx, + ch, + sem, + conn, + repositories, + 1, + 0, + ) + + t.Log("YadiskWorker was finished.\n") +} diff --git a/src/internal/context/abstract/handler.go b/src/internal/context/abstract/handler.go index 2184f5e..4978b36 100644 --- a/src/internal/context/abstract/handler.go +++ b/src/internal/context/abstract/handler.go @@ -2,6 +2,6 @@ package abstract type HandlerContext interface { Get(key string, defaultValue ...string) string + Status(status int) HandlerContext - BindJSON(data any) error } diff --git a/src/internal/context/adapter/fiber.go b/src/internal/context/adapter/fiber.go index a6b03df..0812e88 100644 --- a/src/internal/context/adapter/fiber.go +++ b/src/internal/context/adapter/fiber.go @@ -18,7 +18,3 @@ func (fiberAdapter *FiberCtxAdapter) Status(status int) abstract.HandlerContext fiberAdapter.Ctx.Status(status) return fiberAdapter } - -func (fiberAdapter *FiberCtxAdapter) BindJSON(data any) error { - return fiberAdapter.Ctx.Bind().JSON(data) -} diff --git a/src/internal/domain/ai_report.go b/src/internal/domain/ai_report.go index 4c2df92..03ca7bc 100644 --- a/src/internal/domain/ai_report.go +++ b/src/internal/domain/ai_report.go @@ -1,6 +1,6 @@ package domain type AiReport struct { - Code int - Report string + code int + report string } diff --git a/src/internal/domain/async_request.go b/src/internal/domain/async_request.go deleted file mode 100644 index a32993f..0000000 --- a/src/internal/domain/async_request.go +++ /dev/null @@ -1,15 +0,0 @@ -package domain - -import "time" - -type AsyncRequest struct { - ID int - Hash string - Code int - Status int - RequestType int - Result string - Error string - CreatedAt time.Time - UpdatedAt time.Time -} diff --git a/src/internal/dto/async_request.go b/src/internal/dto/async_request.go deleted file mode 100644 index 2669147..0000000 --- a/src/internal/dto/async_request.go +++ /dev/null @@ -1,42 +0,0 @@ -package dto - -type CreateReportRequest struct { - Code int `json:"code" binding:"required"` -} - -type ReportCodeParams struct { - Code int `uri:"code" binding:"required"` -} - -type ReportHashParams struct { - Hash string `uri:"hash" binding:"required"` -} - -type CreateReportResponse struct { - Hash string `json:"hash"` - Message string `json:"message"` -} - -type RequestStatusResponse struct { - Hash string `json:"hash"` - Status int `json:"status"` - Error string `json:"error,omitempty"` -} - -type ReportResponse struct { - Report AIReportContent `json:"report"` -} - -type AIReportContent struct { - Summary string `json:"summary"` - Forecast Forecast `json:"forecast"` - Trends []string `json:"trends"` - Recommendations []string `json:"recommendations"` -} - -type Forecast struct { - Scenario string `json:"scenario"` - Population string `json:"population"` - NaturalGrowth string `json:"natural_growth"` - MigrationGrowth string `json:"migration_growth"` -} diff --git a/src/internal/handler/public/async_report.go b/src/internal/handler/public/async_report.go deleted file mode 100644 index 0b465f8..0000000 --- a/src/internal/handler/public/async_report.go +++ /dev/null @@ -1,129 +0,0 @@ -package public - -import ( - request_type "backend/src/internal/async/request" - context "backend/src/internal/context/abstract" - "backend/src/internal/domain" - "backend/src/internal/dto" - service "backend/src/internal/service/abstract" - "encoding/json" - "net/http" - "strconv" -) - -func GetReportAsyncHandler(ctx context.HandlerContext, - aiReportAsyncService service.IAiReportAsyncService, -) dto.ReportResponse { - codeStr := ctx.Get("code") - - code, err := strconv.Atoi(codeStr) - if err != nil { - ctx.Status(http.StatusBadRequest) - return dto.ReportResponse{} - } - - if code <= 0 { - ctx.Status(http.StatusBadRequest) - return dto.ReportResponse{} - } - - report, err := aiReportAsyncService.GetReportByCode(code) - if err != nil || report == nil { - ctx.Status(http.StatusBadRequest) - return dto.ReportResponse{} - } - - return buildAsyncReportResponse(ctx, report) -} - -func buildAsyncReportResponse(ctx context.HandlerContext, report *domain.AsyncRequest) dto.ReportResponse { - var content dto.AIReportContent - if err := json.Unmarshal([]byte(report.Result), &content); err != nil { - ctx.Status(http.StatusInternalServerError) - return dto.ReportResponse{} - } - - return dto.ReportResponse{ - Report: content, - } -} - -func PostReportAsyncHandler(ctx context.HandlerContext, - aiReportAsyncService service.IAiReportAsyncService, -) dto.CreateReportResponse { - var request dto.CreateReportRequest - - if err := ctx.BindJSON(&request); err != nil { - ctx.Status(http.StatusBadRequest) - return dto.CreateReportResponse{ - Hash: "", - Message: err.Error(), - } - } - - if request.Code <= 0 { - ctx.Status(http.StatusBadRequest) - return dto.CreateReportResponse{ - Hash: "", - Message: "code must be positive", - } - } - - hash, err := aiReportAsyncService.CreateReportRequest(request.Code, request_type.AIREPORT) - if err != nil { - ctx.Status(http.StatusBadRequest) - return dto.CreateReportResponse{ - Hash: "", - Message: err.Error(), - } - } - - ctx.Status(http.StatusAccepted) // дада 202 а не 204 тк 204 это error - return dto.CreateReportResponse{ - Hash: hash, - Message: "Запрос принят в работу", - } -} - -func GetRequestStatusHandler(ctx context.HandlerContext, - aiReportAsyncService service.IAiReportAsyncService, -) dto.RequestStatusResponse { - codeStr := ctx.Get("code") - hash := ctx.Get("hash") - - if codeStr == "" || hash == "" { - ctx.Status(http.StatusBadRequest) - return dto.RequestStatusResponse{ - Hash: "", - Status: 0, - Error: "code and hash parameters are required", - } - } - - request, err := aiReportAsyncService.GetRequestStatusByHash(hash) - if err != nil { - ctx.Status(http.StatusBadRequest) - return dto.RequestStatusResponse{ - Hash: hash, - Status: 0, - Error: err.Error(), - } - } - - if request == nil { - ctx.Status(http.StatusBadRequest) - return dto.RequestStatusResponse{ - Hash: hash, - Status: 0, - Error: "there are no requests with this hash", - } - } - - ctx.Status(http.StatusOK) - return dto.RequestStatusResponse{ - Hash: request.Hash, - Status: request.Status, - Error: request.Error, - } - -} diff --git a/src/internal/model/async_request.go b/src/internal/model/async_request.go index cd72588..dbdde97 100644 --- a/src/internal/model/async_request.go +++ b/src/internal/model/async_request.go @@ -1,37 +1,15 @@ package model -import ( - "backend/src/internal/async/status" - "backend/src/internal/domain" - "time" -) +import "time" type AsyncRequest struct { - ID int `gorm:"primaryKey;autoIncrement"` - Hash string `gorm:"column:hash;type:text;unique;not null"` - Code int `gorm:"column:code;type:int;not null"` - Status int `gorm:"column:status;type:int;not null"` - RequestType int `gorm:"column:request_type;type:int;not null;"` - Result string `gorm:"column:result;type:text"` - Attempts int `gorm:"column:attempts;type:int"` - Error string `gorm:"column:error;type:text"` - CreatedAt time.Time `gorm:"column:created_at;type:timestamp;not null"` - UpdatedAt time.Time `gorm:"column:updated_at;type:timestamp;autoUpdateTime;not null"` - DeadlineAt *time.Time `gorm:"column:deadline_at;type:timestamp;nullable"` -} - -func (modelObj *AsyncRequest) ToDomain() (*domain.AsyncRequest, error) { - return ToDomain[AsyncRequest, domain.AsyncRequest](modelObj) -} - -func NewAsyncRequest(hash string, code int, requestType int) *AsyncRequest { - now := time.Now() - return &AsyncRequest{ - Hash: hash, - Code: code, - Status: status.StatusQueued, - RequestType: requestType, - CreatedAt: now, - UpdatedAt: now, - } + ID int `gorm:"primaryKey;autoIncrement"` + Hash string `gorm:"column:hash;type:text;unique;not null"` + Type int `gorm:"column:type;type:int;not null"` + Parameter int `gorm:"column:parameter;type:int;not null"` + Status int `gorm:"column:status;type:int;not null"` + Attempts int `gorm:"column:attempts;type:int;not null"` + UpdatedAt time.Time `gorm:"column:updated_at;type:timestamp;autoUpdateTime;not null"` + DeadlineAt *time.Time `gorm:"column:deadline_at;type:timestamp;nullable"` + TimeoutSeconds int `gorm:"column:timeout_seconds;type:int;not null"` } diff --git a/src/internal/repository/abstract/async_request.go b/src/internal/repository/abstract/async_request.go index 7a6c5ab..b5279f0 100644 --- a/src/internal/repository/abstract/async_request.go +++ b/src/internal/repository/abstract/async_request.go @@ -2,7 +2,6 @@ package abstract import ( "backend/src/internal/db/abstract" - "backend/src/internal/domain" "backend/src/internal/model" "context" ) @@ -12,8 +11,4 @@ type IAsyncRequestRepository interface { SetStatusById(ctx context.Context, conn abstract.IDBConnection, id int, status int) error SetStatusAndIncrementById(ctx context.Context, conn abstract.IDBConnection, id int, status int) error CloseTimeoutRequests(ctx context.Context, conn abstract.IDBConnection) error - - CreateRequest(conn abstract.IDBConnection, requst *model.AsyncRequest) error - GetSuccessRequestByCode(conn abstract.IDBConnection, code int) (*model.AsyncRequest, error) - GetRequestByHash(conn abstract.IDBConnection, hash string) (*domain.AsyncRequest, error) } diff --git a/src/internal/repository/impl/async_request.go b/src/internal/repository/impl/async_requests.go similarity index 57% rename from src/internal/repository/impl/async_request.go rename to src/internal/repository/impl/async_requests.go index 9b17425..db48285 100644 --- a/src/internal/repository/impl/async_request.go +++ b/src/internal/repository/impl/async_requests.go @@ -1,13 +1,10 @@ package impl import ( - "backend/src/internal/async/request" "backend/src/internal/async/status" "backend/src/internal/db/abstract" - "backend/src/internal/domain" "backend/src/internal/model" "context" - "errors" "time" "gorm.io/gorm" @@ -43,7 +40,7 @@ func (r *AsyncRequestRepository) SetStatusById(ctx context.Context, conn abstrac return db.WithContext(ctx). Model(&model.AsyncRequest{}). - Where("request_type = ? AND id = ?", request.AIREPORT, id). + Where("id = ?", id). Updates(map[string]interface{}{ "status": status, "updated_at": time.Now(), @@ -56,8 +53,8 @@ func (r *AsyncRequestRepository) SetStatusAndIncrementById(ctx context.Context, return db.WithContext(ctx). Model(&model.AsyncRequest{}). - Where("request_type = ? AND id = ? AND status NOT IN (?, ?)", - request.AIREPORT, id, status.StatusSuccess, status.StatusFailed). + Where("id = ? AND status NOT IN (?, ?)", + id, status.StatusSuccess, status.StatusFailed). Updates(map[string]interface{}{ "status": gorm.Expr("CASE WHEN attempts + 1 >= 3 THEN ? ELSE ? END", status.StatusFailed, stat), @@ -72,47 +69,11 @@ func (r *AsyncRequestRepository) CloseTimeoutRequests(ctx context.Context, conn return db.WithContext(ctx). Model(&model.AsyncRequest{}). - Where("request_type = ? AND status = ? AND deadline_at IS NOT NULL AND deadline_at < ?", - request.AIREPORT, status.StatusInProgress, time.Now()). + Where("status = ? AND deadline_at IS NOT NULL AND deadline_at < ?", + status.StatusInProgress, time.Now()). Updates(map[string]interface{}{ "status": gorm.Expr("CASE WHEN attempts >= 3 THEN ? ELSE ? END", status.StatusFailed, status.StatusQueued), "updated_at": time.Now(), }).Error } - -func (r *AsyncRequestRepository) CreateRequest(conn abstract.IDBConnection, request *model.AsyncRequest) error { - db := conn.Get().(*gorm.DB) - return db.Create(request).Error -} - -func (r *AsyncRequestRepository) GetSuccessRequestByCode(conn abstract.IDBConnection, code int) (*model.AsyncRequest, error) { - db := conn.Get().(*gorm.DB) - - var asyncRequest model.AsyncRequest - err := db. - Where("equest_type = ? AND code = ? AND status = ?", request.AIREPORT, code, status.StatusSuccess). - First(&asyncRequest).Error - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return nil, nil - } - return nil, err - } - return &asyncRequest, nil -} - -func (r *AsyncRequestRepository) GetRequestByHash(conn abstract.IDBConnection, hash string) (*domain.AsyncRequest, error) { - db := conn.Get().(*gorm.DB) - - var request model.AsyncRequest - err := db.Where("hash = ?", hash).First(&request).Error - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return nil, nil - } - return nil, err - } - - return request.ToDomain() -} diff --git a/src/internal/service/abstract/async_request.go b/src/internal/service/abstract/async_request.go deleted file mode 100644 index 891f795..0000000 --- a/src/internal/service/abstract/async_request.go +++ /dev/null @@ -1,11 +0,0 @@ -package abstract - -import ( - "backend/src/internal/domain" -) - -type IAiReportAsyncService interface { - GetReportByCode(code int) (*domain.AsyncRequest, error) - CreateReportRequest(code int, requestType int) (string, error) - GetRequestStatusByHash(hash string) (*domain.AsyncRequest, error) -} diff --git a/src/internal/service/impl/async_request.go b/src/internal/service/impl/async_request.go deleted file mode 100644 index bde86e1..0000000 --- a/src/internal/service/impl/async_request.go +++ /dev/null @@ -1,79 +0,0 @@ -package impl - -import ( - connection "backend/src/internal/db/abstract" - "backend/src/internal/domain" - "backend/src/internal/model" - repository "backend/src/internal/repository/abstract" - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "strconv" -) - -type AiReportAsyncService struct { - conn connection.IDBConnection - asyncRequestRepo repository.IAsyncRequestRepository -} - -func NewAiReportAsyncService(conn connection.IDBConnection, - asyncRequestRepo repository.IAsyncRequestRepository, -) *AiReportAsyncService { - return &AiReportAsyncService{ - conn: conn, - asyncRequestRepo: asyncRequestRepo, - } -} - -func (service *AiReportAsyncService) GetReportByCode(code int) (*domain.AsyncRequest, error) { - request, err := service.asyncRequestRepo.GetSuccessRequestByCode(service.conn, code) - if err != nil { - return nil, err - } - - if request == nil { - return nil, nil - } - - return request.ToDomain() -} - -func (service *AiReportAsyncService) CreateReportRequest(code int, requestType int) (string, error) { - exitingRequest, err := service.asyncRequestRepo.GetOneRequest(context.Background(), service.conn) - if err != nil { - return "", err - } - - if exitingRequest != nil { - return "", errors.New("request already exists") - } - - hash := generateHash(code) - - request := model.NewAsyncRequest(hash, code, requestType) - - err = service.asyncRequestRepo.CreateRequest(service.conn, request) - if err != nil { - return "", err - } - - return hash, nil -} - -func (service *AiReportAsyncService) GetRequestStatusByHash(hash string) (*domain.AsyncRequest, error) { - request, err := service.asyncRequestRepo.GetRequestByHash(service.conn, hash) - if err != nil { - return nil, err - } - if request == nil { - return nil, nil - } - - return request, nil -} - -func generateHash(code int) string { - hash := sha256.Sum256([]byte(strconv.Itoa(code))) - return hex.EncodeToString(hash[:]) -} diff --git a/src/pkg/.DS_Store b/src/pkg/.DS_Store deleted file mode 100644 index 918756a..0000000 Binary files a/src/pkg/.DS_Store and /dev/null differ diff --git a/src/pkg/parser/.DS_Store b/src/pkg/parser/.DS_Store deleted file mode 100644 index af8b6db..0000000 Binary files a/src/pkg/parser/.DS_Store and /dev/null differ diff --git a/src/pkg/parser/rosstat/.DS_Store b/src/pkg/parser/rosstat/.DS_Store deleted file mode 100644 index af8b6db..0000000 Binary files a/src/pkg/parser/rosstat/.DS_Store and /dev/null differ diff --git a/src/pkg/parser/rosstat/domain/rosstat.go b/src/pkg/parser/rosstat/domain/rosstat.go index ceafe3f..c3048a5 100644 --- a/src/pkg/parser/rosstat/domain/rosstat.go +++ b/src/pkg/parser/rosstat/domain/rosstat.go @@ -8,10 +8,11 @@ type RosstatAgeParsed struct { type RosstatParsed struct { Code int - ParentCode int - Name string + SubjectCode int Year int Population *int + RuralUsed bool + UrbanUsed bool Birth *int Death *int Arrival *int diff --git a/src/pkg/parser/rosstat/rosstat/.DS_Store b/src/pkg/parser/rosstat/rosstat/.DS_Store deleted file mode 100644 index 5008ddf..0000000 Binary files a/src/pkg/parser/rosstat/rosstat/.DS_Store and /dev/null differ diff --git a/src/pkg/parser/rosstat/rosstat/config/config.go b/src/pkg/parser/rosstat/rosstat/config/config.go deleted file mode 100644 index 0451cdd..0000000 --- a/src/pkg/parser/rosstat/rosstat/config/config.go +++ /dev/null @@ -1,87 +0,0 @@ -package config - -const ( - BURYATIA_CODE = 81 -) - -type Config struct { - populationIndicator int - populationIndicatorBuryatia int - - birthIndicator int - deathIndicator int - - SubjectCodes []int - - DownloadHTMLMaxAttempts int - DownloadHTMLTimeoutSeconds int - DownloadHTMLBatchSize int - DownloadHTMLTimeSleepSeconds int - - DownloadCSVMaxAttempts int - DownloadCSVTimeoutSeconds int - DownloadCSVBatchSize int - DownloadCSVTimeSleepSeconds int -} - -func NewConfig() *Config { - return &Config{ - populationIndicator: 8112027, - populationIndicatorBuryatia: 8312027, - birthIndicator: 8112003, - deathIndicator: 8112001, - SubjectCodes: []int{ - 1, 3, 4, 5, 7, 8, 10, 11, 12, 14, 15, 17, 18, 19, 20, 22, 24, - 25, 26, 27, 28, 29, 30, 32, 33, 34, 35, 36, 37, 38, 40, 41, 42, - 44, 45, 46, 47, 49, 50, 52, 53, 54, 56, 57, 58, 60, 61, 63, 64, - 65, 66, 67, 68, 69, 70, 71, 73, 75, 76, 77, 78, 79, 80, 81, 82, - 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, - }, - DownloadHTMLMaxAttempts: 6, - DownloadHTMLTimeoutSeconds: 5, - DownloadHTMLBatchSize: 10, - DownloadHTMLTimeSleepSeconds: 2, - DownloadCSVMaxAttempts: 8, - DownloadCSVTimeoutSeconds: 7, - DownloadCSVBatchSize: 10, - DownloadCSVTimeSleepSeconds: 2, - } -} - -func (c *Config) GetPopulationIndicator(code int) int { - switch code { - case BURYATIA_CODE: - return c.populationIndicatorBuryatia - default: - return c.populationIndicator - } -} - -func (c *Config) GetPeriod(indicator int) int { - switch indicator { - case c.birthIndicator: - return 17 - case c.populationIndicator: - return 208 - case c.populationIndicatorBuryatia: - return 208 - case c.deathIndicator: - return 17 - default: - return 208 - } -} - -func (c *Config) GetBirthIndicator(code int) int { - switch code { - default: - return c.birthIndicator - } -} - -func (c *Config) GetDeathIndicator(code int) int { - switch code { - default: - return c.birthIndicator - } -} diff --git a/src/pkg/parser/rosstat/rosstat/downloader/downloader.go b/src/pkg/parser/rosstat/rosstat/downloader/downloader.go deleted file mode 100644 index 4080c65..0000000 --- a/src/pkg/parser/rosstat/rosstat/downloader/downloader.go +++ /dev/null @@ -1,200 +0,0 @@ -package downloader - -import ( - "backend/src/pkg/parser/rosstat/rosstat/config" - "context" - "crypto/tls" - "fmt" - "io" - "log" - "net/http" - "net/url" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "golang.org/x/text/encoding/charmap" -) - -func DownloadCSV(ctx context.Context, subjectCode int, indicator int, codes []int) (string, error) { - config := config.NewConfig() - - log.Printf("Downloading CSV: code=%d, indicator=%d\n", subjectCode, indicator) - - params := NewRequestParameters( - indicator, - getMunr(codes), - codes, - 2026, - ) - - url := fmt.Sprintf("https://rosstat.gov.ru/dbscripts/munst/munst%02d/DBInet.cgi", subjectCode) - - bodyStr := params.buildRequestBodyStr() - attempts := 0 - - for attempts < config.DownloadCSVMaxAttempts { - req, err := http.NewRequestWithContext(ctx, "POST", url, strings.NewReader(bodyStr)) - if err != nil { - return "", fmt.Errorf("error creating request: %w", err) - } - - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") - req.Header.Set("Accept-Language", "en-US,en;q=0.9") - req.Header.Set("Cache-Control", "max-age=0") - req.Header.Set("Referer", "https://rosstat.gov.ru/dbscripts/munst/munst87/DBInet.cgi") - req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36") - - client := &http.Client{ - Timeout: time.Duration(config.DownloadCSVTimeoutSeconds) * time.Second, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, - } - - resp, err := client.Do(req) - if err != nil { - if resp != nil { - resp.Body.Close() - } - attempts++ - time.Sleep(time.Duration(config.DownloadHTMLTimeSleepSeconds) * time.Second) - continue - } - - if err != nil { - return "", fmt.Errorf("error executing request: %w", err) - } - - if resp.StatusCode != http.StatusOK { - log.Printf("ERROR: unexpected status %d: code: %d\n", resp.StatusCode, subjectCode) - continue - } - - rawBody, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("reading body: %w", err) - } - - decoder := charmap.Windows1251.NewDecoder() - utf8Body, err := decoder.Bytes(rawBody) - if err != nil { - return "", fmt.Errorf("decoding Windows-1251: %w", err) - } - - fileName := getCSVFileName(subjectCode, indicator) - - if err := os.WriteFile(fileName, utf8Body, 0644); err != nil { - return "", fmt.Errorf("error writing file %s: %w", fileName, err) - } - - resp.Body.Close() - log.Printf("Saved CSV to %s\n", fileName) - return fileName, nil - } - - return "", fmt.Errorf("Failed CSV request by code %d\n", subjectCode) -} - -func DownloadHTML(ctx context.Context, subjectCode int) (string, error) { - config := config.NewConfig() - - log.Printf("Downloading HTML: code=%d\n", subjectCode) - - urlStr := fmt.Sprintf("https://rosstat.gov.ru/dbscripts/munst/munst%02d/DBInet.cgi", subjectCode) - - body := url.Values{} - body.Set("pl", strconv.Itoa(config.GetPopulationIndicator(subjectCode))) - attempts := 0 - - for attempts < config.DownloadHTMLMaxAttempts { - req, err := http.NewRequestWithContext(ctx, "POST", urlStr, strings.NewReader(body.Encode())) - if err != nil { - return "", fmt.Errorf("error creating request: %w", err) - } - - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; OktmoGrabber/1.0)") - - client := &http.Client{ - Timeout: time.Duration(config.DownloadHTMLTimeoutSeconds) * time.Second, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, - } - - resp, err := client.Do(req) - - if err != nil { - if resp != nil { - resp.Body.Close() - } - attempts++ - time.Sleep(time.Duration(config.DownloadHTMLTimeSleepSeconds) * time.Second) - continue - } - - if err != nil { - return "", fmt.Errorf("error executing request: %w", err) - } - - if resp.StatusCode != http.StatusOK { - log.Printf("ERROR: unexpected status %d: code: %d\n", resp.StatusCode, subjectCode) - continue - } - - rawBody, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("reading body: %w", err) - } - - decoder := charmap.Windows1251.NewDecoder() - utf8Body, err := decoder.Bytes(rawBody) - if err != nil { - return "", fmt.Errorf("decoding Windows-1251: %w", err) - } - - fileName := getHTMLFileName(subjectCode) - - if err := os.WriteFile(fileName, utf8Body, 0644); err != nil { - return "", fmt.Errorf("error writing file %s: %w", fileName, err) - } - - resp.Body.Close() - log.Printf("Saved HTML to %s\n", fileName) - return fileName, nil - } - - return "", fmt.Errorf("Failed HTML request by code %d\n", subjectCode) -} - -func CleanupFile(path string) { - if err := os.Remove(path); err != nil { - log.Printf("Warning: failed to remove %s: %v", path, err) - } -} - -func getCSVFileName(code int, indicator int) string { - now := time.Now().UnixNano() - return filepath.Join(".", fmt.Sprintf("temp_%d_%d_%d.csv", code, indicator, now)) -} - -func getHTMLFileName(code int) string { - now := time.Now().UnixNano() - return filepath.Join(".", fmt.Sprintf("temp_%d_%d.html", code, now)) -} - -func getMunr(codes []int) []int { - munr := make([]int, 0, len(codes)) - - for _, code := range codes { - if code%1000 == 0 { - munr = append(munr, code) - } - } - - return munr -} diff --git a/src/pkg/parser/rosstat/rosstat/downloader/request.go b/src/pkg/parser/rosstat/rosstat/downloader/request.go deleted file mode 100644 index 0faf74a..0000000 --- a/src/pkg/parser/rosstat/rosstat/downloader/request.go +++ /dev/null @@ -1,130 +0,0 @@ -package downloader - -import ( - "backend/src/pkg/parser/rosstat/rosstat/config" - "fmt" - "net/url" - "strconv" - "strings" -) - -type QueryDimensions struct { - Pokazateli []int - Munr []int - Tippos []int - Oktmo []int - Vozr int - Grup_2 []int - God []int - Period int - Mest []int -} - -type QueryGm struct { - Vozr_z int - Period_z int - God_s int - Munr_b int - Oktmo_b int - Grup_2_b int - Pokazateli_b int - Tippos_b int - Mest_b int -} - -type RequestParameters struct { - Format string - YearFrom int - YearTo int - Qry QueryDimensions - QryGm QueryGm - YearsList []int -} - -func NewRequestParameters(indicator int, munr []int, oktmo []int, yearTo int) *RequestParameters { - config := config.NewConfig() - years := generateYears(2000, yearTo) - return &RequestParameters{ - Format: "CSV", - YearFrom: 2000, - YearTo: yearTo, - Qry: QueryDimensions{ - Pokazateli: []int{indicator}, - Munr: munr, - Tippos: []int{10, 7, 1, 4, 20}, - Oktmo: oktmo, - Vozr: 151, - Grup_2: []int{1, 2, 3}, - God: years, - Period: config.GetPeriod(indicator), - Mest: []int{11}, - }, - QryGm: QueryGm{ - Vozr_z: 1, - Period_z: 2, - God_s: 1, - Munr_b: 1, - Oktmo_b: 2, - Grup_2_b: 3, - Pokazateli_b: 4, - Tippos_b: 5, - Mest_b: 6, - }, - YearsList: years, - } -} - -func (p *RequestParameters) buildRequestBodyStr() string { - qryRaw := fmt.Sprintf( - "Pokazateli:%s;munr:%s;tippos:%s;oktmo:%s;vozr:%d;grup_2:%s;god:%s;period:%d;mest:%s;", - joinInts(p.Qry.Pokazateli, ","), - joinInts(p.Qry.Munr, ","), - joinInts(p.Qry.Tippos, ","), - joinInts(p.Qry.Oktmo, ","), - p.Qry.Vozr, - joinInts(p.Qry.Grup_2, ","), - joinInts(p.Qry.God, ","), - p.Qry.Period, - joinInts(p.Qry.Mest, ","), - ) - qryEncoded := url.QueryEscape(qryRaw) - - qryGmRaw := fmt.Sprintf( - "vozr_z:%d;period_z:%d;god_s:%d;munr_b:%d;oktmo_b:%d;grup_2_b:%d;Pokazateli_b:%d;tippos_b:%d;mest_b:%d;", - p.QryGm.Vozr_z, p.QryGm.Period_z, p.QryGm.God_s, - p.QryGm.Munr_b, p.QryGm.Oktmo_b, p.QryGm.Grup_2_b, - p.QryGm.Pokazateli_b, p.QryGm.Tippos_b, p.QryGm.Mest_b, - ) - qryGmEncoded := url.QueryEscape(qryGmRaw) - - yearsRaw := joinInts(p.YearsList, ";") + ";" - yearsEncoded := url.QueryEscape(yearsRaw) - - body := fmt.Sprintf( - "DiagSz=800x600&Format=%s&YearTo=%d&YearFrom=%d&Qry=%s&QryGm=%s&QryFootNotes=%%3B&YearsList=%s&tbl=%s", - p.Format, - p.YearTo, - p.YearFrom, - qryEncoded, - qryGmEncoded, - yearsEncoded, - "%CF%EE%EA%E0%E7%E0%F2%FC+%F2%E0%E1%EB%E8%F6%F3", - ) - return body -} - -func joinInts(nums []int, sep string) string { - strs := make([]string, len(nums)) - for i, n := range nums { - strs[i] = strconv.Itoa(n) - } - return strings.Join(strs, sep) -} - -func generateYears(yearFrom int, yearTo int) []int { - years := make([]int, 0, yearTo-yearFrom+1) - for y := yearFrom; y <= yearTo; y++ { - years = append(years, y) - } - return years -} diff --git a/src/pkg/parser/rosstat/rosstat/extractor/base.go b/src/pkg/parser/rosstat/rosstat/extractor/base.go deleted file mode 100644 index 9a40d64..0000000 --- a/src/pkg/parser/rosstat/rosstat/extractor/base.go +++ /dev/null @@ -1,63 +0,0 @@ -package extractor - -import ( - "context" - "encoding/csv" - "fmt" - "io" - "os" - - "backend/src/pkg/parser/rosstat/rosstat/state_manager" -) - -type BaseCSVExtractor[T any] struct { -} - -func (r *BaseCSVExtractor[T]) getReader(filePath string) (*csv.Reader, *os.File, error) { - var err error - file, err := os.Open(filePath) - - if err != nil { - return nil, nil, err - } - - reader := csv.NewReader(file) - reader.FieldsPerRecord = -1 - reader.Comma = ';' - reader.LazyQuotes = true - - return reader, file, nil -} - -func (r *BaseCSVExtractor[T]) ExtractRows( - ctx context.Context, - filePath string, - manager *state_manager.StateManager[T], -) error { - reader, file, err := r.getReader(filePath) - - if err != nil { - return err - } - - defer file.Close() - - for { - if err := ctx.Err(); err != nil { - return err - } - - row, err := reader.Read() - - if err == io.EOF { - break - } - - if err != nil { - return fmt.Errorf("read error at file %s: %w", filePath, err) - } - - manager.DoOnStateState(row) - } - return nil -} diff --git a/src/pkg/parser/rosstat/rosstat/extractor/birth.go b/src/pkg/parser/rosstat/rosstat/extractor/birth.go deleted file mode 100644 index b64b416..0000000 --- a/src/pkg/parser/rosstat/rosstat/extractor/birth.go +++ /dev/null @@ -1,95 +0,0 @@ -package extractor - -import ( - "context" - "strconv" - - "backend/src/pkg/parser/rosstat/rosstat/dao" - "backend/src/pkg/parser/rosstat/rosstat/state_manager" - "backend/src/pkg/parser/rosstat/rosstat/storage" -) - -type BirthExtractor struct { - BaseCSVExtractor[dao.BirthExtracted] - - storage *storage.Storage - - years []int - code int -} - -func NewBirthExtractor(storage *storage.Storage) *BirthExtractor { - return &BirthExtractor{storage: storage} -} - -func (ext *BirthExtractor) Extract(ctx context.Context, filePath string) error { - manager := state_manager.NewStateManager[dao.BirthExtracted]( - state_manager.StateMapFuncType[dao.BirthExtracted]{ - SKIP: ext.skip, - GET_YEARS: ext.getYears, - GET_NAME_OR_BIRTH: ext.getNameOrBirth, - }, - ) - - return ext.ExtractRows(ctx, filePath, manager) -} - -func (ext *BirthExtractor) skip(row []string, manager *state_manager.StateManager[dao.BirthExtracted]) { - manager.ChangeState(GET_YEARS) -} - -func (ext *BirthExtractor) getYears(row []string, manager *state_manager.StateManager[dao.BirthExtracted]) { - ext.years = make([]int, len(row)-1) - - for i := 1; i < len(row); i++ { - if row[i] == "" { - continue - } - year, err := strconv.Atoi(row[i]) - if err == nil { - ext.years[i-1] = year - } - } - - manager.ChangeState(GET_NAME_OR_BIRTH) -} - -func (ext *BirthExtractor) getNameOrBirth(row []string, manager *state_manager.StateManager[dao.BirthExtracted]) { - if len(row) == 0 || row[0] == "" { - return - } - - if (row[0] == "Муниципальный район") || row[0] == "Муниципальный округ" || - row[0] == "Городской округ, городской округ с внутригородским делением" || - row[0] == "Городские поселения" || row[0] == "Сельские поселения" && len(row) > 1 && row[1] != "" { - ext.getBirth(row, manager) - return - } - - code := ext.storage.GetCode(row[0]) - if code != 0 { - ext.code = code - } -} - -func (ext *BirthExtractor) getBirth(row []string, manager *state_manager.StateManager[dao.BirthExtracted]) { - birthDAOs := make([]*dao.BirthExtracted, 0, len(ext.years)) - - for i := 1; i < len(row) && i-1 < len(ext.years); i++ { - if row[i] == "" { - continue - } - birth, err := strconv.Atoi(row[i]) - if err == nil && birth > 0 { - birthDAOs = append(birthDAOs, &dao.BirthExtracted{ - Code: ext.code, - Year: ext.years[i-1], - Birth: birth, - }) - } - } - - if len(birthDAOs) > 0 { - ext.storage.SetBirth(birthDAOs) - } -} diff --git a/src/pkg/parser/rosstat/rosstat/extractor/code.go b/src/pkg/parser/rosstat/rosstat/extractor/code.go deleted file mode 100644 index 51a96e7..0000000 --- a/src/pkg/parser/rosstat/rosstat/extractor/code.go +++ /dev/null @@ -1,120 +0,0 @@ -package extractor - -import ( - "backend/src/pkg/parser/rosstat/rosstat/dao" - "context" - "fmt" - "log" - "os" - "regexp" - "strconv" - "strings" - - "github.com/PuerkitoBio/goquery" -) - -type CodeExtractor struct { -} - -func NewCodeExtractor() *CodeExtractor { - return &CodeExtractor{} -} - -func (ext *CodeExtractor) Extract(ctx context.Context, filePath string) ([]*dao.CodeExtracted, error) { - fileContentByte, err := os.ReadFile(filePath) - if err != nil { - return nil, err - } - - if err := ctx.Err(); err != nil { - return nil, err - } - - content := string(fileContentByte) - - codes, err := ext.extractCodes(content) - - if err != nil { - return nil, err - } - - if err := ctx.Err(); err != nil { - return nil, err - } - - names, err := ext.extractNames(content) - - if err != nil { - return nil, err - } - - if err := ctx.Err(); err != nil { - return nil, err - } - - if len(names) != len(codes) { - return nil, fmt.Errorf("error different count: names %d, codes %d", len(names), len(codes)) - } - - result := make([]*dao.CodeExtracted, len(names)) - for i := range names { - result[i] = &dao.CodeExtracted{ - Name: names[i], - Code: codes[i], - } - } - - return result, nil -} - -func (_ *CodeExtractor) extractCodes(html string) ([]int, error) { - re := regexp.MustCompile(`p_oktmo\[(\d+)\]="(\d+)"`) - matches := re.FindAllStringSubmatch(html, -1) - if matches == nil { - log.Printf("CODE_EXTRACTOR_ERROR p_oktmo not found: html=%s\n", html) - return nil, fmt.Errorf("error p_oktmo not found") - } - - maxIdx := -1 - for _, m := range matches { - idx, _ := strconv.Atoi(m[1]) - if idx > maxIdx { - maxIdx = idx - } - } - - codes := make([]int, maxIdx+1) - for _, m := range matches { - idx, _ := strconv.Atoi(m[1]) - code, err := strconv.Atoi(m[2]) - - if err != nil { - log.Printf("CODE_EXTRACTOR_ERROR convert code to int: html=%s\n", html) - return nil, fmt.Errorf("error converting code to int: %w", err) - } - - codes[idx] = code - } - return codes, nil -} - -func (_ *CodeExtractor) extractNames(html string) ([]string, error) { - doc, err := goquery.NewDocumentFromReader(strings.NewReader(html)) - if err != nil { - return nil, err - } - - var names []string - doc.Find("select[name='oktmo'] option").Each(func(i int, s *goquery.Selection) { - name := strings.TrimSpace(s.Text()) - if name != "" { - names = append(names, name) - } - }) - - if len(names) == 0 { - log.Printf("CODE_EXTRACTOR_ERROR