Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion config.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import os
from pathlib import Path
from urllib.parse import urlparse

from dotenv import load_dotenv
from fastapi import HTTPException
from urllib.parse import urlparse

BASE_DIR = Path(__file__).resolve().parent

Expand Down
6 changes: 3 additions & 3 deletions generate_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@

import argparse
import csv
from datetime import datetime
import html
import io
from datetime import datetime
from pathlib import Path
from typing import Any
from urllib.parse import parse_qs, urlparse
Expand All @@ -20,7 +20,7 @@ def parse_drive_file_id(value: str) -> str | None:
parsed = urlparse(value)
query = parse_qs(parsed.query)

if "id" in query and query["id"]:
if query.get("id"):
return query["id"][0]

if parsed.netloc and "drive.google.com" in parsed.netloc and parsed.path:
Expand Down Expand Up @@ -269,7 +269,7 @@ def main() -> None:
args = parser.parse_args()

rows = fetch_csv_rows(args.source)
generated_at = datetime.now().strftime("%d/%m/%Y %H:%M")
generated_at = datetime.now().astimezone().strftime("%d/%m/%Y %H:%M")

if args.city:
if not rows:
Expand Down
11 changes: 6 additions & 5 deletions main.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
from fastapi import BackgroundTasks, FastAPI
from fastapi.responses import HTMLResponse, FileResponse
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, HTMLResponse
from fastapi.staticfiles import StaticFiles

from config import BASE_DIR
from utils.macrotemas import MACROTEMAS, TODOS_MACROTEMAS_NOME, TODOS_MACROTEMAS_SLUG
from utils.cities import carregar_cidades
from utils.ssr import start_server as start_ssr_server, stop_server as stop_ssr_server
from reports import (
listar_relatorios_handler,
apagar_relatorio_handler,
gerar_relatorio_handler,
listar_relatorios_handler,
)
from utils.cities import carregar_cidades
from utils.macrotemas import MACROTEMAS, TODOS_MACROTEMAS_NOME, TODOS_MACROTEMAS_SLUG
from utils.ssr import start_server as start_ssr_server
from utils.ssr import stop_server as stop_ssr_server

app = FastAPI(on_startup=[start_ssr_server], on_shutdown=[stop_ssr_server])

Expand Down
4 changes: 3 additions & 1 deletion plotting.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import matplotlib.pyplot as plt
import pathlib

import matplotlib.pyplot as plt


def gerar_grafico_sexo(cidade, OUTPUT_DIR: pathlib.Path, safe_city: str):
mulheres = int(str(cidade["pop_mulher"]).replace('.', ''))
homens = int(str(cidade["pop_homem"]).replace('.', ''))
Expand Down
35 changes: 19 additions & 16 deletions reports.py
Original file line number Diff line number Diff line change
@@ -1,29 +1,32 @@
import logging
import re
from datetime import datetime, timezone
from pathlib import Path

import pandas as pd
from datetime import datetime
from fastapi import BackgroundTasks, HTTPException
from fastapi.responses import HTMLResponse
from weasyprint import HTML
from pathlib import Path

from config import OUTPUT_DIR, resolve_csv_source, require_config_value
from utils.macrotemas import MACROTEMAS, TODOS_MACROTEMAS_NOME, TODOS_MACROTEMAS_SLUG
from config import OUTPUT_DIR, require_config_value, resolve_csv_source
from plotting import gerar_grafico_porte, gerar_grafico_sexo, gerar_grafico_top_cidades
from utils.cities import filtrar_linhas_por_cidade
from utils.cover import montar_capa_relatorio
from utils.docs import (
carregar_texto_do_docs,
extrair_descricao_tema,
extrair_resumo_tema,
extrair_resumo_relatorio,
extrair_resumo_cidade,
extrair_resumo_relatorio,
extrair_resumo_tema,
)
from utils.macrotemas import MACROTEMAS, TODOS_MACROTEMAS_NOME, TODOS_MACROTEMAS_SLUG
from utils.renderer import (
render_descricao_tema_html,
render_mapa_marker,
substituir_placeholders,
texto_para_html,
)
from utils.cover import montar_capa_relatorio
from utils.renderer import render_descricao_tema_html, render_mapa_marker, substituir_placeholders, texto_para_html
from utils.ssr import render_react_ssr
from plotting import gerar_grafico_sexo
from plotting import gerar_grafico_porte
from plotting import gerar_grafico_top_cidades


logger = logging.getLogger(__name__)

Expand All @@ -34,7 +37,7 @@ def _gerar_pdf(html_content: str, pdf_file: Path) -> None:
try:
pdf_html = re.sub(r'src="/output/', 'src="', html_content)
HTML(string=pdf_html, base_url=str(OUTPUT_DIR.resolve())).write_pdf(str(pdf_file))
except Exception as e:
except (OSError, RuntimeError, TypeError, ValueError) as e:
logger.warning("Falha ao gerar PDF %s: %s", pdf_file, e)


Expand Down Expand Up @@ -106,7 +109,7 @@ async def listar_relatorios_handler():
mapa_file = OUTPUT_DIR / f"mapa_regiao_{slug_completo}.png"

stat = pdf_file.stat()
criado_em = datetime.fromtimestamp(stat.st_mtime)
criado_em = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).astimezone()

macrotema = "Demografia"
if "__" in slug_completo:
Expand Down Expand Up @@ -140,7 +143,7 @@ async def listar_relatorios_handler():
relatorios.sort(
key=lambda item: datetime.strptime(
f"{item['data']} {item['hora']}", "%d/%m/%Y %H:%M:%S"
),
).replace(tzinfo=timezone.utc),
reverse=True
)

Expand Down Expand Up @@ -182,7 +185,7 @@ async def apagar_relatorio_handler(arquivo_pdf: str):

async def gerar_relatorio_handler(cidade: str, macrotema: str = "demografia", charts: str = "all", *, background_tasks: BackgroundTasks):
macrotema_slugs = get_macrotema_slugs_para_relatorio(macrotema)
gerado_em = datetime.now()
gerado_em = datetime.now().astimezone()
allowed = set(CHART_TYPES.keys())
requested_charts = list(CHART_TYPES.keys()) if charts == "all" else [c.strip() for c in charts.split(",")]
invalid = set(requested_charts) - allowed
Expand Down
1 change: 0 additions & 1 deletion scripts/download_map_shapes.py
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
from urllib.parse import urlparse
from urllib.request import Request, urlopen


ROOT_DIR = Path(__file__).resolve().parents[1]
DEFAULT_OUTPUT_DIR = ROOT_DIR / "map_shape"
REQUIRED_FILES = (
Expand Down
12 changes: 8 additions & 4 deletions scripts/generate_all_maps.py
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,17 @@
import unicodedata
from pathlib import Path


ROOT_DIR = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT_DIR))

from config import OUTPUT_DIR # noqa: E402
from utils.cities import carregar_cidades # noqa: E402
from utils.maps import carregar_malhas, gerar_mapa_regiao, localizar_municipio, separar_cidade_uf # noqa: E402
from config import OUTPUT_DIR
from utils.cities import carregar_cidades
from utils.maps import (
carregar_malhas,
gerar_mapa_regiao,
localizar_municipio,
separar_cidade_uf,
)


def slugify(value: object) -> str:
Expand Down
1 change: 1 addition & 0 deletions utils/cities.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import ast
import re

import pandas as pd

from config import CITIES_FILE
Expand Down
6 changes: 4 additions & 2 deletions utils/contentful.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import logging
import re

import httpx

from config import get_config_value

logger = logging.getLogger(__name__)

CONTENTFUL_SPACE_ID = get_config_value("CONTENTFUL_SPACE_ID")
CONTENTFUL_ACCESS_TOKEN = get_config_value("CONTENTFUL_ACCESS_TOKEN")
Expand Down Expand Up @@ -50,7 +52,7 @@ def obter_url_mapa_contentful(nome_municipio: str) -> str | None:
file_url = items[0].get("fields", {}).get("file", {}).get("url", "")
if file_url:
return f"https:{file_url}" if file_url.startswith("//") else file_url
except Exception:
pass
except (httpx.HTTPError, KeyError, TypeError, AttributeError, ValueError) as e:
logger.debug("Conteúdo inesperado ao extrair URL do Contentful: %s", e)

return None
12 changes: 7 additions & 5 deletions utils/docs.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
import json
import logging
import re
import time
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.parse import urlparse
from urllib.request import urlopen
from urllib.error import URLError, HTTPError

from config import BASE_DIR

logger = logging.getLogger(__name__)

DOCS_CACHE_DIR = BASE_DIR / "output" / "docs_cache"
DOCS_CACHE_TTL = 3600 # 1 hour
Expand All @@ -25,8 +27,8 @@ def _carregar_do_cache(doc_id: str) -> str | None:
dados = json.loads(cache_path.read_text(encoding="utf-8"))
if time.time() - dados["timestamp"] < DOCS_CACHE_TTL:
return dados["texto"]
except Exception:
pass
except (OSError, json.JSONDecodeError, KeyError, TypeError) as e:
logger.debug("Falha ao ler cache de docs (%s): %s", cache_path, e)
return None


Expand All @@ -37,8 +39,8 @@ def _salvar_no_cache(doc_id: str, texto: str) -> None:
_cache_path(doc_id).write_text(
json.dumps(dados, ensure_ascii=False), encoding="utf-8"
)
except Exception:
pass
except (OSError, TypeError, ValueError) as e:
logger.debug("Falha ao salvar cache de docs (%s): %s", _cache_path(doc_id), e)


def extrair_doc_id(link_ou_id: str) -> str:
Expand Down
16 changes: 8 additions & 8 deletions utils/macrotemas.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
from config import (
DEMOGRAFIA_CSV_URL,
EDUCACAO_CSV_URL,
SAUDE_CSV_URL,
ECONOMIA_RENDA_CSV_URL,
SANEAMENTO_CSV_URL,
HIDRAULICA_CSV_URL,
DEMOGRAFIA_DOCS_URL,
EDUCACAO_DOCS_URL,
SAUDE_DOCS_URL,
ECONOMIA_RENDA_CSV_URL,
ECONOMIA_RENDA_DOCS_URL,
SANEAMENTO_DOCS_URL,
EDUCACAO_CSV_URL,
EDUCACAO_DOCS_URL,
HIDRAULICA_CSV_URL,
HIDRAULICA_DOCS_URL,
SANEAMENTO_CSV_URL,
SANEAMENTO_DOCS_URL,
SAUDE_CSV_URL,
SAUDE_DOCS_URL,
)

MACROTEMAS = {
Expand Down
Loading
Loading