Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 16 additions & 21 deletions Controller/Admin/EtlDashboardController.php
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
<?php

declare(strict_types=1);

namespace Oliverde8\PhpEtlEasyAdminBundle\Controller\Admin;

use Oliverde8\PhpEtlBundle\Entity\EtlExecution;
Expand All @@ -11,49 +13,42 @@

class EtlDashboardController extends AbstractController
{

protected EtlExecutionRepository $etlExecutionRepository;

/**
* EtlDashboardController constructor.
*
* @param EtlExecutionRepository $etlExecutionRepository
*/
public function __construct(EtlExecutionRepository $etlExecutionRepository)
public function __construct(protected EtlExecutionRepository $etlExecutionRepository)
{
$this->etlExecutionRepository = $etlExecutionRepository;
}


/**
* @Route("/etl/execution/dashboard", name="etl_execution_dashboard")
*/
public function index($startDate = null, $endDate = null): Response
{
$this->denyAccessUnlessGranted(EtlExecutionVoter::DASHBOARD, EtlExecution::class);

if (is_null($endDate)) {
if (null === $endDate) {
$endDate = new \DateTime();
}

if (is_null($startDate)) {
if (null === $startDate) {
$startDate = new \DateTime('7 days ago');
}

//TODO add Acl here for future proofing.
// TODO add Acl here for future proofing.
return $this->render(
"@Oliverde8PhpEtlEasyAdmin/admin/dashboard.html.twig",
'@Oliverde8PhpEtlEasyAdmin/admin/dashboard.html.twig',
[
'num_waiting' => $this->etlExecutionRepository->getCountInStatus($startDate, $endDate, EtlExecution::STATUS_WAITING),
'num_running' => $this->etlExecutionRepository->getCountInStatus($startDate, $endDate, EtlExecution::STATUS_RUNNING),
'num_success' => $this->etlExecutionRepository->getCountInStatus($startDate, $endDate, EtlExecution::STATUS_SUCCESS),
'num_failure' => $this->etlExecutionRepository->getCountInStatus($startDate, $endDate, EtlExecution::STATUS_FAILURE),
'max_wait_time' => $this->etlExecutionRepository->getMaxWaitTime($startDate, $endDate),
'avg_wait_time' => $this->etlExecutionRepository->getAvgWaitTime($startDate, $endDate),
'most_executed' => $this->etlExecutionRepository->getMostExecutedJobs($startDate, $endDate, 10),
'num_waiting' => $this->etlExecutionRepository->getCountInStatus($startDate, $endDate, EtlExecution::STATUS_WAITING),
'num_running' => $this->etlExecutionRepository->getCountInStatus($startDate, $endDate, EtlExecution::STATUS_RUNNING),
'num_success' => $this->etlExecutionRepository->getCountInStatus($startDate, $endDate, EtlExecution::STATUS_SUCCESS),
'num_failure' => $this->etlExecutionRepository->getCountInStatus($startDate, $endDate, EtlExecution::STATUS_FAILURE),
'max_wait_time' => $this->etlExecutionRepository->getMaxWaitTime($startDate, $endDate),
'avg_wait_time' => $this->etlExecutionRepository->getAvgWaitTime($startDate, $endDate),
'most_executed' => $this->etlExecutionRepository->getMostExecutedJobs($startDate, $endDate, 10),
'most_time_spent' => $this->etlExecutionRepository->getMostTimeSpentJobs($startDate, $endDate, 10),
'longest' => $this->etlExecutionRepository->getLongestJobs($startDate, $endDate, 10),
'crudController' => EtlExecutionCrudController::class
'longest' => $this->etlExecutionRepository->getLongestJobs($startDate, $endDate, 10),
'crudController' => EtlExecutionCrudController::class,
]
);
}
Expand Down
21 changes: 7 additions & 14 deletions Controller/Admin/EtlDownloadFileController.php
Original file line number Diff line number Diff line change
@@ -1,52 +1,45 @@
<?php

declare(strict_types=1);

namespace Oliverde8\PhpEtlEasyAdminBundle\Controller\Admin;

use Oliverde8\PhpEtlBundle\Entity\EtlExecution;
use Oliverde8\PhpEtlBundle\Security\EtlExecutionVoter;
use Oliverde8\PhpEtlBundle\Services\ChainWorkDirManager;
use Oliverde8\PhpEtlBundle\Services\ExecutionContextFactory;
use Sensio\Bundle\FrameworkExtraBundle\Configuration\ParamConverter;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
use Symfony\Component\HttpFoundation\HeaderUtils;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpFoundation\StreamedResponse;
use Symfony\Component\Routing\Annotation\Route;

class EtlDownloadFileController extends AbstractController
{
/** @var ExecutionContextFactory */
protected $executionContextFactory;

/**
* @param ExecutionContextFactory $executionContextFactory
*/
public function __construct(ExecutionContextFactory $executionContextFactory)
public function __construct(protected ExecutionContextFactory $executionContextFactory)
{
$this->executionContextFactory = $executionContextFactory;
}

/**
* @Route("/etl/execution/download", name="etl_execution_download_file")
*
* @ParamConverter(name="execution", Class="Oliverde8PhpEtlBundle:EtlExecution")
*/
public function index(EtlExecution $execution, string $filename): Response
{
$this->denyAccessUnlessGranted(EtlExecutionVoter::DOWNLOAD, EtlExecution::class);

$context = $this->executionContextFactory->get(['etl' => ['execution' => $execution]]);
$file = $context->getFileSystem()->readStream($filename);
$file = $context->getFileSystem()->readStream($filename);

$response = new StreamedResponse(function () use ($file) {
$outputStream = fopen('php://output', 'wb');
$response = new StreamedResponse(static function () use ($file): void {
$outputStream = fopen('php://output', 'w');
stream_copy_to_stream($file, $outputStream);
});

$disposition = HeaderUtils::makeDisposition(
HeaderUtils::DISPOSITION_ATTACHMENT,
"execution-{$execution->getName()}-{$execution->getId()}-" . $filename
\sprintf('execution-%s-%s-', $execution->getName(), $execution->getId()).$filename
);
$response->headers->set('Content-Disposition', $disposition);

Expand Down
96 changes: 45 additions & 51 deletions Controller/Admin/EtlExecutionCrudController.php
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
<?php

declare(strict_types=1);

namespace Oliverde8\PhpEtlEasyAdminBundle\Controller\Admin;

use Doctrine\ORM\EntityManagerInterface;
Expand All @@ -8,6 +10,7 @@
use EasyCorp\Bundle\EasyAdminBundle\Config\Assets;
use EasyCorp\Bundle\EasyAdminBundle\Config\Crud;
use EasyCorp\Bundle\EasyAdminBundle\Config\Filters;
use EasyCorp\Bundle\EasyAdminBundle\Controller\AbstractCrudController;
use EasyCorp\Bundle\EasyAdminBundle\Field\ChoiceField;
use EasyCorp\Bundle\EasyAdminBundle\Field\CodeEditorField;
use EasyCorp\Bundle\EasyAdminBundle\Field\Field;
Expand All @@ -16,34 +19,16 @@
use EasyCorp\Bundle\EasyAdminBundle\Filter\ChoiceFilter;
use EasyCorp\Bundle\EasyAdminBundle\Router\AdminUrlGenerator;
use Oliverde8\PhpEtlBundle\Entity\EtlExecution;
use EasyCorp\Bundle\EasyAdminBundle\Controller\AbstractCrudController;
use Oliverde8\PhpEtlBundle\Security\EtlExecutionVoter;
use Oliverde8\PhpEtlBundle\Services\ChainProcessorsManager;
use Oliverde8\PhpEtlBundle\Services\ChainWorkDirManager;
use Oliverde8\PhpEtlBundle\Services\ExecutionContextFactory;

class EtlExecutionCrudController extends AbstractCrudController
{
/** @var ExecutionContextFactory */
protected $executionContextFactory;

/** @var ChainProcessorsManager */
protected $chainProcessorManager;

/** @var AdminUrlGenerator */
protected $adminUrlGenerator;

public function __construct(
ExecutionContextFactory $executionContextFactory,
ChainProcessorsManager $chainProcessorManager,
AdminUrlGenerator $adminUrlGenerator
) {
$this->executionContextFactory = $executionContextFactory;
$this->chainProcessorManager = $chainProcessorManager;
$this->adminUrlGenerator = $adminUrlGenerator;
public function __construct(protected ExecutionContextFactory $executionContextFactory, protected ChainProcessorsManager $chainProcessorManager, protected AdminUrlGenerator $adminUrlGenerator)
{
}


public static function getEntityFqcn(): string
{
return EtlExecution::class;
Expand All @@ -62,6 +47,7 @@ public function configureActions(Actions $actions): Actions
if (!$this->isGranted(EtlExecutionVoter::QUEUE, EtlExecution::class)) {
$actions->remove(Crud::PAGE_INDEX, Action::NEW);
}

if (!$this->isGranted(EtlExecutionVoter::VIEW, EtlExecution::class)) {
$actions->remove(Crud::PAGE_INDEX, Action::DETAIL);
}
Expand All @@ -72,42 +58,42 @@ public function configureActions(Actions $actions): Actions
public function configureCrud(Crud $crud): Crud
{
return $crud
->setPageTitle("index", "Etl Executions")
->setPageTitle('index', 'Etl Executions')
->setDateTimeFormat('dd/MM/y - HH:mm:ss')
->setSearchFields(["name", "id"])
->setSearchFields(['name', 'id'])
->setDefaultSort(['id' => 'DESC']);
}

public function configureFields(string $pageName): iterable
{
if (Crud::PAGE_DETAIL === $pageName) {
return [
FormField::addPanel("Details")->addCssClass("col-12 col-xl-6"),
FormField::addFieldset('Details')->addCssClass('col-12 col-xl-6'),
Field::new('name'),
Field::new('username'),
TextField::new('status')->setTemplatePath('@Oliverde8PhpEtlEasyAdmin/fields/status.html.twig'),
FormField::addPanel()->addCssClass("col-12 col-xl-6"),
FormField::addFieldset()->addCssClass('col-12 col-xl-6'),
Field::new('createTime'),
Field::new('startTime'),
Field::new('endTime'),
Field::new('failTime'),

FormField::addPanel('Execution Inputs')->addCssClass('col-12'),
FormField::addFieldset('Execution Inputs')->addCssClass('col-12'),
CodeEditorField::new('inputData')->setTemplatePath('@Oliverde8PhpEtlEasyAdmin/fields/code_editor.html.twig')->addCssClass('etl-json-div'),
CodeEditorField::new('inputOptions')->setTemplatePath('@Oliverde8PhpEtlEasyAdmin/fields/code_editor.html.twig')->addCssClass('etl-json-div'),
CodeEditorField::new('definition')->setTemplatePath('@Oliverde8PhpEtlEasyAdmin/fields/code_editor.html.twig'),

FormField::addPanel('Execution outpus')->addCssClass("col-12"),
TextField::new('Files')->formatValue(function ($value, EtlExecution $entity) {
FormField::addFieldset('Execution outpus')->addCssClass('col-12'),
TextField::new('Files')->formatValue(function ($value, EtlExecution $entity): array {
$urls = [];
if ($this->isGranted(EtlExecutionVoter::DOWNLOAD, EtlExecution::class)) {

$context = $this->executionContextFactory->get(['etl' => ['execution' => $entity]]);
$files = $context->getFileSystem()->listContents("/");
$files = $context->getFileSystem()->listContents('/');
foreach ($files as $file) {
if (strpos($file, '.') !== 0) {
if (!str_starts_with($file, '.')) {
$url = $this->adminUrlGenerator
->setRoute("etl_execution_download_file", ['execution' => $entity->getId(), 'filename' => $file])
->setRoute('etl_execution_download_file', ['execution' => $entity->getId(), 'filename' => $file])
->generateUrl();

$urls[$url] = $file;
Expand All @@ -119,39 +105,42 @@ public function configureFields(string $pageName): iterable
})->setTemplatePath('@Oliverde8PhpEtlEasyAdmin/fields/files.html.twig'),

CodeEditorField::new('errorMessage')->setTemplatePath('@Oliverde8PhpEtlEasyAdmin/fields/code_editor.html.twig'),
TextField::new('Logs')->formatValue(function ($value, EtlExecution $entity) {
TextField::new('Logs')->formatValue(function ($value, EtlExecution $entity): array {
$context = $this->executionContextFactory->get(['etl' => ['execution' => $entity]]);
$logs = [];
if ($context->getFileSystem()->fileExists("execution.log")) {
$file = $context->getFileSystem()->readStream("execution.log");
$i = 0;
$logs = [];
if ($context->getFileSystem()->fileExists('execution.log')) {
$file = $context->getFileSystem()->readStream('execution.log');
$i = 0;
while ($i < 100 && $line = fgets($file)) {
$logs[] = $line;
$i++;
++$i;
}

fclose($file);
}

$url = "";
$url = '';
$moreLogs = false;
if (!empty($logs)) {
if ([] !== $logs) {
$url = $this->adminUrlGenerator
->setRoute("etl_execution_download_file", ['execution' => $entity->getId(), 'filename' => 'execution.log'])
->setRoute('etl_execution_download_file', ['execution' => $entity->getId(), 'filename' => 'execution.log'])
->generateUrl();
}
if (count($logs) > 100) {

if (\count($logs) > 100) {
$moreLogs = true;
}

return [
"lines" => $logs,
'lines' => $logs,
'downloadUrl' => $url,
'moreLogs' => $moreLogs,
'moreLogs' => $moreLogs,
];
})->setTemplatePath('@Oliverde8PhpEtlEasyAdmin/fields/logs.html.twig'),

];
}

if (Crud::PAGE_INDEX === $pageName) {
return [
Field::new('id'),
Expand All @@ -163,12 +152,13 @@ public function configureFields(string $pageName): iterable
Field::new('endTime'),
];
}

if (Crud::PAGE_NEW === $pageName) {
return [
ChoiceField::new('name', 'Chain Name')
->setChoices($this->getChainOptions()),
CodeEditorField::new('inputData')->setCssClass("etl-json-input"),
CodeEditorField::new('inputOptions')->setCssClass("etl-json-input"),
CodeEditorField::new('inputData')->setCssClass('etl-json-input'),
CodeEditorField::new('inputOptions')->setCssClass('etl-json-input'),
];
}

Expand Down Expand Up @@ -200,16 +190,17 @@ public function configureFilters(Filters $filters): Filters
->add('endTime');
}

public function createEntity(string $entityFqcn)
public function createEntity(string $entityFqcn): object
{
$user = $this->getUser();
$user = $this->getUser();
$username = null;
if ($user) {
$username = $user->getUsername();
if ($user instanceof \Symfony\Component\Security\Core\User\UserInterface) {
$username = $user->getUserIdentifier();
}

$execution = new EtlExecution("", "", [], []);
$execution = new EtlExecution('', '', [], []);
$execution->setUsername($username);

return $execution;
}

Expand All @@ -219,10 +210,13 @@ public function persistEntity(EntityManagerInterface $entityManager, $entityInst
$entityManager->flush();
}

protected function getChainOptions()
/**
* @return int[]|string[]
*/
protected function getChainOptions(): array
{
$options = [];
foreach (array_keys($this->chainProcessorManager->getRewDefinitions()) as $definitionName) {
foreach (array_keys($this->chainProcessorManager->getRawDefinitions()) as $definitionName) {
$options[$definitionName] = $definitionName;
}

Expand Down
7 changes: 2 additions & 5 deletions DependencyInjection/Oliverde8PhpEtlEasyAdminExtension.php
Original file line number Diff line number Diff line change
@@ -1,20 +1,17 @@
<?php

declare(strict_types=1);

namespace Oliverde8\PhpEtlEasyAdminBundle\DependencyInjection;


use Symfony\Component\Config\FileLocator;
use Symfony\Component\DependencyInjection\ContainerBuilder;
use Symfony\Component\DependencyInjection\Extension\Extension;
use Symfony\Component\DependencyInjection\Loader\YamlFileLoader;

class Oliverde8PhpEtlEasyAdminExtension extends Extension
{
/**
* @inheritDoc
*/
public function load(array $configs, ContainerBuilder $container)
public function load(array $configs, ContainerBuilder $container): void
{
$loader = new YamlFileLoader($container, new FileLocator(\dirname(__DIR__).'/Resources/config'));
$loader->load('services.yml');
Expand Down
7 changes: 2 additions & 5 deletions Oliverde8PhpEtlEasyAdminBundle.php
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
<?php

declare(strict_types=1);

namespace Oliverde8\PhpEtlEasyAdminBundle;

use Oliverde8\PhpEtlBundle\DependencyInjection\Compiler\ChainBuilderCompiler;
use Oliverde8\PhpEtlBundle\DependencyInjection\Compiler\ChainCompiler;
use Oliverde8\PhpEtlBundle\DependencyInjection\Compiler\ChainParameterCompiler;
use Oliverde8\PhpEtlBundle\DependencyInjection\Compiler\RuleEngineCompiler;
use Symfony\Component\DependencyInjection\ContainerBuilder;
use Symfony\Component\HttpKernel\Bundle\Bundle;

class Oliverde8PhpEtlEasyAdminBundle extends Bundle
Expand Down
Loading