diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..59629746 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,158 @@ + +# Created by https://www.toptal.com/developers/gitignore/api/python,jupyternotebooks +# Edit at https://www.toptal.com/developers/gitignore?templates=python,jupyternotebooks + +/.idea + +### JupyterNotebooks ### +# gitignore template for Jupyter Notebooks +# website: http://jupyter.org/ + +.ipynb_checkpoints +*/.ipynb_checkpoints/* + +# IPython +profile_default/ +ipython_config.py + +# Remove previous ipynb_checkpoints +# git rm -r .ipynb_checkpoints/ + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook + +# IPython + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# End of https://www.toptal.com/developers/gitignore/api/python,jupyternotebooks diff --git a/docs/.readthedocs.yaml b/docs/.readthedocs.yaml new file mode 100644 index 00000000..2537dd43 --- /dev/null +++ b/docs/.readthedocs.yaml @@ -0,0 +1,13 @@ +version: 2 + +build: + os: "ubuntu-22.04" + tools: + python: "3.10" + +python: + install: + - requirements: requirements.txt + +sphinx: + configuration: docs/conf.py diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..b97de95f --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = docs +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/README.md b/docs/README.md index bf030adb..5a659824 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,64 +1,22 @@ -## Installation +# Yokai Batch documentation -```bash -composer require yokai/batch -``` +Sources of documentation for [`yokai/batch`](https://github.com/yokai-php/batch) library. +See [online version](https://yokai-batch.readthedocs.io/). -## What is Batch ? -The Batch library solves all your massive data processing problems. -Batch can also be used as an ETL. +## Contribution -Batch can also work asynchronously. +This package is a readonly split of a [larger repository](https://github.com/yokai-php/batch-src), +containing all tests and sources for all librairies of the batch universe. -Batch help you to make reporting during process. +Please feel free to open an [issue](https://github.com/yokai-php/batch-src/issues) +or a [pull request](https://github.com/yokai-php/batch-src/pulls) +in the [main repository](https://github.com/yokai-php/batch-src). -## How it works ? -Batch is a library that allows you to declare and execute jobs. -And having control over each step of the process to be able to extend the technical logic to meet business needs. +The library was originally created by [Yann Eugoné](https://github.com/yann-eugone). +See the list of [contributors](https://github.com/yokai-php/batch-src/contributors). -## Vocabulary -Because when you start with any library -it is important to understand what are the concepts introduced in it. +## License -This is highly recommended that you read this entire page -before starting to work with this library. - -### Job -A job is the class that is responsible for **what** your code is doing. - -This is the class you will have to create (or reuse), -as it contains the business logic required for what you wish to achieve. - -The only requirement is implementing [`JobInterface`](../src/batch/src/Job/JobInterface.php), - -#### See More: -For more information about jobs, see [Job](batch/domain/job.md) - -### Job Launcher - -The job launcher is responsible for executing/scheduling every jobs. - -Yeah, executing OR scheduling. There is multiple implementation of a job launcher across bridges. -Job's execution might be asynchronous, and thus, when you ask the job launcher to "launch" a job, -you have to check the `JobExecution` status that it had returned to know if the job is already executed. - -#### What is needed to use a Job Launcher ? - -For use a Job Launcher you need to have: -- **A Container of Jobs:** A container use any PSR-11 [container implementation](https://packagist.org/providers/psr/container-implementation) -- **A Storage for Job Executions:** A storage to store the execution of jobs, it can be a database, a file, a cache, etc. -- **A JobExecutor:** The executor will execute the job. -- **A JobExecutionAccessor:** The accessor will access the job execution. - -### JobExecution: - -A [JobExecution](../src/batch/src/JobExecution.php) is the class that holds information about one execution of a job. - -### JobExecutionStorage - -Whenever a job is launched, whether is starts immediately or not, an execution is stored for it. -The execution are stored to allow you to keep an eye on what is happening. -This persistence is on the responsibility of the job execution storage. -- [Job Execution Storage](batch/domain/job-execution-storage.md) +This library is under MIT [LICENSE](LICENSE). diff --git a/docs/batch-doctrine-dbal/job-execution-storage.md b/docs/batch-doctrine-dbal/job-execution-storage.md deleted file mode 100644 index 8f093895..00000000 --- a/docs/batch-doctrine-dbal/job-execution-storage.md +++ /dev/null @@ -1,3 +0,0 @@ -# Job execution storage - -todo diff --git a/docs/batch-doctrine-orm/entity-item-reader.md b/docs/batch-doctrine-orm/entity-item-reader.md deleted file mode 100644 index d71dfcbb..00000000 --- a/docs/batch-doctrine-orm/entity-item-reader.md +++ /dev/null @@ -1,3 +0,0 @@ -# Entity item reader - -todo diff --git a/docs/batch-doctrine-persistence/object-item-writer.md b/docs/batch-doctrine-persistence/object-item-writer.md deleted file mode 100644 index c0755e1f..00000000 --- a/docs/batch-doctrine-persistence/object-item-writer.md +++ /dev/null @@ -1,3 +0,0 @@ -# Object item writer - -todo diff --git a/docs/batch-doctrine-persistence/object-registry.md b/docs/batch-doctrine-persistence/object-registry.md deleted file mode 100644 index c4cb0ecb..00000000 --- a/docs/batch-doctrine-persistence/object-registry.md +++ /dev/null @@ -1,81 +0,0 @@ -# Object registry - -Imagine that in an `ItemJob` you need to find objects from a database. - -```php -use App\Entity\Product; -use Doctrine\Persistence\ObjectRepository; -use Yokai\Batch\Job\Item\ItemProcessorInterface; - -class DenormalizeProductProcessor implements ItemProcessorInterface -{ - public function __construct( - private ObjectRepository $repository, - ) { - } - - /** - * @param array $item - */ - public function process(mixed $item): Product - { - $product = $this->repository->findOneBy(['sku' => $item['sku']]); - - $product ??= new Product($item['sku']); - - $product->setName($item['name']); - $product->setPrice($item['price']); - //... - - return $product; - } -} -``` - -The problem here is that every time you will call `findOneBy`, you will have to query the database. -The object might already be in Doctrine's memory, so it won't be hydrated twice, but the query will be done every time. - -The role of the `ObjectRegistry` is to remember found objects identities, and query these objects with it instead. - -```diff -use App\Entity\Product; --use Doctrine\Persistence\ObjectRepository; -+use Yokai\Batch\Bridge\Doctrine\Persistence\ObjectRegistry; -use Yokai\Batch\Job\Item\ItemProcessorInterface; - -class DenormalizeProductProcessor implements ItemProcessorInterface -{ - public function __construct( -- private ObjectRepository $repository, -+ private ObjectRegistry $registry, - ) { - } - - /** - * @param array $item - */ - public function process(mixed $item): Product - { -- $product = $this->repository->findOneBy(['sku' => $item['sku']]); -+ $product = $this->registry->findOneBy(Product::class, ['sku' => $item['sku']]); - - $product ??= new Product($item['sku']); - - $product->setName($item['name']); - $product->setPrice($item['price']); - //... - - return $product; - } -} -``` - -The first time, the query will hit the database, and the object identity will be remembered in the registry. -Everytime after that, the registry will call `Doctrine\Persistence\ObjectManager::find` instead. -If the object is still in Doctrine's memory, it will be returned directly. -Otherwise, the query will be the fastest possible because it will use the object identity. - - -## On the same subject - -- [What is an item job ?](../batch/domain/item-job.md) diff --git a/docs/batch-league-flysystem/copy-files-job.md b/docs/batch-league-flysystem/copy-files-job.md deleted file mode 100644 index 258cd572..00000000 --- a/docs/batch-league-flysystem/copy-files-job.md +++ /dev/null @@ -1 +0,0 @@ -todo diff --git a/docs/batch-league-flysystem/move-files-job.md b/docs/batch-league-flysystem/move-files-job.md deleted file mode 100644 index 258cd572..00000000 --- a/docs/batch-league-flysystem/move-files-job.md +++ /dev/null @@ -1 +0,0 @@ -todo diff --git a/docs/batch-openspout/flat-file-item-reader.md b/docs/batch-openspout/flat-file-item-reader.md deleted file mode 100644 index bc353c9c..00000000 --- a/docs/batch-openspout/flat-file-item-reader.md +++ /dev/null @@ -1,48 +0,0 @@ -# Item reader with CSV/ODS/XLSX files - -The [FlatFileReader](../../src/batch-openspout/src/Reader/FlatFileReader.php) is a reader -that will read from CSV/ODS/XLSX file and return each line as an array. - -```php -FIELD_DELIMITER = ';'; -$options->FIELD_ENCLOSURE = '|'; -new FlatFileReader( - new StaticValueParameterAccessor('/path/to/file.csv'), - $options, - null, - HeaderStrategy::none(), -); - -// Read .ods file -// Only sheet named "Sheet name to read" will be read -// Each item will be an array_combine of first line as key and line as values -new FlatFileReader( - new StaticValueParameterAccessor('/path/to/file.ods'), - null, - SheetFilter::nameIs('Sheet name to read'), - HeaderStrategy::combine(), -); -``` - -## On the same subject - -- [What is an item reader ?](../batch/domain/item-job/item-reader.md) diff --git a/docs/batch-openspout/flat-file-item-writer.md b/docs/batch-openspout/flat-file-item-writer.md deleted file mode 100644 index 80f03925..00000000 --- a/docs/batch-openspout/flat-file-item-writer.md +++ /dev/null @@ -1,50 +0,0 @@ -# Item writer with CSV/ODS/XLSX files - -The [FlatFileWriter](../../src/batch-openspout/src/Writer/FlatFileWriter.php) is a writer that will write to CSV/ODS/XLSX file and each item will -written its own line. - -```php -FIELD_DELIMITER = ';'; -$options->FIELD_ENCLOSURE = '|'; -new FlatFileWriter( - new StaticValueParameterAccessor('/path/to/file.csv'), - $options, -); - -// Write items to .ods file -// That file will contain a header line with : static | header | keys -// Change the sheet name data will be written -// Change the default style of each cell -$options = new ODSOptions(); -$options->DEFAULT_ROW_STYLE = (new Style())->setFontBold(); -new FlatFileWriter( - new StaticValueParameterAccessor('/path/to/file.ods'), - $options, - 'The sheet name', - ['static', 'header', 'keys'], -); -``` - -## On the same subject - -- [What is an item writer ?](../batch/domain/item-job/item-writer.md) diff --git a/docs/batch-symfony-console/command.md b/docs/batch-symfony-console/command.md deleted file mode 100644 index 369f0988..00000000 --- a/docs/batch-symfony-console/command.md +++ /dev/null @@ -1,13 +0,0 @@ -# Command - -The [RunJobCommand](../../src/batch-symfony-console/src/RunJobCommand.php) can execute any job. - -The command accepts 2 arguments : -- the job name to execute -- the job parameters for the `JobExecution` (optional) - -Examples : -``` -bin/console import -bin/console export '{"toFile":"/path/to/file.xml"}' -``` diff --git a/docs/batch-symfony-console/job-launcher.md b/docs/batch-symfony-console/job-launcher.md deleted file mode 100644 index 7dcebaf7..00000000 --- a/docs/batch-symfony-console/job-launcher.md +++ /dev/null @@ -1,12 +0,0 @@ -# Job launcher - -The [RunCommandJobLauncher](../../src/batch-symfony-console/src/RunCommandJobLauncher.php) execute jobs via an asynchronous symfony command. - -The command called is [`yokai:batch:run`](command.md), and the command will actually execute the job. - -Additionally, the command will run with an output redirect (`>>`) to `var/log/batch_execute.log`. - - -## On the same subject - -- [What is a job launcher ?](../batch/domain/job-launcher.md) diff --git a/docs/batch-symfony-framework/getting-started.md b/docs/batch-symfony-framework/getting-started.md deleted file mode 100644 index c411d446..00000000 --- a/docs/batch-symfony-framework/getting-started.md +++ /dev/null @@ -1,188 +0,0 @@ -# Getting started - -## Configuring the bundle - -```php -// config/bundles.php -return [ - // ... - Yokai\Batch\Bridge\Symfony\Framework\YokaiBatchBundle::class => ['all' => true], -]; -``` - -### Job launcher - -You can use many different job launcher in your application, you will be able to register these using configuration: - -```yaml -# config/packages/yokai_batch.yaml -yokai_batch: - launcher: - default: simple - launchers: - simple: ... - async: ... -``` - -> **note**: if you do not configure anything here, you will be using the [`SimpleJobLauncher`](../../src/batch/src/Launcher/SimpleJobLauncher.php). - -The `default` job launcher, must reference a launcher name, defined in the `launchers` list. -The `default` job launcher will be the autowired instance of job launcher when you ask for one. -All `launchers` will be registered as a service, and an autowire named alias will be configured for it. -For instance, in the example below, you will be able to register all these launchers like this: - -```php - **note**: the default storage is `filesystem`, because it only requires a writeable filesystem. -> But if you already have `doctrine/dbal` in your project, it is highly recommended to use it instead. -> Because querying `JobExecution` in a filesystem might be slow, specially if you are planing to add UIs on top. - -### Job as a service - -As Symfony supports registering all classes in `src/` as a service, -we can leverage this behaviour to register all jobs in `src/`. -We will add a tag to every found class in `src/` that implements `Yokai\Batch\Job\JobInterface`: - -```yaml -# config/services.yaml -services: - _defaults: - _instanceof: - Yokai\Batch\Job\JobInterface: - tags: ['yokai_batch.job'] -``` - -## Your first job - -In a Symfony project, we will prefer using one class per job, because service discovery is so easy to use. -But also because it will be very far easier to configure your job using PHP than any other format. -For instance, there is components that uses `Closure`, has static constructors, ... -But keep in mind you can register your jobs with any other format of your choice. - -```php - **note**: when registering jobs with dedicated class, you can use the -> [JobWithStaticNameInterface](../../src/batch-symfony-framework/src/JobWithStaticNameInterface.php) interface -> to be able to specify the job name of your service. -> Otherwise, the service id will be used, and in that case, the service id is the FQCN. - -### Triggering the job -Then the job will be triggered with its name (or service id when not specified): - -```php -jobLauncher->launch('job.name'); - } -} -``` - -The job launcher that will be injected depends on the packages you have installed, order matter: -- if `yokai/batch-symfony-messenger` is installed, you will receive a `Yokai\Batch\Bridge\Symfony\Messenger\DispatchMessageJobLauncher` -- if `yokai/batch-symfony-console` is installed, you will receive a `Yokai\Batch\Bridge\Symfony\Console\RunCommandJobLauncher` -- otherwise you will receive a `Yokai\Batch\Launcher\SimpleJobLauncher` - - -## Use the batchLogger -The aim of the batchLogger is to store log inside the jobExecution. -In a symfony project, you can use the batchLogger with the symfony autowiring by naming your variable as `$yokaiBatchLogger` - -```php -yokaiBatchLogger->error(...); - } -} -``` - -## On the same subject - -- [What is a job execution storage ?](../batch/domain/job-execution-storage.md) -- [What is a job ?](../batch/domain/job.md) -- [What is a job launcher ?](../batch/domain/job-launcher.md) diff --git a/docs/batch-symfony-framework/ui.md b/docs/batch-symfony-framework/ui.md deleted file mode 100644 index dca146a2..00000000 --- a/docs/batch-symfony-framework/ui.md +++ /dev/null @@ -1,196 +0,0 @@ -# User Interface - -The package is shipped with few routes that will allow you and your users, to watch for `JobExecution`. - -Bootstrap 4 - List action Bootstrap 4 - Detail : Information Bootstrap 4 - Detail : Children Bootstrap 4 - Detail : Warnings - - -## Installation - -For the UI to be enabled, it is required that you install some dependencies: -```shell -composer require symfony/translation symfony/twig-bundle -``` - - -## Configuration - -The UI is disabled by default, you must enable it explicitely: -```yaml -# config/packages/yokai_batch.yaml -yokai_batch: - ui: - enabled: true -``` - -You will also need to import bundle routes: -```yaml -# config/routes/yokai_batch.yaml -_yokai_batch: - resource: "@YokaiBatchBundle/Resources/routing/ui.xml" -``` - -### Templating - -The templating service is used by the [JobController](../../src/batch-symfony-framework/src/UserInterface/Controller/JobController.php) to render its templates. -It's a wrapper around [Twig](https://twig.symfony.com/), for you to control templates used, and variables passed. - -> By default -> - the templating will find templates like `@YokaiBatch/bootstrap4/*.html.twig` -> - the template base view will be `base.html.twig` - -You can configure a prefix for all templates: -```yaml -# config/packages/yokai_batch.yaml -yokai_batch: - ui: - templating: - prefix: 'batch/job/' -``` -> With this configuration, we will look for templates like `batch/job/*.html.twig`. - -You can also configure the name of the base template for the root views of that bundle: -```yaml -# config/packages/yokai_batch.yaml -yokai_batch: - ui: - templating: - base_template: 'layout.html.twig' -``` -> With this configuration, the template base view will be `layout.html.twig`. - -If these are not enough, or if you need to add more variables to context, you can configure a service: -```yaml -# config/packages/yokai_batch.yaml -yokai_batch: - ui: - templating: - service: 'App\Batch\AppTemplating' -``` - -And create the class that will cover the templating: -```php - 'bar']); // add variables to $context if you want - } -} -``` - -> **Note** You can also use the `Yokai\Batch\Bridge\Symfony\Framework\UserInterface\Templating\ConfigurableTemplating` that will cover both prefix and static variables at construction. - - -### Filtering - -The `JobExecution` list includes a filter form, but you will need another optional dependency: -```shell -composer require symfony/form -``` - -### Security - -There is no access control over `JobExecution` by default, you will need another optional dependency: -```shell -composer require symfony/security-bundle -``` - -Every security attribute the bundle is using is configurable: -```yaml -# config/packages/yokai_batch.yaml -yokai_batch: - ui: - security: - attributes: - list: ROLE_JOB_LIST # defaults to IS_AUTHENTICATED - view: ROLE_JOB_VIEW # defaults to IS_AUTHENTICATED - traces: ROLE_JOB_TRACES # defaults to IS_AUTHENTICATED - logs: ROLE_JOB_LOGS # defaults to IS_AUTHENTICATED -``` - -Optionally, you can register a voter for these attributes. -This is especially useful if you need different access control rules per `JobExecution`. -```php - Sonata - Detail : Information Sonata - Detail : Children Sonata - Detail : Warnings - -```shell -composer require sonata-project/admin-bundle -``` - -```yaml -# config/packages/yokai_batch.yaml -yokai_batch: - ui: - templating: sonata -``` -> With this configuration, we will look for templates like `@YokaiBatch/sonata/*.html.twig`. - - -## Customizing templates - -You can override templates like [described it Symfony's documentation](https://symfony.com/doc/current/bundles/override.html). -Examples: -- `templates/bundles/YokaiBatchBundle/bootstrap4/list.html.twig` -- `templates/bundles/YokaiBatchBundle/bootstrap4/show/_parameters.html.twig` - -But you can also register job name dedicated templates if you need some specific view for one of your jobs: -- `templates/bundles/YokaiBatchBundle/bootstrap4/show/{job name}/_children-executions.html.twig` -- `templates/bundles/YokaiBatchBundle/bootstrap4/show/{job name}/_failures.html.twig` -- `templates/bundles/YokaiBatchBundle/bootstrap4/show/{job name}/_general.html.twig` -- `templates/bundles/YokaiBatchBundle/bootstrap4/show/{job name}/_information.html.twig` -- `templates/bundles/YokaiBatchBundle/bootstrap4/show/{job name}/_parameters.html.twig` -- `templates/bundles/YokaiBatchBundle/bootstrap4/show/{job name}/_summary.html.twig` -- `templates/bundles/YokaiBatchBundle/bootstrap4/show/{job name}/_warnings.html.twig` - -## On the same subject - -- [What is a job execution storage ?](../batch/domain/job-execution-storage.md) -- [What is a job ?](../batch/domain/job.md) -- [What is a job launcher ?](../batch/domain/job-launcher.md) diff --git a/docs/batch-symfony-messenger/dispatch-each-item-writer.md b/docs/batch-symfony-messenger/dispatch-each-item-writer.md deleted file mode 100644 index 5d21cfd5..00000000 --- a/docs/batch-symfony-messenger/dispatch-each-item-writer.md +++ /dev/null @@ -1 +0,0 @@ -todo \Yokai\Batch\Bridge\Symfony\Messenger\Writer\DispatchEachItemAsMessageWriter diff --git a/docs/batch-symfony-messenger/job-launcher.md b/docs/batch-symfony-messenger/job-launcher.md deleted file mode 100644 index b7e09a01..00000000 --- a/docs/batch-symfony-messenger/job-launcher.md +++ /dev/null @@ -1,32 +0,0 @@ -# Job launcher - -The [DispatchMessageJobLauncher](../../src/batch-symfony-messenger/src/DispatchMessageJobLauncher.php) execute -jobs via a symfony command message dispatch. - -A [LaunchJobMessage](../../src/batch-symfony-messenger/src/LaunchJobMessage.php) message will be dispatched -and handled by the [LaunchJobMessageHandler](../../src/batch-symfony-messenger/src/LaunchJobMessageHandler.php) -will be called with that message after being routed. - -## How to configure an async transport for the launcher ? - -You first need to configure an async transport, like explained -[in Symfony's doc](https://symfony.com/doc/current/messenger.html#transports-async-queued-messages). - -Then you will have to route the message to this async transport you created, like explained -[in Symfony's doc](https://symfony.com/doc/current/messenger.html#routing-messages-to-a-transport). - -You will end with something like : - -```yaml -# config/packages/messenger.yaml -framework: - messenger: - transports: - async: "%env(MESSENGER_TRANSPORT_DSN)%" - routing: - 'Yokai\Batch\Bridge\Symfony\Messenger\LaunchJobMessage': async -``` - -## On the same subject - -- [What is a job launcher ?](../batch/domain/job-launcher.md) diff --git a/docs/batch-symfony-serializer/denormalize-item-processor.md b/docs/batch-symfony-serializer/denormalize-item-processor.md deleted file mode 100644 index 258cd572..00000000 --- a/docs/batch-symfony-serializer/denormalize-item-processor.md +++ /dev/null @@ -1 +0,0 @@ -todo diff --git a/docs/batch-symfony-serializer/job-execution-serializer.md b/docs/batch-symfony-serializer/job-execution-serializer.md deleted file mode 100644 index 258cd572..00000000 --- a/docs/batch-symfony-serializer/job-execution-serializer.md +++ /dev/null @@ -1 +0,0 @@ -todo diff --git a/docs/batch-symfony-serializer/normalize-item-processor.md b/docs/batch-symfony-serializer/normalize-item-processor.md deleted file mode 100644 index 258cd572..00000000 --- a/docs/batch-symfony-serializer/normalize-item-processor.md +++ /dev/null @@ -1 +0,0 @@ -todo diff --git a/docs/batch-symfony-validator/skip-invalid-item-processor.md b/docs/batch-symfony-validator/skip-invalid-item-processor.md deleted file mode 100644 index 258cd572..00000000 --- a/docs/batch-symfony-validator/skip-invalid-item-processor.md +++ /dev/null @@ -1 +0,0 @@ -todo diff --git a/docs/batch/domain/item-job.md b/docs/batch/domain/item-job.md deleted file mode 100644 index 4e8e5c59..00000000 --- a/docs/batch/domain/item-job.md +++ /dev/null @@ -1,66 +0,0 @@ -# Item Job - -## What is an item ? - -The library allows you to declare and execute jobs, but wait why do we named it batch then ? -There you are, the `ItemJob` is where batch processing actually starts. - -This is just a job that has been prepared to batch handle items. - -If you are familiar with the concept of an [ETL](https://en.wikipedia.org/wiki/Extract,_transform,_load), -this is pretty much the same. - -The item job allows you to split your logic into 3 different component : -- an item [reader](item-job/item-reader.md): stands for **Extract** in ETL -- an item [processor](item-job/item-processor.md): stands for **Transform** in ETL -- an item [writer](item-job/item-writer.md): stands for **Load** in ETL - -## How to create an item job ? - -```php - **Note:** Sometimes the storage may implement these interfaces, but -> due to the way executions are stored, it might not be recommended heavily rely on these extra methods. - -## What types of storages exists ? - -**Built-in storages:** -- [NullJobExecutionStorage](../../../src/batch/src/Storage/NullJobExecutionStorage.php): - do not persist any job execution. -- [FilesystemJobExecutionStorage](../../../src/batch/src/Storage/FilesystemJobExecutionStorage.php): - store job executions to a file on local filesystem. - -**Storages from bridges:** -- [DoctrineDBALJobExecutionStorage (`doctrine/dbal`)](../../../src/batch-doctrine-dbal/src/DoctrineDBALJobExecutionStorage.php): - store job executions to a relational database. - -**Storages for testing purpose:** -- [InMemoryJobExecutionStorage](../../../src/batch/src/Test/Storage/InMemoryJobExecutionStorage.php): - store executions in a private var that can be accessed afterwards in your tests. - -## On the same subject - -- [How do I create my own storage ?](../recipes/custom-job-execution-storage.md) diff --git a/docs/batch/domain/job-execution.md b/docs/batch/domain/job-execution.md deleted file mode 100644 index ed007ddd..00000000 --- a/docs/batch/domain/job-execution.md +++ /dev/null @@ -1,24 +0,0 @@ -# Job execution - -## What is a Job execution ? - -A [JobExecution](../../../src/batch/src/JobExecution.php) is the class that holds information about one execution of a [job](job.md). - -## What kind of information does it hold ? - -- `JobExecution::$jobName` : The Job name (job id) -- `JobExecution::$id` : The execution id -- `JobExecution::$parameters` : Some parameters with which job was executed -- `JobExecution::$status` : A status (pending, running, stopped, completed, abandoned, failed) -- `JobExecution::$startTime` : Start time -- `JobExecution::$endTime` : End time -- `JobExecution::$failures` : A list of failures (usually exceptions) -- `JobExecution::$warnings` : A list of warnings (usually skipped items) -- `JobExecution::$summary` : A summary (can contain any data you wish to store) -- `JobExecution::$logs` : Some logs -- `JobExecution::$childExecutions` : Some child execution - -## On the same subject - -- [How do I create a job execution ?](job-launcher.md) -- [How do I get a job execution afterwards ?](job-execution-storage.md) diff --git a/docs/batch/domain/job-launcher.md b/docs/batch/domain/job-launcher.md deleted file mode 100644 index 69007b3f..00000000 --- a/docs/batch/domain/job-launcher.md +++ /dev/null @@ -1,68 +0,0 @@ -# Job Launcher - -## What is a job launcher ? - -The job launcher is responsible for executing/scheduling every jobs. - -Yeah, executing OR scheduling. There is multiple implementation of a job launcher across bridges. -Job's execution might be asynchronous, and thus, when you ask the job launcher to "launch" a job, -you have to check the `JobExecution` status that it had returned to know if the job is already executed. - -## What is the simplest way to launch a job ? - -```php - new class implements JobInterface { - public function execute(JobExecution $jobExecution): void - { - // your business logic - } - }, -]); -$jobExecutionStorage = new NullJobExecutionStorage(); - -$launcher = new SimpleJobLauncher( - new JobExecutionAccessor(new JobExecutionFactory(new UniqidJobExecutionIdGenerator()), $jobExecutionStorage), - new JobExecutor(new JobRegistry($jobs), $jobExecutionStorage, null), -); - -$execution = $launcher->launch('your.job.name', ['job' => ['configuration']]); -``` - -## What types of launcher exists ? - -**Built-in launchers:** -- [SimpleJobLauncher](../../../src/batch/src/Launcher/SimpleJobLauncher.php): - execute the job directly in the same PHP process. - -**Launchers from bridges:** -- [RunCommandJobLauncher (`symfony/console`)](../../../src/batch-symfony-console/src/RunCommandJobLauncher.php): - execute the job via an asynchronous symfony command. -- [DispatchMessageJobLauncher (`symfony/messenger`)](../../../src/batch-symfony-messenger/src/DispatchMessageJobLauncher.php): - execute the job via a symfony message dispatch. - -**Launchers for testing purpose:** -- [BufferingJobLauncher](../../../src/batch/src/Test/Launcher/BufferingJobLauncher.php): - do not execute job, but store execution in a private var that can be accessed afterwards in your tests. - -## On the same subject - -- [What is a job ?](job.md) -- [What is a job execution ?](job-execution.md) diff --git a/docs/batch/domain/job-parameter-accessor.md b/docs/batch/domain/job-parameter-accessor.md deleted file mode 100644 index ce9ffe57..00000000 --- a/docs/batch/domain/job-parameter-accessor.md +++ /dev/null @@ -1,63 +0,0 @@ -# Job parameter accessor - -When a job (or a component within a job) can be working with a parameterized value, -it can rely on a [JobParameterAccessorInterface](../../../src/batch/src/Job/Parameters/JobParameterAccessorInterface.php) -instance to retrieve that value. - -```php -path->get($jobExecution); - // do something with $path - } -} -``` -## What types of parameter accessors exists ? - -**Built-in parameter accessors:** -- [ChainParameterAccessor.php](../../../src/batch/src/Job/Parameters/ChainParameterAccessor.php): - try multiple parameter accessors, the first that is not failing is used. -- [ClosestJobExecutionAccessor](../../../src/batch/src/Job/Parameters/ClosestJobExecutionAccessor.php): - try another parameter accessor on each job execution in hierarchy, until not failed. -- [DefaultParameterAccessor](../../../src/batch/src/Job/Parameters/DefaultParameterAccessor.php): - try accessing parameter using another parameter accessor, use default value if failed. -- [JobExecutionParameterAccessor](../../../src/batch/src/Job/Parameters/JobExecutionParameterAccessor.php): - extract value from job execution's [parameters](../../../src/batch/src/JobParameters.php). -- [JobExecutionSummaryAccessor](../../../src/batch/src/Job/Parameters/JobExecutionSummaryAccessor.php): - extract value from job execution's [summary](../../../src/batch/src/Summary.php). -- [ParentJobExecutionAccessor](../../../src/batch/src/Job/Parameters/ParentJobExecutionAccessor.php): - use another parameter accessor on job execution's parent execution. -- [ReplaceWithVariablesParameterAccessor](../../../src/batch/src/Job/Parameters/ReplaceWithVariablesParameterAccessor.php): - use another parameter accessor to get string value, and replace variables before returning. -- [RootJobExecutionAccessor](../../../src/batch/src/Job/Parameters/RootJobExecutionAccessor.php): - use another parameter accessor on job execution's root execution. -- [SiblingJobExecutionAccessor](../../../src/batch/src/Job/Parameters/SiblingJobExecutionAccessor.php): - use another parameter accessor on job execution's sibling execution. -- [StaticValueParameterAccessor](../../../src/batch/src/Job/Parameters/StaticValueParameterAccessor.php): - use static value provided at construction. - -**Parameter accessors from bridges:** -- [ContainerParameterAccessor (`symfony/framework-bundle`)](../../../src/batch-symfony-framework/src/ContainerParameterAccessor.php): - use a parameter from Symfony's container. - -## On the same subject - -- [What is a job ?](job.md) -- [When does a job execution hierarchy is created ?](job-with-children.md) -- [What is a job execution ?](job-execution.md) diff --git a/docs/batch/domain/job-with-children.md b/docs/batch/domain/job-with-children.md deleted file mode 100644 index 0a74aa48..00000000 --- a/docs/batch/domain/job-with-children.md +++ /dev/null @@ -1,44 +0,0 @@ -# Job with children - -todo - -```php - new class implements JobInterface { - public function execute(JobExecution $jobExecution): void - { - $exportSince = new DateTimeImmutable($jobExecution->getParameter('since')); - // your export logic here - } - }, - 'upload' => new class implements JobInterface { - public function execute(JobExecution $jobExecution): void - { - $pathToUploadExportedFile = $jobExecution->getParameter('path'); - // your upload logic here - } - }, -]); - -$job = new JobWithChildJobs( - $jobExecutionStorage = new NullJobExecutionStorage(), - new JobExecutor(new JobRegistry($container), $jobExecutionStorage, null), - ['export', 'upload'] -); -``` diff --git a/docs/batch/domain/job.md b/docs/batch/domain/job.md deleted file mode 100644 index 66d1c44f..00000000 --- a/docs/batch/domain/job.md +++ /dev/null @@ -1,53 +0,0 @@ -# Job - -## What is a job ? - -A job is the class that is responsible for **what** your code is doing. - -This is the class you will have to create (or reuse), -as it contains the business logic required for what you wish to achieve. - -## How to create a job ? - -```php - new class implements JobInterface { - public function execute(JobExecution $jobExecution): void - { - $fileToImport = $jobExecution->getParameter('path'); - // your import logic here - } - }, - 'export' => new class implements JobInterface { - public function execute(JobExecution $jobExecution): void - { - $exportSince = new DateTimeImmutable($jobExecution->getParameter('since')); - // your export logic here - } - }, -]); - -$jobExecutionStorage = new FilesystemJobExecutionStorage(new JsonJobExecutionSerializer(), '/dir/where/jobs/are/stored'); -$launcher = new SimpleJobLauncher( - new JobExecutionAccessor( - new JobExecutionFactory(new UniqidJobExecutionIdGenerator()), - $jobExecutionStorage - ), - new JobExecutor( - new JobRegistry($container), - $jobExecutionStorage, - null // or an instance of \Psr\EventDispatcher\EventDispatcherInterface - ) -); - -// now you can use $launcher to start any job you registered in $container - -$importExecution = $launcher->launch('import', ['path' => '/path/to/file/to/import']); -$exportExecution = $launcher->launch('export', ['since' => '2020-07-03']); -``` diff --git a/docs/batch/recipes/aware-interfaces.md b/docs/batch/recipes/aware-interfaces.md deleted file mode 100644 index 851a6e15..00000000 --- a/docs/batch/recipes/aware-interfaces.md +++ /dev/null @@ -1,56 +0,0 @@ -# *Aware interfaces - -When a job execution starts, a [JobExecution](../../../src/batch/src/JobExecution.php) is created for it. -This object contains information about the current execution. - -You will often want to access this object or one of its child to : -- access provided user parameters in your components -- leave some information on the job execution: logs, summary, warning... - -To do that, your component will need to implement an interface, telling the library that you need something. - -## What is `JobExecutionAwareInterface` ? - -The [JobExecutionAwareInterface](../../../src/batch/src/Job/JobExecutionAwareInterface.php) -will allow you to gain access to the current [JobExecution](../../../src/batch/src/JobExecution.php). - -> **note:** this interface is covered by [JobExecutionAwareTrait](../../../src/batch/src/Job/JobExecutionAwareTrait.php) -> for a default implementation that is most of the time sufficient. - -## What is `JobParametersAwareInterface` ? - -The [JobParametersAwareInterface](../../../src/batch/src/Job/JobParametersAwareInterface.php) -will allow you to gain access to the [JobParameters](../../../src/batch/src/JobParameters.php) of the current execution. - -> **note:** this interface is covered by [JobParametersAwareTrait](../../../src/batch/src/Job/JobParametersAwareTrait.php) -> for a default implementation that is most of the time sufficient. - -## What is `SummaryAwareInterface` ? - -The [SummaryAwareInterface](../../../src/batch/src/Job/SummaryAwareInterface.php) -will allow you to gain access to the [Summary](../../../src/batch/src/Summary.php) of the current execution. - -> **note:** this interface is covered by [SummaryAwareTrait](../../../src/batch/src/Job/SummaryAwareTrait.php) -> for a default implementation that is most of the time sufficient. - -## How does that work exactly ? - -There is no magic involved here, -every component is responsible for propagating the context through these interfaces. - -In the library, you will find that : -- [ItemJob](../../../src/batch/src/Job/Item/ItemJob.php) is propagating context to - [ItemReaderInterface](../../../src/batch/src/Job/Item/ItemReaderInterface.php), - [ItemProcessorInterface](../../../src/batch/src/Job/Item/ItemProcessorInterface.php) and - [ItemWriterInterface](../../../src/batch/src/Job/Item/ItemWriterInterface.php).7 -- Every - [ItemReaderInterface](../../../src/batch/src/Job/Item/ItemReaderInterface.php), - [ItemProcessorInterface](../../../src/batch/src/Job/Item/ItemProcessorInterface.php) and - [ItemWriterInterface](../../../src/batch/src/Job/Item/ItemWriterInterface.php) - acting as a decorator, is propagating context to their decorated element. - -You can add this interface to any class, but you are responsible for the context propagation. - -## On the same subject - -- [What is an ItemJob ?](../domain/item-job.md) diff --git a/docs/batch/recipes/custom-job-execution-storage.md b/docs/batch/recipes/custom-job-execution-storage.md deleted file mode 100644 index 7dd252f4..00000000 --- a/docs/batch/recipes/custom-job-execution-storage.md +++ /dev/null @@ -1,42 +0,0 @@ -# Create a Job Execution Storage - -```php -memory[$execution->getJobName()][$execution->getId()] = $execution; - } - - public function remove(JobExecution $execution) : void - { - unset( - $this->memory[$execution->getJobName()][$execution->getId()] - ); - } - - public function retrieve(string $jobName, string $executionId) : JobExecution - { - $execution = $this->memory[$jobName][$executionId] ?? null; - if ($execution === null) { - throw new JobExecutionNotFoundException($jobName, $executionId); - } - - return $execution; - } -} -``` - -## On the same subject - -- [What is a Job Execution Storage ?](../domain/job-execution-storage.md) diff --git a/docs/batch-symfony-framework/images/bootstrap4-children.png b/docs/docs/_static/images/symfony/ui/bootstrap4-children.png similarity index 100% rename from docs/batch-symfony-framework/images/bootstrap4-children.png rename to docs/docs/_static/images/symfony/ui/bootstrap4-children.png diff --git a/docs/batch-symfony-framework/images/bootstrap4-details.png b/docs/docs/_static/images/symfony/ui/bootstrap4-details.png similarity index 100% rename from docs/batch-symfony-framework/images/bootstrap4-details.png rename to docs/docs/_static/images/symfony/ui/bootstrap4-details.png diff --git a/docs/batch-symfony-framework/images/bootstrap4-list.png b/docs/docs/_static/images/symfony/ui/bootstrap4-list.png similarity index 100% rename from docs/batch-symfony-framework/images/bootstrap4-list.png rename to docs/docs/_static/images/symfony/ui/bootstrap4-list.png diff --git a/docs/batch-symfony-framework/images/bootstrap4-warnings.png b/docs/docs/_static/images/symfony/ui/bootstrap4-warnings.png similarity index 100% rename from docs/batch-symfony-framework/images/bootstrap4-warnings.png rename to docs/docs/_static/images/symfony/ui/bootstrap4-warnings.png diff --git a/docs/batch-symfony-framework/images/sonata-children.png b/docs/docs/_static/images/symfony/ui/sonata-children.png similarity index 100% rename from docs/batch-symfony-framework/images/sonata-children.png rename to docs/docs/_static/images/symfony/ui/sonata-children.png diff --git a/docs/batch-symfony-framework/images/sonata-details.png b/docs/docs/_static/images/symfony/ui/sonata-details.png similarity index 100% rename from docs/batch-symfony-framework/images/sonata-details.png rename to docs/docs/_static/images/symfony/ui/sonata-details.png diff --git a/docs/batch-symfony-framework/images/sonata-list.png b/docs/docs/_static/images/symfony/ui/sonata-list.png similarity index 100% rename from docs/batch-symfony-framework/images/sonata-list.png rename to docs/docs/_static/images/symfony/ui/sonata-list.png diff --git a/docs/batch-symfony-framework/images/sonata-warnings.png b/docs/docs/_static/images/symfony/ui/sonata-warnings.png similarity index 100% rename from docs/batch-symfony-framework/images/sonata-warnings.png rename to docs/docs/_static/images/symfony/ui/sonata-warnings.png diff --git a/docs/docs/_static/styles/custom.css b/docs/docs/_static/styles/custom.css new file mode 100644 index 00000000..e69de29b diff --git a/docs/docs/best-practices/naming-jobs.rst b/docs/docs/best-practices/naming-jobs.rst new file mode 100644 index 00000000..ab600bfb --- /dev/null +++ b/docs/docs/best-practices/naming-jobs.rst @@ -0,0 +1,8 @@ +Naming Jobs +============================================================ + + | *There are only two hard things in Computer Science: cache invalidation and naming things.* + | -- Phil Karlton + +todo + diff --git a/docs/docs/best-practices/performance.rst b/docs/docs/best-practices/performance.rst new file mode 100644 index 00000000..13f8767c --- /dev/null +++ b/docs/docs/best-practices/performance.rst @@ -0,0 +1,4 @@ +Performance optimization +============================================================ + +todo diff --git a/docs/docs/bridges.rst b/docs/docs/bridges.rst new file mode 100644 index 00000000..b524fd46 --- /dev/null +++ b/docs/docs/bridges.rst @@ -0,0 +1,19 @@ +Bridges +============================================================ + +| ``yokai/batch`` has no dependency on itself. +| But it is part of a larger package ecosystem, that are bridges with popular libraries and frameworks. +| Everytime you add a ``yokai/batch*`` package to your project, you gain more capabilities in jobs development. + +Here is the complete list of what to expect: + +.. toctree:: + Doctrine DBAL + Doctrine ORM + Doctrine Persistence + Flysystem + OpenSpout + Symfony Console + Symfony Messenger + Symfony Serializer + Symfony Validator diff --git a/docs/docs/bridges/doctrine-dbal.rst b/docs/docs/bridges/doctrine-dbal.rst new file mode 100644 index 00000000..4d39deaf --- /dev/null +++ b/docs/docs/bridges/doctrine-dbal.rst @@ -0,0 +1,34 @@ +Bridge with ``doctrine/dbal`` +============================================================ + +todo + + +Store JobExecution objects in an SQL database +------------------------------------------------------------ + +todo + + +Read from a paginated SQL query +------------------------------------------------------------ + +todo + + +Read from a cursored SQL query +------------------------------------------------------------ + +todo + + +Write items using inserts in a SQL table +------------------------------------------------------------ + +todo + + +Write items using upsert in a SQL table +------------------------------------------------------------ + +todo diff --git a/docs/docs/bridges/doctrine-orm.rst b/docs/docs/bridges/doctrine-orm.rst new file mode 100644 index 00000000..cac6d3bf --- /dev/null +++ b/docs/docs/bridges/doctrine-orm.rst @@ -0,0 +1,9 @@ +Bridge with ``doctrine/orm`` +============================================================ + +todo + +Read from entities in database +------------------------------------------------------------ + +todo diff --git a/docs/docs/bridges/doctrine-persistence.rst b/docs/docs/bridges/doctrine-persistence.rst new file mode 100644 index 00000000..da16d36a --- /dev/null +++ b/docs/docs/bridges/doctrine-persistence.rst @@ -0,0 +1,97 @@ +Bridge with ``doctrine/persistence`` +============================================================ + +todo + +Object item writer +------------------------------------------------------------ + +todo + +Object registry +------------------------------------------------------------ + +Imagine that in an ``ItemJob`` you need to find objects from a database. + +.. code-block:: php + + $item + */ + public function process(mixed $item): Product + { + $product = $this->repository->findOneBy(['sku' => $item['sku']]); + + $product ??= new Product($item['sku']); + + $product->setName($item['name']); + $product->setPrice($item['price']); + //... + + return $product; + } + } + +| The problem here is that every time you will call ``findOneBy``, you + will have to query the database. The object might already be in +| Doctrine’s memory, so it won’t be hydrated twice, but the query will be + done every time. + +| The role of the ``ObjectRegistry`` is to remember found objects + identities, and query these objects with it instead. + +.. code-block:: diff + + use App\Entity\Product; + -use Doctrine\Persistence\ObjectRepository; + +use Yokai\Batch\Bridge\Doctrine\Persistence\ObjectRegistry; + use Yokai\Batch\Job\Item\ItemProcessorInterface; + + class DenormalizeProductProcessor implements ItemProcessorInterface + { + public function __construct( + - private ObjectRepository $repository, + + private ObjectRegistry $registry, + ) { + } + + /** + * @param array $item + */ + public function process(mixed $item): Product + { + - $product = $this->repository->findOneBy(['sku' => $item['sku']]); + + $product = $this->registry->findOneBy(Product::class, ['sku' => $item['sku']]); + + $product ??= new Product($item['sku']); + + $product->setName($item['name']); + $product->setPrice($item['price']); + //... + + return $product; + } + } + +| The first time, the query will hit the database, and the object identity + will be remembered in the registry. +| Everytime after that, the registry will call + ``Doctrine\Persistence\ObjectManager::find`` instead. +| If the object is still in Doctrine’s memory, it will be returned directly. +| Otherwise, the query will be the fastest possible because it will use the object identity. + +.. seealso:: + | :doc:`What is an item job? ` diff --git a/docs/docs/bridges/league-flysystem.rst b/docs/docs/bridges/league-flysystem.rst new file mode 100644 index 00000000..a157c5b8 --- /dev/null +++ b/docs/docs/bridges/league-flysystem.rst @@ -0,0 +1,14 @@ +Bridge with ``league/flysystem`` +============================================================ + +todo + +Copy files job +------------------------------------------------------------ + +todo + +Move files job +------------------------------------------------------------ + +todo diff --git a/docs/docs/bridges/openspout.rst b/docs/docs/bridges/openspout.rst new file mode 100644 index 00000000..fdbf73e1 --- /dev/null +++ b/docs/docs/bridges/openspout.rst @@ -0,0 +1,105 @@ +Bridge with ``openspout/openspout`` +============================================================ + +| The ``OpenSpout`` library allows to read and write with the same API from CSV/ODS/XLSX files. +| The bridge will allow you to do the same, within an ``ItemJob``. + +Item reader +------------------------------------------------------------ + +The `FlatFileReader `__ is a reader +that will read from CSV/ODS/XLSX file and return each line as an array. + +.. code-block:: php + + FIELD_DELIMITER = ';'; + $options->FIELD_ENCLOSURE = '|'; + new FlatFileReader( + new StaticValueParameterAccessor('/path/to/file.csv'), + $options, + null, + HeaderStrategy::none(), + ); + + // Read .ods file + // Only sheet named "Sheet name to read" will be read + // Each item will be an array_combine of first line as key and line as values + new FlatFileReader( + new StaticValueParameterAccessor('/path/to/file.ods'), + null, + SheetFilter::nameIs('Sheet name to read'), + HeaderStrategy::combine(), + ); + +.. seealso:: + | :doc:`What is an item reader? ` + +Item writer +------------------------------------------------------------ + +The `FlatFileWriter `__ is a writer +that will write to CSV/ODS/XLSX file and each item will written its own line. + +.. code-block:: php + + FIELD_DELIMITER = ';'; + $options->FIELD_ENCLOSURE = '|'; + new FlatFileWriter( + new StaticValueParameterAccessor('/path/to/file.csv'), + $options, + ); + + // Write items to .ods file + // That file will contain a header line with: static | header | keys + // Change the sheet name data will be written + // Change the default style of each cell + $options = new ODSOptions(); + $options->DEFAULT_ROW_STYLE = (new Style())->setFontBold(); + new FlatFileWriter( + new StaticValueParameterAccessor('/path/to/file.ods'), + $options, + 'The sheet name', + ['static', 'header', 'keys'], + ); + +.. seealso:: + | :doc:`What is an item writer? ` diff --git a/docs/docs/bridges/symfony-console.rst b/docs/docs/bridges/symfony-console.rst new file mode 100644 index 00000000..e5c69167 --- /dev/null +++ b/docs/docs/bridges/symfony-console.rst @@ -0,0 +1,35 @@ +Bridge with ``symfony/console`` +============================================================ + +todo + +Command +------------------------------------------------------------ + +The `RunJobCommand `__ +can execute any job. + +The command accepts 2 arguments: + +* the job name to execute +* the job parameters for the ``JobExecution`` (optional) + +Examples: + +.. code-block:: console + + bin/console import + bin/console export '{"toFile":"/path/to/file.xml"}' + +Job launcher +------------------------------------------------------------ + +The `RunCommandJobLauncher `__ +execute jobs via an asynchronous symfony command. + +The command called is ``yokai:batch:run``, and the command will actually execute the job. + +Additionally, the command will run with an output redirect (``>>``) to ``var/log/batch_execute.log``. + +.. seealso:: + | :doc:`What is a job launcher? ` diff --git a/docs/docs/bridges/symfony-framework.rst b/docs/docs/bridges/symfony-framework.rst new file mode 100644 index 00000000..862d245b --- /dev/null +++ b/docs/docs/bridges/symfony-framework.rst @@ -0,0 +1,345 @@ +Bridge with ``symfony/framework-bundle`` +============================================================ + +The ``Messenger`` component for ``Symfony`` allows dispatch messages through a bus. + +Configure the Job launcher(s) +------------------------------------------------------------ + +You can use many different job launcher in your application, +you will be able to register these using configuration: + +.. code-block:: yaml + + # config/packages/yokai_batch.yaml + yokai_batch: + launcher: + default: simple + launchers: + simple: ... + async: ... + +.. note:: + If you do not configure anything here, you will be using the + `SimpleJobLauncher `__. + +| The ``default`` job launcher, must reference a launcher name, defined in the ``launchers`` list. +| The ``default`` job launcher will be the autowired instance of job launcher when you ask for one. +| All ``launchers`` will be registered as a service, and an autowire named alias will be configured for it. +| For instance, in the example below, you will be able to register all these launchers like this: + +.. code-block:: php + + `__, no configuration allowed +* ``messenger://messenger``: a `DispatchMessageJobLauncher `__, no configuration allowed +* ``console://console``: a `RunCommandJobLauncher `__, configurable options: + + * ``log``: the filename where command output will be redirected (defaults to ``batch_execute.log``) + +* ``service://service``: pointing to a service of your choice, configurable options: + + * ``service``: the id of the service to use (required, an exception will be thrown otherwise) + +.. seealso:: + | :doc:`What is a job launcher? ` + +Configure the JobExecution storage +------------------------------------------------------------ + +You can have only one storage for your ``JobExecution``, and you have several options: + +* ``filesystem`` will create a file for each ``JobExecution`` in + ``%kernel.project_dir%/var/batch/{execution.jobName}/{execution.id}.json`` +* ``dbal`` will create a row in a table for each ``JobExecution`` +* ``service`` will use a service you have defined in your application + +.. code-block:: yaml + + # config/packages/yokai_batch.yaml + yokai_batch: + storage: + filesystem: ~ + # Or with yokai/batch-doctrine-dbal (& doctrine/dbal) + # dbal: ~ + # Or with a service of yours + # service: ~ + +.. note:: + | The default storage is ``filesystem``, because it only requires a writeable filesystem. + | But if you already have ``doctrine/dbal`` in your project, it is highly recommended to use it instead. + | Because querying ``JobExecution`` in a filesystem might be slow, specially if you are planing to add UIs on top. + +.. seealso:: + | :doc:`What is a job execution? ` + | :doc:`What is a job execution storage? ` + +User interface to visualize ``JobExecution`` +------------------------------------------------------------ + +The package is shipped with few routes that will allow you and your users, to watch for ``JobExecution``. + +.. image:: /_static/images/symfony/ui/bootstrap4-list.png +.. image:: /_static/images/symfony/ui/bootstrap4-details.png +.. image:: /_static/images/symfony/ui/bootstrap4-children.png +.. image:: /_static/images/symfony/ui/bootstrap4-warnings.png + +Installation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For the UI to be enabled, it is required that you install some dependencies: + +.. code-block:: shell + + composer require symfony/translation symfony/twig-bundle + +The UI is disabled by default, you must enable it explicitly: + +.. code-block:: yaml + + # config/packages/yokai_batch.yaml + yokai_batch: + ui: + enabled: true + +You will also need to import bundle routes: + +.. code-block:: yaml + + # config/routes/yokai_batch.yaml + _yokai_batch: + resource: "@YokaiBatchBundle/Resources/routing/ui.xml" + +Templating +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +| The templating service is used by the + `JobController `__ + to render its templates. +| It’s a wrapper around `Twig `__, for you to control templates used, + and variables passed. + +By default + +* the templating will find templates like ``@YokaiBatch/bootstrap4/*.html.twig`` +* the template base view will be ``base.html.twig`` + +You can configure a prefix for all templates: + +.. code-block:: yaml + + # config/packages/yokai_batch.yaml + yokai_batch: + ui: + templating: + prefix: 'batch/job/' + +.. note:: + With this configuration, we will look for templates like ``batch/job/*.html.twig``. + +You can also configure the name of the base template for the root views of that bundle: + +.. code-block:: yaml + + # config/packages/yokai_batch.yaml + yokai_batch: + ui: + templating: + base_template: 'layout.html.twig' + +.. note:: + With this configuration, the template base view will be ``layout.html.twig``. + +If these are not enough, or if you need to add more variables to context, you can configure a service: + +.. code-block:: yaml + + # config/packages/yokai_batch.yaml + yokai_batch: + ui: + templating: + service: 'App\Batch\AppTemplating' + +And create the class that will cover the templating: + +.. code-block:: php + + 'bar']); // add variables to $context if you want + } + } + +.. note:: + You can also use the + ``Yokai\Batch\Bridge\Symfony\Framework\UserInterface\Templating\ConfigurableTemplating`` + that will cover both prefix and static variables at construction. + +Filtering +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``JobExecution`` list includes a filter form, but you will need another optional dependency: + +.. code-block:: shell + + composer require symfony/form + +Security +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There is no access control over ``JobExecution`` by default, you will need another optional dependency: + +.. code-block:: shell + + composer require symfony/security-bundle + +Every security attribute the bundle is using is configurable: + +.. code-block:: yaml + + # config/packages/yokai_batch.yaml + yokai_batch: + ui: + security: + attributes: + list: ROLE_JOB_LIST # defaults to IS_AUTHENTICATED + view: ROLE_JOB_VIEW # defaults to IS_AUTHENTICATED + traces: ROLE_JOB_TRACES # defaults to IS_AUTHENTICATED + logs: ROLE_JOB_LOGS # defaults to IS_AUTHENTICATED + +| Optionally, you can register a voter for these attributes. +| This is especially useful if you need different access control rules per ``JobExecution``. + +.. code-block:: php + + `__ + project. +| The bundle got you covered with a dedicated templating services + and templates. + +.. image:: /_static/images/symfony/ui/sonata-list.png +.. image:: /_static/images/symfony/ui/sonata-details.png +.. image:: /_static/images/symfony/ui/sonata-children.png +.. image:: /_static/images/symfony/ui/sonata-warnings.png + +.. code-block:: shell + + composer require sonata-project/admin-bundle + +.. code-block:: yaml + + # config/packages/yokai_batch.yaml + yokai_batch: + ui: + templating: sonata + +.. note:: + With this configuration, we will look for templates like ``@YokaiBatch/sonata/*.html.twig``. + +Customizing templates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +| You can override templates like + `described it Symfony’s documentation `__. +| Examples: + +* ``templates/bundles/YokaiBatchBundle/bootstrap4/list.html.twig`` +* ``templates/bundles/YokaiBatchBundle/bootstrap4/show/_parameters.html.twig`` + +But you can also register job name dedicated templates if you need some specific view for one of your jobs: + +* ``templates/bundles/YokaiBatchBundle/bootstrap4/show/{job name}/_children-executions.html.twig`` +* ``templates/bundles/YokaiBatchBundle/bootstrap4/show/{job name}/_failures.html.twig`` +* ``templates/bundles/YokaiBatchBundle/bootstrap4/show/{job name}/_general.html.twig`` +* ``templates/bundles/YokaiBatchBundle/bootstrap4/show/{job name}/_information.html.twig`` +* ``templates/bundles/YokaiBatchBundle/bootstrap4/show/{job name}/_parameters.html.twig`` +* ``templates/bundles/YokaiBatchBundle/bootstrap4/show/{job name}/_summary.html.twig`` +* ``templates/bundles/YokaiBatchBundle/bootstrap4/show/{job name}/_warnings.html.twig`` + + +Logger service that log in ``JobExecution`` +------------------------------------------------------------ + +| The batch logger will log inside the JobExecution. +| In a Symfony project, you can use that with the symfony autowiring + by naming your variable as ``$yokaiBatchLogger`` + +.. code-block:: php + + yokaiBatchLogger->error(...); + } + } + +.. seealso:: + | :doc:`What is the job execution? ` diff --git a/docs/docs/bridges/symfony-messenger.rst b/docs/docs/bridges/symfony-messenger.rst new file mode 100644 index 00000000..16be8c4d --- /dev/null +++ b/docs/docs/bridges/symfony-messenger.rst @@ -0,0 +1,40 @@ +Bridge with ``symfony/messenger`` +============================================================ + +The ``Messenger`` component for ``Symfony`` allows dispatch messages through a bus. + +Launch Job through Messenger dispatcher +------------------------------------------------------------ + +The +`DispatchMessageJobLauncher `__ +execute jobs via a symfony command message dispatch. + +A `LaunchJobMessage `__ +message will be dispatched and handled by the +`LaunchJobMessageHandler `__ +will be called with that message after being routed. + +How to configure an async transport for the launcher? +------------------------------------------------------------ + +You first need to configure an async transport, like explained +`in Symfony’s doc `__. + +Then you will have to route the message to this async transport you created, like explained +`in Symfony’s doc `__. + +You will end with something like: + +.. code-block:: yaml + + # config/packages/messenger.yaml + framework: + messenger: + transports: + async: "%env(MESSENGER_TRANSPORT_DSN)%" + routing: + 'Yokai\Batch\Bridge\Symfony\Messenger\LaunchJobMessage': async + +.. seealso:: + | :doc:`What is a job launcher? ` diff --git a/docs/docs/bridges/symfony-serializer.rst b/docs/docs/bridges/symfony-serializer.rst new file mode 100644 index 00000000..7e3b8dfb --- /dev/null +++ b/docs/docs/bridges/symfony-serializer.rst @@ -0,0 +1,20 @@ +Bridge with ``symfony/serializer`` +============================================================ + +todo + +Denormalize item processor +------------------------------------------------------------ + +todo + +Job execution serializer +------------------------------------------------------------ + +todo + +Normalize item processor +------------------------------------------------------------ + +todo + diff --git a/docs/docs/bridges/symfony-validator.rst b/docs/docs/bridges/symfony-validator.rst new file mode 100644 index 00000000..bda0c9b5 --- /dev/null +++ b/docs/docs/bridges/symfony-validator.rst @@ -0,0 +1,10 @@ +Bridge with ``symfony/validator`` +============================================================ + +todo + +Skip invalid item processor +------------------------------------------------------------ + +todo + diff --git a/docs/docs/conf.py b/docs/docs/conf.py new file mode 100644 index 00000000..3efe62cb --- /dev/null +++ b/docs/docs/conf.py @@ -0,0 +1,42 @@ +# Configuration file for the Sphinx documentation builder. + +# -- Project information + +project = 'Yokai Batch' +copyright = '2019, Yann Eugoné' +author = 'Yann Eugoné' + +release = '0.5.0' +version = '0.5.0' + +# -- General configuration + +extensions = [ + 'sphinx.ext.duration', + 'sphinx.ext.doctest', + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.intersphinx', +] + +intersphinx_mapping = { + 'python': ('https://docs.python.org/3/', None), + 'sphinx': ('https://www.sphinx-doc.org/en/master/', None), +} +intersphinx_disabled_domains = ['std'] + +templates_path = ['_templates'] + +html_static_path = ['_static'] + +html_css_files = [ + 'styles/custom.css', +] + +# -- Options for HTML output + +html_theme = 'sphinx_rtd_theme' + +# -- Options for EPUB output + +epub_show_urls = 'footnote' diff --git a/docs/docs/core-concepts/aware-interfaces.rst b/docs/docs/core-concepts/aware-interfaces.rst new file mode 100644 index 00000000..c13b3482 --- /dev/null +++ b/docs/docs/core-concepts/aware-interfaces.rst @@ -0,0 +1,72 @@ +\*Aware interfaces +============================================================ + +| When a job execution starts, + a `JobExecution `__ is created for it. +| This object contains information about the current execution. + +You will often want to access this object or one of its child to : + +* access provided user parameters in your components +* leave some information on the job execution: logs, summary, warning... + +To do that, your component will need to implement an interface, telling the library that you need something. + +What is ``JobExecutionAwareInterface``? +------------------------------------------------------------ + +The `JobExecutionAwareInterface `__ +will allow you to gain access to the current +`JobExecution `__. + +.. note:: + This interface is covered by + `JobExecutionAwareTrait `__ + for a default implementation that is most of the time sufficient. + +What is ``JobParametersAwareInterface``? +------------------------------------------------------------ + +The +`JobParametersAwareInterface `__ +will allow you to gain access to the +`JobParameters `__ of the current execution. + +.. note:: + This interface is covered by + `JobParametersAwareTrait `__ + for a default implementation that is most of the time sufficient. + +What is ``SummaryAwareInterface``? +------------------------------------------------------------ + +The `SummaryAwareInterface `__ +will allow you to gain access to the `Summary `__ of the current execution. + +.. note:: + This interface is covered by `SummaryAwareTrait `__ + for a default implementation that is most of the time sufficient. + +How does that work exactly? +------------------------------------------------------------ + +There is no magic involved here, every component is responsible for propagating the context through these interfaces. + +In the library, you will find that : + +* `ItemJob `__ is propagating context to + `ItemReaderInterface `__, + `ItemProcessorInterface `__ + and + `ItemWriterInterface `__. +* Every + `ItemReaderInterface `__, + `ItemProcessorInterface `__ + and + `ItemWriterInterface `__ + acting as a decorator, is propagating context to their decorated element. + +You can add this interface to any class, but you are responsible for the context propagation. + +.. seealso:: + | :doc:`What is an item reader? ` diff --git a/docs/docs/core-concepts/item-job.rst b/docs/docs/core-concepts/item-job.rst new file mode 100644 index 00000000..e19648a8 --- /dev/null +++ b/docs/docs/core-concepts/item-job.rst @@ -0,0 +1,68 @@ +Item Job +============================================================ + +What is an item? +------------------------------------------------------------ + +The library allows you to declare and execute jobs, but wait why do we +named it batch then? There you are, the ``ItemJob`` is where batch +processing actually starts. + +This is just a job that has been prepared to batch handle items. + +If you are familiar with the concept of an `ETL `__, +this is pretty much the same. + +How to create an item job? +------------------------------------------------------------ + +The item job allows you to split your logic into 3 different component: + +* an :doc:`item reader `: stands for **Extract** in ETL +* an :doc:`item processor `: stands for **Transform** in ETL +* an :doc:`item writer `: stands for **Load** in ETL + +.. code-block:: php + + ` + | :doc:`What is a job launcher? ` diff --git a/docs/docs/core-concepts/item-job/item-processor.rst b/docs/docs/core-concepts/item-job/item-processor.rst new file mode 100644 index 00000000..fb42028a --- /dev/null +++ b/docs/docs/core-concepts/item-job/item-processor.rst @@ -0,0 +1,47 @@ +What is an item processor? +============================================================ + +The item processor is used by the item job to transform every read item. + +It can be any class implementing +`ItemProcessorInterface `__. + +What types of item processors exists? +------------------------------------------------------------ + +**Built-in item processors:** + +* `ArrayMapProcessor `__: + apply a callback to each element of array items. +* `CallbackProcessor `__: + use a callback to transform each items. +* `ChainProcessor `__: + chain transformation of multiple item processor, one after the other. +* `FilterUniqueProcessor `__: + assign an identifier to each item, and skip already encountered items. +* `NullProcessor `__: + perform no transformation on items. +* `RoutingProcessor `__: + route processing to different processor based on your logic. + +**Item processors from bridges:** + +* From ``symfony/validator`` bridge: + + * `SkipInvalidItemProcessor `__: + validate item and throw exception if invalid that will cause item to be skipped. + +* From ``symfony/serializer`` bridge: + + * `DenormalizeItemProcessor `__: + denormalize each item. + * `NormalizeItemProcessor `__: + normalize each item. + +**Item processors for testing purpose:** + +* `TestDebugProcessor `__: + dummy item processor that you can use in your unit tests. + +.. seealso:: + | :doc:`What is an item job? ` diff --git a/docs/docs/core-concepts/item-job/item-reader.rst b/docs/docs/core-concepts/item-job/item-reader.rst new file mode 100644 index 00000000..20f6efd9 --- /dev/null +++ b/docs/docs/core-concepts/item-job/item-reader.rst @@ -0,0 +1,56 @@ +What is an item reader? +============================================================ + +The item reader is used by the item job to extract item from a source. + +It can be any class implementing +`ItemReaderInterface `__. + +What types of item readers exists? +------------------------------------------------------------ + +**Built-in item readers:** + +* `FixedColumnSizeFileReader `__: + read a file line by line, and decode each line with fixed columns size to an array. +* `JsonLinesReader `__: + read a file line by line, and decode each line as JSON. +* `AddMetadataReader `__: + decorates another reader by adding static information to each read item. +* `IndexWithReader `__: + decorates another reader by changing index of each item. +* `ParameterAccessorReader `__: + read from an inmemory value located at some configurable place. +* `SequenceReader `__: + read from multiple item reader, one after the other. +* `StaticIterableReader `__: + read from an iterable you provide during construction. +* `CallbackReader `__: + read from a ``Closure`` you provide during construction. + +**Item readers from bridges:** + +* From ``openspout/openspout`` bridge: + + * `FlatFileReader `__: + read from any CSV/ODS/XLSX file. + +* From ``doctrine/dbal`` bridge: + + * `DoctrineDBALQueryOffsetReader `__: + execute an SQL query and iterate over results, using a limit + offset pagination strategy. + * `DoctrineDBALQueryCursorReader `__: + execute an SQL query and iterate over results, using a column based cursor strategy. + +* From ``doctrine/orm`` bridge: + + * `EntityReader `__: + read from any Doctrine ORM entity. + +**Item readers for testing purpose:** + +* `TestDebugReader `__: + dummy item reader that you can use in your unit tests. + +.. seealso:: + | :doc:`What is an item job? ` diff --git a/docs/docs/core-concepts/item-job/item-writer.rst b/docs/docs/core-concepts/item-job/item-writer.rst new file mode 100644 index 00000000..30b590e7 --- /dev/null +++ b/docs/docs/core-concepts/item-job/item-writer.rst @@ -0,0 +1,69 @@ +What is an item writer? +============================================================ + +The item writer is used by the item job to load every processed item. + +It can be any class implementing +`ItemWriterInterface `__. + +What types of item writers exists? +------------------------------------------------------------ + +**Built-in item writers:** + +* `JsonLinesWriter `__: + write items as a json string each on a line of a file. +* `ChainWriter `__: + write items on multiple item writers. +* `ConditionalWriter `__: + will only write items that are matching your conditions. +* `DispatchEventsWriter `__: + will dispatch events before and after writing. +* `LaunchJobForEachItemWriter `__: + launch another job for each items. +* `LaunchJobForItemsBatchWriter `__: + launch another job for each item batches. +* `NullWriter `__: + do not write items. +* `RoutingWriter `__: + route writing to different writer based on your logic. +* `SummaryWriter `__: + write items to a job summary value. +* `TransformingWriter `__: + perform items transformation before delegating to another writer. +* `CallbackWriter `__: + delegate items write operations to a closure passed at construction. + +**Item writers from bridges:** + +* From ``symfony/messenger`` bridge: + + * `DispatchEachItemAsMessageWriter `__: + dispatch each item as a message in a bus. + +* From ``doctrine/dbal`` bridge: + + * `DoctrineDBALInsertWriter `__: + write items by inserting in a table via a Doctrine ``Connection``. + * `DoctrineDBALUpsertWriter `__: + write items by inserting/updating in a table via a Doctrine ``Connection``. + +* From ``doctrine/persistence`` bridge: + + * `ObjectWriter `__: + write items to any Doctrine ``ObjectManager``. + +* From ``openspout/openspout`` bridge: + + * `FlatFileWriter `__: + write items to any CSV/ODS/XLSX file. + +**Item writers for testing purpose:** + +* `InMemoryWriter `__: + write in a private var which can be accessed afterward in your tests. +* `TestDebugWriter `__: + dummy item writer that you can use in your unit tests. + +.. seealso:: + | :doc:`What is an item job? ` diff --git a/docs/docs/core-concepts/job-execution-storage.rst b/docs/docs/core-concepts/job-execution-storage.rst new file mode 100644 index 00000000..4d8896d0 --- /dev/null +++ b/docs/docs/core-concepts/job-execution-storage.rst @@ -0,0 +1,63 @@ +Job execution storage +============================================================ + +What is a job execution storage? +------------------------------------------------------------ + +Whenever a job is launched, whether is starts immediately or not, an +execution is stored for it. + +The execution are stored to allow you to keep an eye on what is +happening. + +This persistence is on the responsibility of the job execution storage. + +How do I store my Job Execution? +------------------------------------------------------------ + +You should never be forced to store ``JobExecution`` yourself. + +This is the role of the :doc:`JobLauncher` job to store it whenever +it is required (usually at the beginning and the end of the job +execution). + +How can I retrieve a Job Execution afterwards? +------------------------------------------------------------ + +Every storage implements +`JobExecutionStorageInterface `__ +that has a method called ``retrieve``. Use this method to retrieve one +execution using job name and execution id. + +Depending on which storage you decided to use, you may also be able to: + +* list of all executions for particular job, if your storage implements + `ListableJobExecutionStorageInterface `__: +* filter list of executions matching criteria you provided, if your storage implements + `QueryableJobExecutionStorageInterface `__: + +.. warning:: + Sometimes the storage may implement these interfaces, + but due to the way executions are stored, it might not be recommended heavily rely on these extra methods. + +What types of storages exists? +------------------------------------------------------------ + +**Built-in storages:** + +* `NullJobExecutionStorage `__: + do not persist any job execution. +* `FilesystemJobExecutionStorage `__: + store job executions to a file on local filesystem. + +**Storages from bridges:** + +* From ``doctrine/dbal`` bridge: + + * `DoctrineDBALJobExecutionStorage `__: + store job executions to a relational database. + +**Storages for testing purpose:** + +* `InMemoryJobExecutionStorage `__: + store executions in a private var that can be accessed afterwards in your tests. diff --git a/docs/docs/core-concepts/job-execution.rst b/docs/docs/core-concepts/job-execution.rst new file mode 100644 index 00000000..9a205f93 --- /dev/null +++ b/docs/docs/core-concepts/job-execution.rst @@ -0,0 +1,27 @@ +Job execution +============================================================ + +What is a Job execution? +------------------------------------------------------------ + +A `JobExecution `__ is the class that holds +information about one execution of a :doc:`job `. + +What kind of information does it hold? +------------------------------------------------------------ + +* ``JobExecution::$jobName``: The Job name (job id) +* ``JobExecution::$id``: The execution id +* ``JobExecution::$parameters``: Some parameters with which job was executed +* ``JobExecution::$status``: A status (pending, running, stopped, completed, abandoned, failed) +* ``JobExecution::$startTime``: Start time +* ``JobExecution::$endTime``: End time +* ``JobExecution::$failures``: A list of failures (usually exceptions) +* ``JobExecution::$warnings``: A list of warnings (usually skipped items) +* ``JobExecution::$summary``: A summary (can contain any data you wish to store) +* ``JobExecution::$logs``: Some logs +* ``JobExecution::$childExecutions``: Some child execution + +.. seealso:: + | :doc:`How is a job execution created? ` + | :doc:`How can I retrieve a job execution afterwards? ` diff --git a/docs/docs/core-concepts/job-launcher.rst b/docs/docs/core-concepts/job-launcher.rst new file mode 100644 index 00000000..29710b39 --- /dev/null +++ b/docs/docs/core-concepts/job-launcher.rst @@ -0,0 +1,81 @@ +Job Launcher +============================================================ + +What is a job launcher? +------------------------------------------------------------ + +The job launcher is responsible for executing/scheduling every jobs. + +| Yeah, executing OR scheduling. There is multiple implementation of a job launcher across bridges. +| Job’s execution might be asynchronous, and thus, when you ask the job launcher to “launch” a job, + you have to check the ``JobExecution`` status that it had returned to know if the job is already executed. + +What is the simplest way to launch a job? +------------------------------------------------------------ + +.. code-block:: php + + new class implements JobInterface { + public function execute(JobExecution $jobExecution): void + { + // your business logic + } + }, + ]); + $jobExecutionStorage = new NullJobExecutionStorage(); + + $launcher = new SimpleJobLauncher( + new JobExecutionAccessor( + new JobExecutionFactory(new UniqidJobExecutionIdGenerator(), new NullJobExecutionParametersBuilder()), + $jobExecutionStorage, + ), + new JobExecutor(new JobRegistry($jobs), $jobExecutionStorage, null), + ); + + $execution = $launcher->launch('your.job.name', ['job' => ['configuration']]); + +What types of launcher exists? +------------------------------------------------------------ + +**Built-in launchers:** + +* `SimpleJobLauncher `__: + execute the job directly in the same PHP process. + +**Launchers from bridges:** + +* From ``symfony/console`` bridge: + + * `RunCommandJobLauncher `__: + execute the job via an asynchronous symfony command. + +* From ``symfony/messenger`` bridge: + + * `DispatchMessageJobLauncher `__: + execute the job via a symfony message dispatch. + +**Launchers for testing purpose:** + +* `BufferingJobLauncher `__: + do not execute job, but store execution in a private var that can be accessed afterwards in your tests. + +.. seealso:: + | :doc:`What is a job? ` + | :doc:`What is a job execution? ` diff --git a/docs/docs/core-concepts/job-parameter-accessor.rst b/docs/docs/core-concepts/job-parameter-accessor.rst new file mode 100644 index 00000000..4faa366a --- /dev/null +++ b/docs/docs/core-concepts/job-parameter-accessor.rst @@ -0,0 +1,67 @@ +Job parameter accessor +============================================================ + +When a job (or a component within a job) can be working with a parameterized value, it can rely on a +`JobParameterAccessorInterface `__ +instance to retrieve that value. + +.. code-block:: php + + path->get($jobExecution); + // do something with $path + } + } + +What types of parameter accessors exists? +------------------------------------------------------------ + +**Built-in parameter accessors:** + +* `ChainParameterAccessor `__: + try multiple parameter accessors, the first that is not failing is used. +* `ClosestJobExecutionAccessor `__: + try another parameter accessor on each job execution in hierarchy, until not failed. +* `DefaultParameterAccessor `__: + try accessing parameter using another parameter accessor, use default value if failed. +* `JobExecutionParameterAccessor `__: + extract value from job execution’s `parameters `__. +* `JobExecutionSummaryAccessor `__: + extract value from job execution’s `summary `__. +* `ParentJobExecutionAccessor `__: + use another parameter accessor on job execution’s parent execution. +* `ReplaceWithVariablesParameterAccessor `__: + use another parameter accessor to get string value, and replace variables before returning. +* `RootJobExecutionAccessor `__: + use another parameter accessor on job execution’s root execution. +* `SiblingJobExecutionAccessor `__: + use another parameter accessor on job execution’s sibling execution. +* `StaticValueParameterAccessor `__: + use static value provided at construction. + +**Parameter accessors from bridges:** + +* From ``symfony/framework-bundle`` bridge: + + * `ContainerParameterAccessor `__: + use a parameter from Symfony’s container. + +.. seealso:: + | :doc:`What is a job? ` + | :doc:`When does a job execution hierarchy is created? ` + | :doc:`What is a job execution? ` diff --git a/docs/docs/core-concepts/job-with-children.rst b/docs/docs/core-concepts/job-with-children.rst new file mode 100644 index 00000000..10df6014 --- /dev/null +++ b/docs/docs/core-concepts/job-with-children.rst @@ -0,0 +1,43 @@ +Job with children +============================================================ + +todo + +.. code-block:: php + + new class implements JobInterface { + public function execute(JobExecution $jobExecution): void + { + $exportSince = new DateTimeImmutable($jobExecution->getParameter('since')); + // your export logic here + } + }, + 'upload' => new class implements JobInterface { + public function execute(JobExecution $jobExecution): void + { + $pathToUploadExportedFile = $jobExecution->getParameter('path'); + // your upload logic here + } + }, + ]); + + $job = new JobWithChildJobs( + $jobExecutionStorage = new NullJobExecutionStorage(), + new JobExecutor(new JobRegistry($container), $jobExecutionStorage, null), + ['export', 'upload'] + ); diff --git a/docs/docs/core-concepts/job.rst b/docs/docs/core-concepts/job.rst new file mode 100644 index 00000000..d3dbf103 --- /dev/null +++ b/docs/docs/core-concepts/job.rst @@ -0,0 +1,59 @@ +Job +============================================================ + +What is a job? +------------------------------------------------------------ + +A job is the class that is responsible for **what** your code is doing. + +This is the class you will have to create (or reuse), as it contains the +business logic required for what you wish to achieve. + +How to create a job? +------------------------------------------------------------ + +.. code-block:: php + + `__, + +What types of job exists? +------------------------------------------------------------ + +**Built-in jobs:** + +* `AbstractDecoratedJob `__: a job + that is designed to be extended, helps job construction. +* `ItemJob `__: ETL like, batch processing + job (:doc:`documentation `). +* `JobWithChildJobs `__: a job that + trigger other jobs (:doc:`documentation `). +* `TriggerScheduledJobsJob `__: + a job that trigger other jobs when schedule is due (todo documentation). + +**Jobs from bridges:** + +* From ``league/flysystem`` bridge: + + * `CopyFilesJob `__: + copy files from one filesystem to another. + * `MoveFilesJob `__: + move files from one filesystem to another. + +.. seealso:: + | :doc:`How do I start a job? ` + | :doc:`How do I build a batch processing job? ` + | :doc:`How do I access parameters of a job? ` diff --git a/docs/docs/getting-started/standalone-library.rst b/docs/docs/getting-started/standalone-library.rst new file mode 100644 index 00000000..ba858a7a --- /dev/null +++ b/docs/docs/getting-started/standalone-library.rst @@ -0,0 +1,144 @@ +Getting started as a standalone library +============================================================ + +.. note:: + + | If you already have a running app, you might consider a different *getting started* guide, + based on the framework your application is working on: + + * :doc:`/getting-started/with-symfony` + +Installation +------------------------------------------------------------ + +.. code-block:: console + + composer require yokai/batch + +.. note:: + + | ``yokai/batch`` is the only package required. + | But there are many bridge packages you can install if you want to unlock more components. + | Have a look to dedicated documentation: :doc:``. + +Step by step example +------------------------------------------------------------ + +As a developer, from your application, you want to launch a job. + +.. code-block:: php + + launch('import', ['path' => '/path/to/file/to/import']); + +| This job will be executed by the ``JobLauncher``, and at some point in the call graph, your code will run. +| This logic have to be implemented in a ``Job``. + +.. code-block:: php + + getParameter('path'); + // your import logic here + } + }; + + $launcher = new SimpleJobLauncher(...); + + $launcher->launch('import', ['path' => '/path/to/file/to/import']); + +The JobLauncher will have to be provided with all the jobs you create in your application, so it can launch any of it. + +.. code-block:: php + + new class implements JobInterface { + public function execute(JobExecution $jobExecution): void + { + $fileToImport = $jobExecution->getParameter('path'); + // your import logic here + } + }, + ]); + + $launcher = new SimpleJobLauncher( + ..., + new JobExecutor( + new JobRegistry($container), + ... + ) + ); + + $launcher->launch('import', ['path' => '/path/to/file/to/import']); + +.. note:: + + | ``JobContainer`` is an implementation of a `PSR11 container `__. + | You can use it if you want, but you can replace it with any implementation from your application. + +| But now, what if the job fails, or what if you wish to analyse what the job produced. +| You need to a able to store JobExecution, so you can fetch it afterwards. + +.. code-block:: php + + new class implements JobInterface { + public function execute(JobExecution $jobExecution): void + { + $fileToImport = $jobExecution->getParameter('path'); + // your import logic here + } + }, + ]); + + $jobExecutionStorage = new FilesystemJobExecutionStorage(new JsonJobExecutionSerializer(), '/dir/where/jobs/are/stored'); + $launcher = new SimpleJobLauncher( + new JobExecutionAccessor( + new JobExecutionFactory(new UniqidJobExecutionIdGenerator(), new NullJobExecutionParametersBuilder()), + $jobExecutionStorage + ), + new JobExecutor( + new JobRegistry($container), + $jobExecutionStorage, + null // or an instance of \Psr\EventDispatcher\EventDispatcherInterface + ) + ); + + $importExecution = $launcher->launch('import', ['path' => '/path/to/file/to/import']); + +There you go, you have a fully functional stack to start working with the library. diff --git a/docs/docs/getting-started/with-symfony.rst b/docs/docs/getting-started/with-symfony.rst new file mode 100644 index 00000000..054f6c31 --- /dev/null +++ b/docs/docs/getting-started/with-symfony.rst @@ -0,0 +1,121 @@ +Getting started in a Symfony project +============================================================ + +Installation +------------------------------------------------------------ + +.. code-block:: console + + composer require yokai/batch + composer require yokai/batch-symfony-framework + +.. note:: + + | ``yokai/batch`` and ``yokai/batch-symfony-framework`` are the only packages required. + | But there are many bridge packages you can install if you want to unlock more components. + | Have a look to dedicated documentation: :doc:``. + +.. code-block:: php + + ['all' => true], + ]; + +Creating a job in a Symfony project +------------------------------------------------------------ + +Creating the job class +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: php + + getParameter('path'); + // your import logic here + } + } + +.. hint:: + | When registering jobs with dedicated class, you can use the + `JobWithStaticNameInterface `__ + interface to be able to specify the job name of your service. + | Otherwise, the service id will be used, and in that case, the service id is the FQCN. + +.. seealso:: + | :doc:`What is a job? ` + +Registering the job as a service +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +| We need to provide the library with all the implemented jobs we have. +| We will be using Symfony's dependency injection system for that. + +| As Symfony supports registering all classes in ``src/`` as a service, we + can leverage this behaviour to register all jobs in ``src/``. +| We will add a tag to every found class in ``src/`` that implements ``Yokai\Batch\Job\JobInterface``: + +.. code-block:: yaml + + # config/services.yaml + services: + _defaults: + _instanceof: + Yokai\Batch\Job\JobInterface: + tags: ['yokai_batch.job'] + +.. note:: + + | In a Symfony project, we will prefer using one class per job, because service discovery is so easy to use. + | But also because it will be very far easier to configure your job using PHP than any other format. + | For example, there are components that requires a ``Closure``, in their constructors. + | But keep in mind you can register your jobs with any other format of your choice. + +Launching a job in a Symfony project +------------------------------------------------------------ + +Then the job will be triggered with its name: + +.. code-block:: php + + launch(ImportJob::getJobName()); + // now you can look for information in JobExecution + // or if execution is asynchronous, redirect user to a UI where he will watch it 🍿 + } + } + +.. hint:: + | Depending on the bundle configuration, you might be injecting different implementation of ``JobLauncherInterface``. + | Have a look to completed documentation: :doc:`/bridges/symfony-framework`. diff --git a/docs/docs/index.rst b/docs/docs/index.rst new file mode 100644 index 00000000..5fa8ff94 --- /dev/null +++ b/docs/docs/index.rst @@ -0,0 +1,113 @@ +Yokai Batch: Batch processing with PHP +============================================================ + +👀 Overview +------------------------------------------------------------ + +**Yokai Batch** is batch processing job library written with PHP, with zero dependencies. + +Key Features +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* 👀 Keep an eye on the execution of your jobs. +* 🚀 Provide key components to handle batch processing jobs. +* 🧱 Have decoupled reusable components to compose your jobs. +* 🖼️ Have bridges with popular libraries and frameworks. + +Quick Start Guide +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +| **Ready to dive in?** +| Check out our :doc:`/getting-started/standalone-library` to get up and running in no time! + +| **Is your application using Symfony?** +| Check out our :doc:`/getting-started/with-symfony` instead. + +🔖 Core Concepts +------------------------------------------------------------ + +Get familiar with the core concepts: + +* :doc:`/core-concepts/job`: Contains the business logic of your tasks. +* :doc:`/core-concepts/job-launcher`: Your application entrypoint to execute a job. +* :doc:`/core-concepts/item-job`: A type a job that is optimized for batch processing. + +📖 Tutorials and Examples +------------------------------------------------------------ + +Build your first Job +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +| Learn how to create and manage your first batch job in **Yokai Batch**. +| Follow our step-by-step :doc:`/tutorials/first-job`. + +Real world examples +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Explore some of the things that could be built with **Yokai Batch**: + +* :doc:`/tutorials/example-before-after`: Before/After **Yokai Batch** showcase. +* :doc:`/tutorials/example-star-wars`: Import StarWars related data through Doctrine ORM. + +.. toctree:: + :maxdepth: 2 + :hidden: + :caption: Getting Started + + getting-started/standalone-library + getting-started/with-symfony + +.. toctree:: + :maxdepth: 2 + :hidden: + :caption: Core Concepts + + core-concepts/job + core-concepts/job-launcher + core-concepts/item-job + core-concepts/job-execution + core-concepts/job-execution-storage + core-concepts/job-with-children + core-concepts/job-parameter-accessor + core-concepts/aware-interfaces + +.. toctree:: + :maxdepth: 2 + :hidden: + :caption: Tutorials and Examples + + tutorials/first-job + tutorials/example-before-after + tutorials/example-star-wars + +.. toctree:: + :maxdepth: 2 + :hidden: + :caption: Best Practices + + best-practices/naming-jobs + best-practices/performance + +.. toctree:: + :maxdepth: 2 + :hidden: + :caption: Bridges with Other Libraries + + What are bridges? + Doctrine DBAL + Doctrine ORM + Doctrine Persistence + Flysystem + OpenSpout + Symfony Console + Symfony Framework + Symfony Messenger + Symfony Serializer + Symfony Validator + +.. toctree:: + :maxdepth: 2 + :hidden: + :caption: Reference + + reference/faq diff --git a/docs/docs/reference/faq.rst b/docs/docs/reference/faq.rst new file mode 100644 index 00000000..df506289 --- /dev/null +++ b/docs/docs/reference/faq.rst @@ -0,0 +1,19 @@ +Frequently asked questions +============================================================ + +.. contents:: + :local: + +What is this library for? +------------------------------------------------------------ + +Is this library an async task manager? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +| **Yes & No.** The library was not created as an async manager. +| But the library has bridges with libraries that do this. +| So you can expect to run your jobs through any async system of your choice. + +.. seealso:: + | :doc:`/bridges/symfony-messenger` + | :doc:`/bridges/symfony-console` diff --git a/docs/docs/tutorials/before-after/after.php b/docs/docs/tutorials/before-after/after.php new file mode 100644 index 00000000..3b9ffabd --- /dev/null +++ b/docs/docs/tutorials/before-after/after.php @@ -0,0 +1,47 @@ + Family::fromData($data)), + new SkipInvalidItemProcessor($this->validator), + ]), + writer: new ObjectWriter($this->doctrine), + executionStorage: new NullJobExecutionStorage(), + ))->execute(JobExecution::createRoot('1', 'import')); + + return self::SUCCESS; + } +} diff --git a/docs/docs/tutorials/before-after/before.php b/docs/docs/tutorials/before-after/before.php new file mode 100644 index 00000000..90b1b866 --- /dev/null +++ b/docs/docs/tutorials/before-after/before.php @@ -0,0 +1,70 @@ +doctrine->getManagerForClass(Family::class); + \assert($manager !== null); + + $families = []; + while ($line = \fgets($file)) { + try { + $data = \json_decode($line, true, 512, JSON_THROW_ON_ERROR); + \assert(\is_array($data)); + } catch (\JsonException) { + continue; + } + + $family = Family::fromData($data); + $violations = $this->validator->validate($family); + if (\count($violations) > 0) { + continue; + } + + $families[] = $family; + + if (\count($families) % 500 === 0) { + foreach ($families as $family) { + $manager->persist($family); + } + $manager->flush($family); + } + } + + \fclose($file); + + if (\count($families) > 0) { + foreach ($families as $family) { + $manager->persist($family); + } + $manager->flush(); + } + + return self::SUCCESS; + } +} diff --git a/docs/docs/tutorials/example-before-after.rst b/docs/docs/tutorials/example-before-after.rst new file mode 100644 index 00000000..a30d3137 --- /dev/null +++ b/docs/docs/tutorials/example-before-after.rst @@ -0,0 +1,48 @@ +Example: Before/After using Yokai Batch +============================================================ + +.. note:: + | The code involved in this example is part of the test suite of **Yokai Batch**. + | You can find the original code in the source repository: + `Data `__, + `Entities `__, + `Jobs `__, + `Tests `__ + +What are we trying to do? +------------------------------------------------------------ + +We have a ``jsonl`` file, containing data that we want to import in our database. + +.. code-block:: jsonl + + {"code":"camcorders","attributes":["description","image_stabilizer","name","optical_zoom","picture","power_requirements","price","release_date","sensor_type","sku","total_megapixels","weight"],"attribute_as_label":"name","attribute_as_image":"picture","labels":{"en_US":"Camcorders","fr_FR":"Cam\u00e9scopes num\u00e9riques","de_DE":"Digitale Videokameras"}} + {"code":"digital_cameras","attributes":["auto_exposure","auto_focus_assist_beam","auto_focus_lock","auto_focus_modes","auto_focus_points","camera_brand","camera_model_name","camera_type","description","focus","focus_adjustement","image_resolutions","iso_sensitivity","iso_sensitivity_max","iso_sensitivity_min","lens_mount_interface","light_exposure_corrections","light_exposure_modes","light_metering","max_image_resolution","name","optical_zoom","picture","power_requirements","price","release_date","sensor_type","short_description","sku","supported_aspect_ratios","supported_image_format","total_megapixels","weight"],"attribute_as_label":"name","attribute_as_image":"picture","labels":{"en_US":"Digital cameras","fr_FR":"Cam\u00e9ras digitales","de_DE":"Digitale Kameras"}} + {"code":"headphones","attributes":["description","headphone_connectivity","name","picture","power_requirements","price","release_date","sku","snr","thd","weight"],"attribute_as_label":"name","attribute_as_image":"picture","labels":{"en_US":"Headphones","fr_FR":"Casques audio","de_DE":"Kopfh\u00f6rer"}} + +.. note:: + | todo + +Before: Without Yokai Batch +------------------------------------------------------------ + +.. literalinclude:: before-after/before.php + :language: php + +.. warning:: + | There are many little things you have to think about when doing batch processing. + | And there are chances that you have these little things shattered in your application. + | As your team grow, it will become more important to avoid duplicating things like this. + | Because it is likely that someone will forget one of those little things, code will start acting funny. + +After: With Yokai Batch +------------------------------------------------------------ + +.. literalinclude:: before-after/after.php + :language: php + +.. note:: + | todo + +.. hint:: + | todo parler du fait que c'est pas obligé d'avoir un storage, etc... diff --git a/docs/docs/tutorials/example-star-wars.rst b/docs/docs/tutorials/example-star-wars.rst new file mode 100644 index 00000000..67661e7b --- /dev/null +++ b/docs/docs/tutorials/example-star-wars.rst @@ -0,0 +1,102 @@ +Example: StarWars import +============================================================ + +.. note:: + | The code involved in this example is part of the test suite of **Yokai Batch**. + | You can find the original code in the source repository: + `Data `__, + `Entities `__, + `Jobs `__, + `Tests `__ + + +What are we trying to do? +------------------------------------------------------------ + +| We have been provided with `data from the StarWars universe `__. +| Our job is to import these data in a relational database. + +We will import the following data: + +* Characters +* Species +* Planets + +Designing the entities +------------------------------------------------------------ + +.. literalinclude:: star-wars/entity-specie.php + :language: php + +.. literalinclude:: star-wars/entity-planet.php + :language: php + +.. literalinclude:: star-wars/entity-character.php + :language: php + +Writing the import +------------------------------------------------------------ + +Install the packages +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: console + + composer require yokai/batch + composer require yokai/batch-openspout + composer require yokai/batch-symfony-validator + composer require yokai/batch-doctrine-persistence + +An import for each entity +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. literalinclude:: star-wars/job-planet.php + :language: php + +.. literalinclude:: star-wars/job-specie.php + :language: php + +.. literalinclude:: star-wars/job-character.php + :language: php + +Factorise common logic +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +All three imports behavior the same way: + +* read data from a CSV file +* convert data to an entity +* ensure entity is valid +* save entity to the database + +| The thing is, most of the time, in your application, you will have similar jobs. +| **Yokai Batch** offers many reusable components, but you should also try to organise your code jobs. + +| We chose the easiest way here: introducing an abstract class for all our jobs. +| We could have been creating a ``JobFactory``, but it's matter of taste. + +.. literalinclude:: star-wars/job-parent.php + :language: php + +A job for the whole import +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +| So far, we created one job per entity to import. +| It is very convenient, because we can leverage the power of ``ItemJob``. + +| But having to trigger the three jobs on a specific order for the import to work is boring. +| What if we introduce a job to trigger these three jobs in a row? + +.. literalinclude:: star-wars/job-parent.php + :language: php + +Running the import +------------------------------------------------------------ + +Now, you can trigger this job anytime you want, using the ``JobLauncher`` configured in your application. + +.. literalinclude:: star-wars/command.php + :language: php + +.. seealso:: + | :doc:`What is an item job? ` diff --git a/docs/docs/tutorials/first-job.rst b/docs/docs/tutorials/first-job.rst new file mode 100644 index 00000000..ba9ad634 --- /dev/null +++ b/docs/docs/tutorials/first-job.rst @@ -0,0 +1,4 @@ +Building your first Job +============================================================ + +todo diff --git a/docs/docs/tutorials/star-wars/command.php b/docs/docs/tutorials/star-wars/command.php new file mode 100644 index 00000000..ae22f343 --- /dev/null +++ b/docs/docs/tutorials/star-wars/command.php @@ -0,0 +1,27 @@ +jobLauncher->launch(ImportStarWarsJob::getJobName()); + + return self::SUCCESS; + } +} diff --git a/docs/docs/tutorials/star-wars/entity-character.php b/docs/docs/tutorials/star-wars/entity-character.php new file mode 100644 index 00000000..9330cc69 --- /dev/null +++ b/docs/docs/tutorials/star-wars/entity-character.php @@ -0,0 +1,35 @@ + $value === 'NA' ? null : $value, + ), + new CallbackProcessor($process), + new SkipInvalidItemProcessor($validator), + ]), + new ObjectWriter($doctrine), + $executionStorage, + ), + ); + } +} diff --git a/docs/docs/tutorials/star-wars/job-character.php b/docs/docs/tutorials/star-wars/job-character.php new file mode 100644 index 00000000..4593e217 --- /dev/null +++ b/docs/docs/tutorials/star-wars/job-character.php @@ -0,0 +1,43 @@ +name = $item['name']; + $entity->birthYear = $item['birth_year'] ? (int)$item['birth_year'] : null; + $entity->gender = $item['gender'] ?? 'unknown'; + $entity->homeWorld = $doctrine->getRepository(Planet::class) + ->findOneBy(['name' => $item['homeworld']]); + $entity->specie = $doctrine->getRepository(Specie::class) + ->findOneBy(['name' => $item['species']]); + + return $entity; + }, + $validator, + $doctrine, + $executionStorage, + ); + } +} diff --git a/docs/docs/tutorials/star-wars/job-parent.php b/docs/docs/tutorials/star-wars/job-parent.php new file mode 100644 index 00000000..804b8a9b --- /dev/null +++ b/docs/docs/tutorials/star-wars/job-parent.php @@ -0,0 +1,28 @@ +name = $item['name']; + $entity->rotationPeriod = $item['rotation_period'] ? (int)$item['rotation_period'] : null; + $entity->orbitalPeriod = $item['orbital_period'] ? (int)$item['orbital_period'] : null; + $entity->population = $item['population'] ? (int)$item['population'] : null; + $entity->terrain = \array_filter( + \array_map('trim', \explode(',', (string)$item['terrain'])) + ); + + return $entity; + }, + $validator, + $doctrine, + $executionStorage, + ); + } +} diff --git a/docs/docs/tutorials/star-wars/job-specie.php b/docs/docs/tutorials/star-wars/job-specie.php new file mode 100644 index 00000000..d671ed71 --- /dev/null +++ b/docs/docs/tutorials/star-wars/job-specie.php @@ -0,0 +1,42 @@ +name = $item['name']; + $entity->classification = $item['classification']; + $entity->language = $item['language']; + if ($item['homeworld']) { + $entity->homeWorld = $doctrine->getRepository(Planet::class) + ->findOneBy(['name' => $item['homeworld']]); + } + + return $entity; + }, + $validator, + $doctrine, + $executionStorage, + ); + } +} diff --git a/docs/pyproject.toml b/docs/pyproject.toml new file mode 100644 index 00000000..31fd2fce --- /dev/null +++ b/docs/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = ["flit_core >=3.2,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "Yokai Batch" +authors = [{name = "Yann Eugoné", email = "eugone.yann@gmail.com"}] +dynamic = ["version", "description"] diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..53fc1f32 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx==7.1.2 +sphinx-rtd-theme==1.3.0rc1 diff --git a/tests/convention/Documentation/DocFile.php b/tests/convention/Documentation/DocFile.php index 03944a67..ff6fba3e 100644 --- a/tests/convention/Documentation/DocFile.php +++ b/tests/convention/Documentation/DocFile.php @@ -7,12 +7,11 @@ use SplFileInfo; /** - * A Markdown documentation file. + * A Sphinx documentation file. */ final class DocFile { public function __construct( - public string $package, public SplFileInfo $file, /** * @var array diff --git a/tests/convention/Documentation/DocLink.php b/tests/convention/Documentation/DocLink.php index bfdf483e..0d505e88 100644 --- a/tests/convention/Documentation/DocLink.php +++ b/tests/convention/Documentation/DocLink.php @@ -7,17 +7,15 @@ use SplFileInfo; /** - * A markdown link in a markdown documentation file. + * A Sphinx link in a Sphinx documentation file. */ final class DocLink { public function __construct( public string $label, - public string $package, public string $uri, public SplFileInfo $pointsToFile, public string $branch, - public bool $absolute, ) { } } diff --git a/tests/convention/Documentation/MarkdownLinksTest.php b/tests/convention/Documentation/DocumentationLinksTest.php similarity index 81% rename from tests/convention/Documentation/MarkdownLinksTest.php rename to tests/convention/Documentation/DocumentationLinksTest.php index eb6616ec..ebe142df 100644 --- a/tests/convention/Documentation/MarkdownLinksTest.php +++ b/tests/convention/Documentation/DocumentationLinksTest.php @@ -17,14 +17,14 @@ use Yokai\Batch\Storage\JobExecutionStorageInterface; /** - * Some assertions on markdown documentation files. + * Some assertions on Sphinx documentation files. */ -final class MarkdownLinksTest extends TestCase +final class DocumentationLinksTest extends TestCase { private const DEFAULT_BRANCH = '0.x'; /** - * Ensure that all links in markdown files points to valid internal resources. + * Ensure that all links in Sphinx files points to valid internal resources. * * @dataProvider filesWithLinks */ @@ -32,13 +32,9 @@ public function testInternalLinksAreValid(DocFile $file): void { /** @var DocLink $link */ foreach ($file->links as $link) { - if ($link->package !== $file->package) { - self::assertTrue($link->absolute, 'When pointing to another package, links must be absolute.'); - } - self::assertNotFalse( $link->pointsToFile->getRealPath(), - "Link \"{$link->label}\" in \"{$link->pointsToFile->getRealPath()}\"," . + "Link \"{$link->label}\" in \"{$link->pointsToFile->getPathname()}\"," . " is pointing to \"{$link->uri}\" which reference an internal file that do not exists." ); @@ -49,7 +45,7 @@ public function testInternalLinksAreValid(DocFile $file): void public function filesWithLinks(): iterable { /** @var DocFile $file */ - foreach (Markdown::listFiles() as $file) { + foreach (Sphinx::listFiles() as $file) { if (\count($file->links) === 0) { continue; } @@ -78,7 +74,7 @@ public function testComponentsAreListed(string $filepath, string $interface): vo } // Find all links in these files that points to file that implement these interfaces - $file = Markdown::getFile($filepath); + $file = Sphinx::getFile($filepath); /** @var DocLink $link */ foreach ($file->links as $link) { if (!\str_ends_with($link->uri, '.php')) { @@ -105,31 +101,31 @@ public function testComponentsAreListed(string $filepath, string $interface): vo public function interfaceRules(): iterable { yield 'JobInterface' => [ - 'docs/batch/domain/job.md', + 'docs/docs/core-concepts/job.rst', JobInterface::class, ]; yield 'JobExecutionStorageInterface' => [ - 'docs/batch/domain/job-execution-storage.md', + 'docs/docs/core-concepts/job-execution-storage.rst', JobExecutionStorageInterface::class, ]; yield 'JobLauncherInterface' => [ - 'docs/batch/domain/job-launcher.md', + 'docs/docs/core-concepts/job-launcher.rst', JobLauncherInterface::class, ]; yield 'JobParameterAccessorInterface' => [ - 'docs/batch/domain/job-parameter-accessor.md', + 'docs/docs/core-concepts/job-parameter-accessor.rst', JobParameterAccessorInterface::class, ]; yield 'ItemReaderInterface' => [ - 'docs/batch/domain/item-job/item-reader.md', + 'docs/docs/core-concepts/item-job/item-reader.rst', ItemReaderInterface::class, ]; yield 'ItemProcessorInterface' => [ - 'docs/batch/domain/item-job/item-processor.md', + 'docs/docs/core-concepts/item-job/item-processor.rst', ItemProcessorInterface::class, ]; yield 'ItemWriterInterface' => [ - 'docs/batch/domain/item-job/item-writer.md', + 'docs/docs/core-concepts/item-job/item-writer.rst', ItemWriterInterface::class, ]; } diff --git a/tests/convention/Documentation/Markdown.php b/tests/convention/Documentation/Sphinx.php similarity index 53% rename from tests/convention/Documentation/Markdown.php rename to tests/convention/Documentation/Sphinx.php index ddb3746d..cead35c2 100644 --- a/tests/convention/Documentation/Markdown.php +++ b/tests/convention/Documentation/Sphinx.php @@ -6,28 +6,28 @@ use Generator; use SplFileInfo; +use Symfony\Component\Filesystem\Path; use Symfony\Component\Finder\Finder; /** * This util class is a wrapper around Symfony's finder. - * It will search for markdown files considered as documentation. + * It will search for Sphinx files considered as documentation. * Every fetched files will be converted to a {@see DocFile}. */ -final class Markdown +final class Sphinx { - private const MARKDOWN_LINKS_REGEX = '/\[([^]]+)\]\(([^)]+)\)/'; - private const GITHUB_BATCH_LINK_REGEX = '#https\:\/\/github\.com\/yokai\-php\/([^/]+)\/blob\/([^/]+)\/(.+)#'; - private const DEFAULT_BRANCH = '0.x'; + private const SPHINX_LINKS_REGEX = '/`([^\n<]+)<([^>]+)>`__/'; + private const GITHUB_BATCH_LINK_REGEX = '#https\:\/\/github\.com\/yokai\-php\/([^/]+)\/(blob|tree)\/([^/]+)\/(.+)#'; private const ROOT_DIR = __DIR__ . '/../../..'; /** - * List every known markdown documentation files in yokai batch packages. + * List every known Sphinx documentation files in yokai batch packages. * * @return iterable */ public static function listFiles(): iterable { - $files = Finder::create()->files()->in(self::ROOT_DIR . '/docs/*/')->name('*.md'); + $files = Finder::create()->files()->in(self::ROOT_DIR . '/docs/docs/*/')->name('*.rst'); foreach ($files as $file) { yield self::createDocFile($file); @@ -35,7 +35,7 @@ public static function listFiles(): iterable } /** - * Convert a markdown documentation file to a {@see DocFile}. + * Convert a Sphinx documentation file to a {@see DocFile}. */ public static function getFile(string $path): DocFile { @@ -44,52 +44,42 @@ public static function getFile(string $path): DocFile private static function createDocFile(SplFileInfo $file): DocFile { - $package = self::getPackageFromFile($file); - $links = \iterator_to_array(self::listLinksInFile($file->getRealPath(), $package)); + $links = \iterator_to_array(self::listLinksInFile($file->getRealPath())); - return new DocFile($package, $file, $links); + return new DocFile($file, $links); } /** * @return Generator */ - private static function listLinksInFile(string $path, string $filePackage): Generator + private static function listLinksInFile(string $path): Generator { - \preg_match_all(self::MARKDOWN_LINKS_REGEX, \file_get_contents($path), $fileLinks); + \preg_match_all(self::SPHINX_LINKS_REGEX, \file_get_contents($path), $fileLinks); foreach (\array_keys($fileLinks[0]) as $idx) { - $label = $fileLinks[1][$idx]; + $label = \trim($fileLinks[1][$idx]); $uri = $fileLinks[2][$idx]; - if (\str_starts_with($label, '!')) { - continue; // it's an image - } - $absolute = \str_starts_with($uri, 'https://') || \str_starts_with($uri, 'http://'); if ($absolute) { if (!\preg_match(self::GITHUB_BATCH_LINK_REGEX, $uri, $internalLink)) { continue; // it's an external link } - [, $package, $branch, $linkPath] = $internalLink; + [, $package, , $branch, $linkPath] = $internalLink; } else { - $package = $filePackage; - $branch = self::DEFAULT_BRANCH; - $linkPath = $uri; + throw new \LogicException($fileLinks[0][$idx]);//todo } $linkFile = self::getFileFromMembers($package, $linkPath); - yield new DocLink($label, $package, $uri, $linkFile, $branch, $absolute); + yield new DocLink($label, $uri, $linkFile, $branch); } } - private static function getPackageFromFile(SplFileInfo $file): string - { - $relativePath = \str_replace(\realpath(self::ROOT_DIR) . '/', '', $file->getRealPath()); - - return \dirname($relativePath); - } - private static function getFileFromMembers(string $package, string $file): SplFileInfo { - return new SplFileInfo(self::ROOT_DIR . '/' . $package . '/' . \ltrim($file, '/')); + if ($package === 'batch-src') { + return new SplFileInfo(Path::canonicalize(self::ROOT_DIR . '/' . \ltrim($file, '/'))); + } + + return new SplFileInfo(Path::canonicalize(self::ROOT_DIR . '/src/' . $package . '/' . \ltrim($file, '/'))); } }