Skip to content

Commit 869be7d

Browse files
norberttechnorberttech
committed
Initial commit
0 parents  commit 869be7d

File tree

21 files changed

+8611
-0
lines changed

21 files changed

+8611
-0
lines changed

‎.gitattributes‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
*.php text eol=lf
2+
/.github export-ignore
3+
/docs export-ignore
4+
/tests export-ignore
5+
/tools export-ignore
6+
/.gitattributes export-ignore
7+
/.gitignore export-ignore
8+
/.php_cs export-ignore
9+
/composer.lock export-ignore
10+
/phpstan.neon export-ignore
11+
/phpunit.xml export-ignore
12+
/psalm.xml export-ignore
13+
/infection.json export-ignore

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
vendor
2+
*.cache
3+
var

‎.php-cs-fixer.php‎

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
<?php
2+
3+
$finder = PhpCsFixer\Finder::create()
4+
->files()
5+
->in([
6+
__DIR__ . '/src',
7+
__DIR__ . '/tests'
8+
]);
9+
10+
if (!\file_exists(__DIR__ . '/var')) {
11+
\mkdir(__DIR__ . '/var');
12+
}
13+
14+
/**
15+
* This configuration was taken from https://github.com/sebastianbergmann/phpunit/blob/master/.php_cs.dist
16+
* and slightly adjusted.
17+
*/
18+
return (new PhpCsFixer\Config())
19+
->setRiskyAllowed(true)
20+
->setCacheFile(__DIR__.'/var/.php_cs.cache')
21+
->setRules([
22+
'align_multiline_comment' => true,
23+
'array_indentation' => true,
24+
'array_syntax' => ['syntax' => 'short'],
25+
'blank_line_after_namespace' => true,
26+
'blank_line_before_statement' => [
27+
'statements' => [
28+
'break',
29+
'continue',
30+
'declare',
31+
'default',
32+
'do',
33+
'exit',
34+
'for',
35+
'foreach',
36+
'goto',
37+
'if',
38+
'include',
39+
'include_once',
40+
'require',
41+
'require_once',
42+
'return',
43+
'switch',
44+
'throw',
45+
'try',
46+
'while',
47+
],
48+
],
49+
'braces' => true,
50+
'cast_spaces' => true,
51+
'class_attributes_separation' => ['elements' => ['const' => 'one', 'method' => 'one', 'property' => 'one']],
52+
'combine_consecutive_issets' => true,
53+
'combine_consecutive_unsets' => true,
54+
'compact_nullable_typehint' => true,
55+
'concat_space' => ['spacing' => 'one'],
56+
'constant_case' => true,
57+
'declare_equal_normalize' => ['space' => 'none'],
58+
'declare_strict_types' => true,
59+
'dir_constant' => true,
60+
'elseif' => true,
61+
'encoding' => true,
62+
'explicit_indirect_variable' => true,
63+
'explicit_string_variable' => true,
64+
'full_opening_tag' => true,
65+
'fully_qualified_strict_types' => true,
66+
'function_typehint_space' => true,
67+
'function_declaration' => true,
68+
'global_namespace_import' => [
69+
'import_classes' => false,
70+
'import_constants' => false,
71+
'import_functions' => false,
72+
],
73+
'heredoc_to_nowdoc' => true,
74+
'increment_style' => [
75+
'style' => PhpCsFixer\Fixer\Operator\IncrementStyleFixer::STYLE_POST,
76+
],
77+
'indentation_type' => true,
78+
'is_null' => true,
79+
'line_ending' => true,
80+
'list_syntax' => ['syntax' => 'short'],
81+
'logical_operators' => true,
82+
'lowercase_keywords' => true,
83+
'lowercase_static_reference' => true,
84+
'magic_constant_casing' => true,
85+
'magic_method_casing' => true,
86+
'method_argument_space' => ['on_multiline' => 'ensure_fully_multiline'],
87+
'modernize_types_casting' => false,
88+
'multiline_comment_opening_closing' => true,
89+
'multiline_whitespace_before_semicolons' => true,
90+
'native_constant_invocation' => false,
91+
'native_function_casing' => false,
92+
'native_function_invocation' => ['include' => ['@all']],
93+
'native_function_type_declaration_casing' => true,
94+
'new_with_braces' => false,
95+
'no_alias_functions' => true,
96+
'no_alternative_syntax' => true,
97+
'no_blank_lines_after_class_opening' => true,
98+
'no_blank_lines_after_phpdoc' => true,
99+
'no_blank_lines_before_namespace' => false,
100+
'no_closing_tag' => true,
101+
'no_empty_comment' => true,
102+
'no_empty_phpdoc' => true,
103+
'no_empty_statement' => true,
104+
'no_extra_blank_lines' => true,
105+
'no_homoglyph_names' => true,
106+
'no_leading_import_slash' => true,
107+
'no_leading_namespace_whitespace' => true,
108+
'no_mixed_echo_print' => ['use' => 'print'],
109+
'no_multiline_whitespace_around_double_arrow' => true,
110+
'no_null_property_initialization' => true,
111+
'no_php4_constructor' => true,
112+
'no_short_bool_cast' => true,
113+
'echo_tag_syntax' => true,
114+
'no_singleline_whitespace_before_semicolons' => true,
115+
'no_spaces_after_function_name' => true,
116+
'no_spaces_around_offset' => true,
117+
'no_spaces_inside_parenthesis' => true,
118+
'no_superfluous_elseif' => true,
119+
'no_superfluous_phpdoc_tags' => false,
120+
'no_trailing_comma_in_list_call' => true,
121+
'no_trailing_comma_in_singleline_array' => true,
122+
'no_trailing_whitespace' => true,
123+
'no_trailing_whitespace_in_comment' => true,
124+
'no_unneeded_control_parentheses' => true,
125+
'no_unneeded_curly_braces' => true,
126+
'no_unneeded_final_method' => true,
127+
'no_unreachable_default_argument_value' => true,
128+
'no_unset_on_property' => true,
129+
'no_unused_imports' => true,
130+
'no_useless_else' => true,
131+
'no_useless_return' => true,
132+
'no_whitespace_before_comma_in_array' => true,
133+
'no_whitespace_in_blank_line' => true,
134+
'non_printable_character' => true,
135+
'normalize_index_brace' => true,
136+
'object_operator_without_whitespace' => true,
137+
'ordered_class_elements' => [
138+
'order' => [
139+
'use_trait',
140+
'constant_public',
141+
'constant_protected',
142+
'constant_private',
143+
'property_public_static',
144+
'property_protected_static',
145+
'property_private_static',
146+
'property_public',
147+
'property_protected',
148+
'property_private',
149+
'construct',
150+
'method_public_static',
151+
'destruct',
152+
'magic',
153+
'phpunit',
154+
'method_public',
155+
'method_protected',
156+
'method_private',
157+
'method_protected_static',
158+
'method_private_static',
159+
],
160+
],
161+
'ordered_imports' => [
162+
'imports_order' => [
163+
PhpCsFixer\Fixer\Import\OrderedImportsFixer::IMPORT_TYPE_CONST,
164+
PhpCsFixer\Fixer\Import\OrderedImportsFixer::IMPORT_TYPE_FUNCTION,
165+
PhpCsFixer\Fixer\Import\OrderedImportsFixer::IMPORT_TYPE_CLASS,
166+
]
167+
],
168+
'ordered_interfaces' => [
169+
'direction' => 'ascend',
170+
'order' => 'alpha',
171+
],
172+
'phpdoc_add_missing_param_annotation' => false,
173+
'phpdoc_align' => ['align' => 'left'],
174+
'phpdoc_annotation_without_dot' => true,
175+
'phpdoc_indent' => true,
176+
'phpdoc_no_access' => true,
177+
'phpdoc_no_empty_return' => true,
178+
'phpdoc_no_package' => true,
179+
'phpdoc_order' => true,
180+
'phpdoc_return_self_reference' => true,
181+
'phpdoc_scalar' => true,
182+
'phpdoc_separation' => true,
183+
'phpdoc_single_line_var_spacing' => true,
184+
'phpdoc_summary' => true,
185+
'phpdoc_to_comment' => false,
186+
'phpdoc_trim' => true,
187+
'phpdoc_trim_consecutive_blank_line_separation' => true,
188+
'phpdoc_types' => ['groups' => ['simple', 'meta']],
189+
'phpdoc_types_order' => true,
190+
'phpdoc_var_without_name' => true,
191+
'pow_to_exponentiation' => true,
192+
'protected_to_private' => true,
193+
'return_assignment' => true,
194+
'return_type_declaration' => ['space_before' => 'one'],
195+
'self_accessor' => true,
196+
'self_static_accessor' => true,
197+
'semicolon_after_instruction' => true,
198+
'set_type_to_cast' => true,
199+
'short_scalar_cast' => true,
200+
'simple_to_complex_string_variable' => true,
201+
'simplified_null_return' => false,
202+
'single_blank_line_at_eof' => true,
203+
'single_import_per_statement' => true,
204+
'single_line_after_imports' => true,
205+
'single_quote' => true,
206+
'standardize_not_equals' => true,
207+
'strict_param' => true,
208+
'ternary_to_null_coalescing' => true,
209+
'trailing_comma_in_multiline' => true,
210+
'trim_array_spaces' => true,
211+
'unary_operator_spaces' => true,
212+
'visibility_required' => [
213+
'elements' => [
214+
'const',
215+
'method',
216+
'property',
217+
],
218+
],
219+
'void_return' => true,
220+
'whitespace_after_comma_in_array' => true,
221+
])
222+
->setFinder($finder);

‎LICENSE‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2020 Flow PHP
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎README.md‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# ETL - Asynchronous Pipeline ReactPHP Implementation
2+
3+
This repository provides implementation for [flow-php/etl-async](https://github.com/flow-php/etl-async) based on [ReactPHP](https://reactphp.org/) components.
4+
5+
Working example:
6+
7+
```php
8+
<?php
9+
10+
use Flow\ETL\Async\LocalPipeline;
11+
use Flow\ETL\Async\ReactPHP\Worker\ChildProcessLauncher;
12+
use Flow\ETL\Async\ReactPHP\Server\TCPServer;
13+
use Flow\ETL\ETL;
14+
use Flow\ETL\Loader;
15+
use Flow\ETL\Rows;
16+
use Flow\ETL\Transformer\ArrayUnpackTransformer;
17+
use Flow\ETL\Transformer\Cast\CastToInteger;
18+
use Flow\ETL\Transformer\CastTransformer;
19+
use Flow\ETL\Transformer\RemoveEntriesTransformer;
20+
use Flow\ETL\Transformer\Rename\EntryRename;
21+
use Flow\ETL\Transformer\RenameEntriesTransformer;
22+
use Flow\ETL\Transformer\StringConcatTransformer;
23+
use Monolog\Handler\StreamHandler;
24+
use Monolog\Logger;
25+
26+
return new class implements Extractor {
27+
public function extract(): \Generator
28+
{
29+
$rows = [];
30+
for ($i = 0; $i <= 10_000_000; $i++) {
31+
$rows[] = Row::create(
32+
new ArrayEntry(
33+
'row', ['id' => $i, 'name' => 'Name', 'last name' => 'Last Name', 'phone' => '123 123 123']
34+
),
35+
);
36+
37+
if (\count($rows) >= 1000) {
38+
echo "extracted " . $i . "\n ";
39+
yield new Rows(...$rows);
40+
41+
$rows = [];
42+
}
43+
}
44+
45+
if (\count($rows) >= 0) {
46+
yield new Rows(...$rows);
47+
}
48+
}
49+
};
50+
51+
52+
$logger = new Logger('server');
53+
$logger->pushHandler(new StreamHandler(__DIR__ . '/var/logs/server.log', Logger::DEBUG));
54+
$logger->pushHandler(new StreamHandler(__DIR__ . '/var/logs/server_error.log', Logger::ERROR, false));
55+
56+
$pipeline = new LocalPipeline(
57+
new TCPServer($port = 6651, $logger),
58+
new ChildProcessLauncher(__DIR__ . "/bin/worker", $port, $logger),
59+
$workers = 10
60+
);
61+
62+
ETL::extract($extractor, $pipeline)
63+
->transform(new ArrayUnpackTransformer('row'))
64+
->transform(new RemoveEntriesTransformer('row'))
65+
->transform(new CastTransformer(new CastToInteger(['id'])))
66+
->transform(new StringConcatTransformer(['name', 'last name'], ' ', '_name'))
67+
->transform(new RemoveEntriesTransformer('name', 'last name'))
68+
->transform(new RenameEntriesTransformer(new EntryRename('_name', 'name')))
69+
->run();
70+
```

‎bin/worker-reactphp‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/usr/bin/env php
2+
<?php
3+
4+
use Flow\ETL\Async\ReactPHP\Worker\TCPClient;
5+
use Flow\ETL\Async\Client\CLI;
6+
use Flow\ETL\Async\Client\CLI\Input;
7+
use Flow\Serializer\CompressingSerializer;
8+
use Flow\Serializer\NativePHPSerializer;
9+
use Monolog\Formatter\LineFormatter;
10+
use Monolog\Handler\StreamHandler;
11+
use Monolog\Logger;
12+
use Psr\Log\LogLevel;
13+
14+
(function (array $argv){
15+
\error_reporting(E_ALL);
16+
\ini_set('display_errors', 'stderr');
17+
18+
$autoloadPathEnv = \getenv('FLOW_PHP_ASYNC_AUTOLOAD');
19+
20+
if (\is_string($autoloadPathEnv) && \file_exists($autoloadPathEnv)) {
21+
require $autoloadPathEnv;
22+
} else {
23+
if (is_file($autoload = getcwd() . '/vendor/autoload.php')) {
24+
require $autoload;
25+
} elseif (is_file($autoload = getcwd() . '/../../autoload.php')) {
26+
require $autoload;
27+
} elseif (is_file($autoload = __DIR__ . '/../vendor/autoload.php')) {
28+
require($autoload);
29+
} elseif (is_file($autoload = __DIR__ . '/../../../autoload.php')) {
30+
require($autoload);
31+
} else {
32+
\fwrite(STDERR,
33+
'You must set up the project dependencies, run the following commands:' . PHP_EOL .
34+
'curl -s http://getcomposer.org/installer | php' . PHP_EOL .
35+
'php composer.phar install' . PHP_EOL
36+
);
37+
exit(1);
38+
}
39+
}
40+
41+
$logger = new Logger('worker');
42+
$stdout = \fopen("php://stdout", "w+");
43+
$stderr = fopen('php://stderr', 'w+');
44+
$stdoutHandler = new StreamHandler($stdout, LogLevel::DEBUG);
45+
$stderrHandler = new StreamHandler($stderr, LogLevel::ERROR, false);
46+
47+
$logger->pushHandler($stdoutHandler);
48+
$logger->pushHandler($stderrHandler);
49+
50+
$serializer = new CompressingSerializer(new NativePHPSerializer());
51+
52+
$cli = new CLI($logger, new TCPClient($logger, $serializer));
53+
54+
exit($cli->run(new Input($argv)));
55+
})($argv);

0 commit comments

Comments
 (0)