Skip to content

Commit 2e2af0a

Browse files
vzabaznovvzabaznov
authored
Merge pull request #6590 from magento-performance/MCP-180
MCP-180: elastic index improvements
2 parents 516acb2 + 0978e92 commit 2e2af0a

File tree

6 files changed

+69
-34
lines changed

6 files changed

+69
-34
lines changed

‎app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,10 @@ class BatchSizeCalculator
4343
private const DEPLOYMENT_CONFIG_INDEXER_BATCHES = 'indexer/batch_size/';
4444

4545
/**
46-
* BatchSizeCalculator constructor.
4746
* @param array $batchRowsCount
4847
* @param array $estimators
4948
* @param array $batchSizeAdjusters
49+
* @param DeploymentConfig|null $deploymentConfig
5050
*/
5151
public function __construct(
5252
array $batchRowsCount,

‎app/code/Magento/CatalogSearch/Model/Indexer/Fulltext.php

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ class Fulltext implements
111111
* @param array $data
112112
* @param ProcessManager|null $processManager
113113
* @param int|null $batchSize
114-
* @param DeploymentConfig $deploymentConfig
114+
* @param DeploymentConfig|null $deploymentConfig
115115
* @SuppressWarnings(PHPMD.UnusedFormalParameter)
116116
*/
117117
public function __construct(
@@ -182,7 +182,7 @@ public function executeByDimensions(array $dimensions, \Traversable $entityIds =
182182
$i = 0;
183183

184184
$this->batchSize = $this->deploymentConfig->get(
185-
self::DEPLOYMENT_CONFIG_INDEXER_BATCHES . self::INDEXER_ID
185+
self::DEPLOYMENT_CONFIG_INDEXER_BATCHES . self::INDEXER_ID . '/partial_reindex'
186186
) ?? $this->batchSize;
187187

188188
foreach ($entityIds as $entityId) {

‎app/code/Magento/CatalogSearch/Model/Indexer/Fulltext/Action/Full.php

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
use Magento\Catalog\Api\Data\ProductInterface;
99
use Magento\CatalogSearch\Model\Indexer\Fulltext;
10+
use Magento\Framework\App\DeploymentConfig;
1011
use Magento\Framework\App\ObjectManager;
1112
use Magento\Framework\App\ResourceConnection;
1213

@@ -199,6 +200,19 @@ class Full
199200
private $batchSize;
200201

201202
/**
203+
* @var DeploymentConfig|null
204+
*/
205+
private $deploymentConfig;
206+
207+
/**
208+
* Deployment config path
209+
*
210+
* @var string
211+
*/
212+
private const DEPLOYMENT_CONFIG_INDEXER_BATCHES = 'indexer/batch_size/';
213+
214+
/**
215+
* Full constructor.
202216
* @param ResourceConnection $resource
203217
* @param \Magento\Catalog\Model\Product\Type $catalogProductType
204218
* @param \Magento\Eav\Model\Config $eavConfig
@@ -216,10 +230,12 @@ class Full
216230
* @param \Magento\CatalogSearch\Model\ResourceModel\Fulltext $fulltextResource
217231
* @param \Magento\Framework\Search\Request\DimensionFactory $dimensionFactory
218232
* @param \Magento\Framework\Indexer\ConfigInterface $indexerConfig
219-
* @param mixed $indexIteratorFactory
233+
* @param null $indexIteratorFactory
220234
* @param \Magento\Framework\EntityManager\MetadataPool|null $metadataPool
221235
* @param DataProvider|null $dataProvider
222236
* @param int $batchSize
237+
* @param DeploymentConfig|null $deploymentConfig
238+
*
223239
* @SuppressWarnings(PHPMD.ExcessiveParameterList)
224240
* @SuppressWarnings(PHPMD.UnusedFormalParameter)
225241
*/
@@ -244,7 +260,8 @@ public function __construct(
244260
$indexIteratorFactory = null,
245261
\Magento\Framework\EntityManager\MetadataPool $metadataPool = null,
246262
DataProvider $dataProvider = null,
247-
$batchSize = 500
263+
$batchSize = 500,
264+
?DeploymentConfig $deploymentConfig = null
248265
) {
249266
$this->resource = $resource;
250267
$this->connection = $resource->getConnection();
@@ -268,6 +285,7 @@ public function __construct(
268285
->get(\Magento\Framework\EntityManager\MetadataPool::class);
269286
$this->dataProvider = $dataProvider ?: ObjectManager::getInstance()->get(DataProvider::class);
270287
$this->batchSize = $batchSize;
288+
$this->deploymentConfig = $deploymentConfig ?: ObjectManager::getInstance()->get(DeploymentConfig::class);
271289
}
272290

273291
/**
@@ -360,6 +378,9 @@ public function rebuildStoreIndex($storeId, $productIds = null)
360378
];
361379

362380
$lastProductId = 0;
381+
$this->batchSize = $this->deploymentConfig->get(
382+
self::DEPLOYMENT_CONFIG_INDEXER_BATCHES . Fulltext::INDEXER_ID . '/mysql_get'
383+
) ?? $this->batchSize;
363384
$products = $this->dataProvider
364385
->getSearchableProducts($storeId, $staticFields, $productIds, $lastProductId, $this->batchSize);
365386
while (count($products) > 0) {

‎app/code/Magento/Elasticsearch/Model/Indexer/IndexerHandler.php

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,11 @@
55
*/
66
namespace Magento\Elasticsearch\Model\Indexer;
77

8+
use Magento\CatalogSearch\Model\Indexer\Fulltext;
89
use Magento\Elasticsearch\Model\Adapter\Elasticsearch as ElasticsearchAdapter;
910
use Magento\Elasticsearch\Model\Adapter\Index\IndexNameResolver;
11+
use Magento\Framework\App\DeploymentConfig;
12+
use Magento\Framework\App\ObjectManager;
1013
use Magento\Framework\App\ScopeResolverInterface;
1114
use Magento\Framework\Indexer\IndexStructureInterface;
1215
use Magento\Framework\Indexer\SaveHandler\Batch;
@@ -59,13 +62,27 @@ class IndexerHandler implements IndexerInterface
5962
private $scopeResolver;
6063

6164
/**
65+
* @var DeploymentConfig|null
66+
*/
67+
private $deploymentConfig;
68+
69+
/**
70+
* Deployment config path
71+
*
72+
* @var string
73+
*/
74+
private const DEPLOYMENT_CONFIG_INDEXER_BATCHES = 'indexer/batch_size/';
75+
76+
/**
77+
* IndexerHandler constructor.
6278
* @param IndexStructureInterface $indexStructure
6379
* @param ElasticsearchAdapter $adapter
6480
* @param IndexNameResolver $indexNameResolver
6581
* @param Batch $batch
6682
* @param ScopeResolverInterface $scopeResolver
6783
* @param array $data
6884
* @param int $batchSize
85+
* @param DeploymentConfig|null $deploymentConfig
6986
*/
7087
public function __construct(
7188
IndexStructureInterface $indexStructure,
@@ -74,7 +91,8 @@ public function __construct(
7491
Batch $batch,
7592
ScopeResolverInterface $scopeResolver,
7693
array $data = [],
77-
$batchSize = self::DEFAULT_BATCH_SIZE
94+
$batchSize = self::DEFAULT_BATCH_SIZE,
95+
?DeploymentConfig $deploymentConfig = null
7896
) {
7997
$this->indexStructure = $indexStructure;
8098
$this->adapter = $adapter;
@@ -83,6 +101,7 @@ public function __construct(
83101
$this->data = $data;
84102
$this->batchSize = $batchSize;
85103
$this->scopeResolver = $scopeResolver;
104+
$this->deploymentConfig = $deploymentConfig ?: ObjectManager::getInstance()->get(DeploymentConfig::class);
86105
}
87106

88107
/**
@@ -92,6 +111,11 @@ public function saveIndex($dimensions, \Traversable $documents)
92111
{
93112
$dimension = current($dimensions);
94113
$scopeId = $this->scopeResolver->getScope($dimension->getValue())->getId();
114+
115+
$this->batchSize = $this->deploymentConfig->get(
116+
self::DEPLOYMENT_CONFIG_INDEXER_BATCHES . Fulltext::INDEXER_ID . '/elastic_save'
117+
) ?? $this->batchSize;
118+
95119
foreach ($this->batch->getItems($documents, $this->batchSize) as $documentsBatch) {
96120
$docs = $this->adapter->prepareDocsPerStore($documentsBatch, $scopeId);
97121
$this->adapter->addDocs($docs, $scopeId, $this->getIndexerId());

‎app/code/Magento/Indexer/Console/Command/IndexerReindexCommand.php

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
use Magento\Framework\Indexer\IndexerRegistry;
1717
use Magento\Framework\Indexer\StateInterface;
1818
use Magento\Indexer\Model\Processor\MakeSharedIndexValid;
19+
use Psr\Log\LoggerInterface;
1920
use Symfony\Component\Console\Input\InputInterface;
2021
use Symfony\Component\Console\Output\OutputInterface;
2122

@@ -50,21 +51,29 @@ class IndexerReindexCommand extends AbstractIndexerManageCommand
5051
*/
5152
private $makeSharedValid;
5253

54+
/**
55+
* @var LoggerInterface
56+
*/
57+
private $logger;
58+
5359
/**
5460
* @param ObjectManagerFactory $objectManagerFactory
5561
* @param IndexerRegistry|null $indexerRegistry
5662
* @param DependencyInfoProvider|null $dependencyInfoProvider
5763
* @param MakeSharedIndexValid|null $makeSharedValid
64+
* @param LoggerInterface|null $logger
5865
*/
5966
public function __construct(
6067
ObjectManagerFactory $objectManagerFactory,
6168
IndexerRegistry $indexerRegistry = null,
6269
DependencyInfoProvider $dependencyInfoProvider = null,
63-
MakeSharedIndexValid $makeSharedValid = null
70+
MakeSharedIndexValid $makeSharedValid = null,
71+
?LoggerInterface $logger = null
6472
) {
6573
$this->indexerRegistry = $indexerRegistry;
6674
$this->dependencyInfoProvider = $dependencyInfoProvider;
6775
$this->makeSharedValid = $makeSharedValid;
76+
$this->logger = $logger ?: ObjectManager::getInstance()->get(LoggerInterface::class);
6877
parent::__construct($objectManagerFactory);
6978
}
7079

@@ -108,15 +117,14 @@ protected function execute(InputInterface $input, OutputInterface $output)
108117
$output->writeln(
109118
__('has been rebuilt successfully in %time', ['time' => gmdate('H:i:s', $resultTime)])
110119
);
111-
} catch (LocalizedException $e) {
112-
$output->writeln(__('exception: %message', ['message' => $e->getMessage()]));
113-
$returnValue = Cli::RETURN_FAILURE;
114-
} catch (\Exception $e) {
115-
$output->writeln('process unknown error:');
120+
} catch (\Throwable $e) {
121+
$output->writeln('process error during indexation process:');
116122
$output->writeln($e->getMessage());
117123

118124
$output->writeln($e->getTraceAsString(), OutputInterface::VERBOSITY_DEBUG);
119125
$returnValue = Cli::RETURN_FAILURE;
126+
127+
$this->logger->critical($e->getMessage());
120128
}
121129
}
122130

‎app/code/Magento/Indexer/Test/Unit/Console/Command/IndexerReindexCommandTest.php

Lines changed: 4 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -422,27 +422,6 @@ public function executeWithIndexDataProvider()
422422
];
423423
}
424424

425-
public function testExecuteWithLocalizedException()
426-
{
427-
$this->configureAdminArea();
428-
$indexerOne = $this->getIndexerMock(
429-
['reindexAll', 'getStatus'],
430-
['indexer_id' => 'indexer_1', 'title' => self::STUB_INDEXER_NAME]
431-
);
432-
$localizedException = new LocalizedException(new Phrase('Some Exception Message'));
433-
$indexerOne->expects($this->once())->method('reindexAll')->willThrowException($localizedException);
434-
$this->initIndexerCollectionByItems([$indexerOne]);
435-
$this->command = new IndexerReindexCommand($this->objectManagerFactory);
436-
$commandTester = new CommandTester($this->command);
437-
$commandTester->execute(['index' => ['indexer_1']]);
438-
$actualValue = $commandTester->getDisplay();
439-
$this->assertSame(Cli::RETURN_FAILURE, $commandTester->getStatusCode());
440-
$this->assertStringStartsWith(
441-
self::STUB_INDEXER_NAME . ' index exception: Some Exception Message',
442-
$actualValue
443-
);
444-
}
445-
446425
public function testExecuteWithException()
447426
{
448427
$this->configureAdminArea();
@@ -459,7 +438,10 @@ public function testExecuteWithException()
459438
$commandTester->execute(['index' => ['indexer_1']]);
460439
$actualValue = $commandTester->getDisplay();
461440
$this->assertSame(Cli::RETURN_FAILURE, $commandTester->getStatusCode());
462-
$this->assertStringStartsWith('Title_indexer_1' . ' index process unknown error:', $actualValue);
441+
$this->assertStringStartsWith(
442+
'Title_indexer_1' . ' index process error during indexation process:',
443+
$actualValue
444+
);
463445
}
464446

465447
public function testExecuteWithExceptionInGetIndexers()

0 commit comments

Comments
 (0)