Skip to content

Commit f1154d4

Browse files
duhonduhon
authored
Merge pull request #2968 from magento-performance/MAGETWO-93678
[performance] MAGETWO-93678: [Forwardport] Implement sharding and parallelization for Price Indexer
2 parents a5107ab + c52e0cb commit f1154d4

File tree

108 files changed

+6736
-1290
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

108 files changed

+6736
-1290
lines changed

‎app/code/Magento/AdvancedSearch/Model/ResourceModel/Index.php

Lines changed: 29 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,22 @@
66
namespace Magento\AdvancedSearch\Model\ResourceModel;
77

88
use Magento\Framework\Model\ResourceModel\Db\AbstractDb;
9+
use Magento\Framework\Search\Request\IndexScopeResolverInterface;
910
use Magento\Store\Model\StoreManagerInterface;
1011
use Magento\Framework\Model\ResourceModel\Db\Context;
1112
use Magento\Framework\EntityManager\MetadataPool;
1213
use Magento\Catalog\Api\Data\CategoryInterface;
1314
use Magento\Framework\App\ObjectManager;
14-
use Magento\Framework\Indexer\ScopeResolver\IndexScopeResolver as TableResolver;
1515
use Magento\Framework\Search\Request\Dimension;
1616
use Magento\Catalog\Model\Indexer\Category\Product\AbstractAction;
17+
use Magento\Framework\Search\Request\IndexScopeResolverInterface as TableResolver;
18+
use Magento\Catalog\Model\Indexer\Product\Price\DimensionCollectionFactory;
1719

1820
/**
1921
* @api
2022
* @since 100.1.0
23+
*
24+
* @SuppressWarnings(PHPMD.CouplingBetweenObjects)
2125
*/
2226
class Index extends AbstractDb
2327
{
@@ -38,25 +42,34 @@ class Index extends AbstractDb
3842
*/
3943
private $tableResolver;
4044

45+
/**
46+
* @var DimensionCollectionFactory|null
47+
*/
48+
private $dimensionCollectionFactory;
49+
4150
/**
4251
* Index constructor.
4352
* @param Context $context
4453
* @param StoreManagerInterface $storeManager
4554
* @param MetadataPool $metadataPool
4655
* @param null $connectionName
4756
* @param TableResolver|null $tableResolver
57+
* @param DimensionCollectionFactory|null $dimensionCollectionFactory
4858
*/
4959
public function __construct(
5060
Context $context,
5161
StoreManagerInterface $storeManager,
5262
MetadataPool $metadataPool,
5363
$connectionName = null,
54-
TableResolver $tableResolver = null
64+
TableResolver $tableResolver = null,
65+
DimensionCollectionFactory $dimensionCollectionFactory = null
5566
) {
5667
parent::__construct($context, $connectionName);
5768
$this->storeManager = $storeManager;
5869
$this->metadataPool = $metadataPool;
59-
$this->tableResolver = $tableResolver ?: ObjectManager::getInstance()->get(TableResolver::class);
70+
$this->tableResolver = $tableResolver ?: ObjectManager::getInstance()->get(IndexScopeResolverInterface::class);
71+
$this->dimensionCollectionFactory = $dimensionCollectionFactory
72+
?: ObjectManager::getInstance()->get(DimensionCollectionFactory::class);
6073
}
6174

6275
/**
@@ -78,18 +91,22 @@ protected function _construct()
7891
protected function _getCatalogProductPriceData($productIds = null)
7992
{
8093
$connection = $this->getConnection();
81-
82-
$select = $connection->select()->from(
83-
$this->getTable('catalog_product_index_price'),
84-
['entity_id', 'customer_group_id', 'website_id', 'min_price']
85-
);
86-
87-
if ($productIds) {
88-
$select->where('entity_id IN (?)', $productIds);
94+
$catalogProductIndexPriceSelect = [];
95+
96+
foreach ($this->dimensionCollectionFactory->create() as $dimensions) {
97+
$catalogProductIndexPriceSelect[] = $connection->select()->from(
98+
$this->tableResolver->resolve('catalog_product_index_price', $dimensions),
99+
['entity_id', 'customer_group_id', 'website_id', 'min_price']
100+
);
101+
if ($productIds) {
102+
current($catalogProductIndexPriceSelect)->where('entity_id IN (?)', $productIds);
103+
}
89104
}
90105

106+
$catalogProductIndexPriceUnionSelect = $connection->select()->union($catalogProductIndexPriceSelect);
107+
91108
$result = [];
92-
foreach ($connection->fetchAll($select) as $row) {
109+
foreach ($connection->fetchAll($catalogProductIndexPriceUnionSelect) as $row) {
93110
$result[$row['website_id']][$row['entity_id']][$row['customer_group_id']] = round($row['min_price'], 2);
94111
}
95112

‎app/code/Magento/AdvancedSearch/Test/Unit/Model/ResourceModel/IndexTest.php

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
use Magento\Framework\App\ResourceConnection;
1616
use Magento\Framework\DB\Select;
1717

18+
/**
19+
* @SuppressWarnings(PHPMD.CouplingBetweenObjects)
20+
*/
1821
class IndexTest extends \PHPUnit\Framework\TestCase
1922
{
2023
/**
@@ -59,10 +62,24 @@ protected function setUp()
5962
$this->resourceConnectionMock->expects($this->any())->method('getConnection')->willReturn($this->adapterMock);
6063
$this->metadataPoolMock = $this->createMock(MetadataPool::class);
6164

65+
$indexScopeResolverMock = $this->createMock(
66+
\Magento\Framework\Indexer\ScopeResolver\IndexScopeResolver::class
67+
);
68+
$traversableMock = $this->createMock(\Traversable::class);
69+
$dimensionsMock = $this->createMock(\Magento\Framework\Indexer\MultiDimensionProvider::class);
70+
$dimensionsMock->method('getIterator')->willReturn($traversableMock);
71+
$dimensionFactoryMock = $this->createMock(
72+
\Magento\Catalog\Model\Indexer\Product\Price\DimensionCollectionFactory::class
73+
);
74+
$dimensionFactoryMock->method('create')->willReturn($dimensionsMock);
75+
6276
$this->model = new Index(
6377
$this->resourceContextMock,
6478
$this->storeManagerMock,
65-
$this->metadataPoolMock
79+
$this->metadataPoolMock,
80+
'connectionName',
81+
$indexScopeResolverMock,
82+
$dimensionFactoryMock
6683
);
6784
}
6885

@@ -71,11 +88,13 @@ public function testGetPriceIndexDataUsesFrontendPriceIndexerTable()
7188
$storeId = 1;
7289
$storeMock = $this->createMock(StoreInterface::class);
7390
$storeMock->expects($this->any())->method('getId')->willReturn($storeId);
91+
$storeMock->method('getWebsiteId')->willReturn(1);
7492
$this->storeManagerMock->expects($this->once())->method('getStore')->with($storeId)->willReturn($storeMock);
7593

7694
$selectMock = $this->createMock(Select::class);
7795
$selectMock->expects($this->any())->method('from')->willReturnSelf();
7896
$selectMock->expects($this->any())->method('where')->willReturnSelf();
97+
$selectMock->expects($this->any())->method('union')->willReturnSelf();
7998
$this->adapterMock->expects($this->once())->method('select')->willReturn($selectMock);
8099
$this->adapterMock->expects($this->once())->method('fetchAll')->with($selectMock)->willReturn([]);
81100

‎app/code/Magento/AdvancedSearch/etc/di.xml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,25 @@
1919
<argument name="title" xsi:type="string" translatable="true">Did you mean</argument>
2020
</arguments>
2121
</type>
22+
<type name="Magento\AdvancedSearch\Model\Client\ClientResolver">
23+
<arguments>
24+
<argument name="path" xsi:type="const">Magento\CatalogSearch\Model\ResourceModel\EngineInterface::CONFIG_ENGINE_PATH</argument>
25+
<argument name="scopeType" xsi:type="const">\Magento\Store\Model\ScopeInterface::SCOPE_STORE</argument>
26+
</arguments>
27+
</type>
2228
<type name="Magento\AdvancedSearch\Model\SuggestedQueries">
2329
<arguments>
2430
<argument name="data" xsi:type="array">
2531
<item name="mysql" xsi:type="string">Magento\AdvancedSearch\Model\DataProvider\Suggestions</item>
2632
</argument>
2733
</arguments>
2834
</type>
35+
<type name="Magento\AdvancedSearch\Model\ResourceModel\Index">
36+
<arguments>
37+
<argument name="tableResolver" xsi:type="object">
38+
Magento\Catalog\Model\Indexer\Product\Price\PriceTableResolver
39+
</argument>
40+
</arguments>
41+
</type>
2942
<preference for="Magento\AdvancedSearch\Model\Adapter\DataMapper\AdditionalFieldsProviderInterface" type="Magento\AdvancedSearch\Model\Adapter\DataMapper\AdditionalFieldsProvider" />
3043
</config>

‎app/code/Magento/Catalog/Console/Command/PriceIndexerDimensionsModeSetCommand.php

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
declare(strict_types=1);
7+
8+
namespace Magento\Catalog\Console\Command;
9+
10+
use Magento\Catalog\Model\Indexer\Product\Price\DimensionModeConfiguration;
11+
use Symfony\Component\Console\Input\InputInterface;
12+
use Symfony\Component\Console\Output\OutputInterface;
13+
use Symfony\Component\Console\Input\InputOption;
14+
use Symfony\Component\Console\Input\InputArgument;
15+
use Magento\Framework\Exception\LocalizedException;
16+
use Magento\Indexer\Console\Command\AbstractIndexerCommand;
17+
use Magento\Framework\App\ObjectManagerFactory;
18+
use Magento\Catalog\Model\Indexer\Product\Price\ModeSwitcher;
19+
use Magento\Framework\App\Config\ScopeConfigInterface;
20+
use Magento\Framework\App\Config\ConfigResource\ConfigInterface;
21+
use Magento\Framework\App\Cache\TypeListInterface;
22+
23+
/**
24+
* Command to change price indexer dimensions mode
25+
*
26+
* @SuppressWarnings(PHPMD.CouplingBetweenObjects)
27+
*/
28+
class PriceIndexerDimensionsModeSetCommand extends AbstractIndexerCommand
29+
{
30+
const INPUT_KEY_MODE = 'mode';
31+
32+
/**
33+
* ScopeConfigInterface
34+
*
35+
* @var ScopeConfigInterface
36+
*/
37+
private $configReader;
38+
39+
/**
40+
* ConfigInterface
41+
*
42+
* @var ConfigInterface
43+
*/
44+
private $configWriter;
45+
46+
/**
47+
* TypeListInterface
48+
*
49+
* @var TypeListInterface
50+
*/
51+
private $cacheTypeList;
52+
53+
/**
54+
* ModeSwitcher
55+
*
56+
* @var ModeSwitcher
57+
*/
58+
private $modeSwitcher;
59+
60+
/**
61+
* @param ObjectManagerFactory $objectManagerFactory
62+
* @param ScopeConfigInterface $configReader
63+
* @param ConfigInterface $configWriter
64+
* @param TypeListInterface $cacheTypeList
65+
* @param ModeSwitcher $modeSwitcher
66+
*/
67+
public function __construct(
68+
ObjectManagerFactory $objectManagerFactory,
69+
ScopeConfigInterface $configReader,
70+
ConfigInterface $configWriter,
71+
TypeListInterface $cacheTypeList,
72+
ModeSwitcher $modeSwitcher
73+
) {
74+
$this->configReader = $configReader;
75+
$this->configWriter = $configWriter;
76+
$this->cacheTypeList = $cacheTypeList;
77+
$this->modeSwitcher = $modeSwitcher;
78+
parent::__construct($objectManagerFactory);
79+
}
80+
81+
/**
82+
* {@inheritdoc}
83+
*/
84+
protected function configure()
85+
{
86+
$this->setName('indexer:set-dimensions-mode:catalog_product_price')
87+
->setDescription('Set Indexer Dimensions Mode')
88+
->setDefinition($this->getInputList());
89+
90+
parent::configure();
91+
}
92+
93+
/**
94+
* {@inheritdoc}
95+
*/
96+
protected function execute(InputInterface $input, OutputInterface $output)
97+
{
98+
$errors = $this->validate($input);
99+
100+
if ($errors) {
101+
throw new \InvalidArgumentException(implode(PHP_EOL, $errors));
102+
}
103+
104+
$returnValue = \Magento\Framework\Console\Cli::RETURN_SUCCESS;
105+
106+
$indexer = $this->getObjectManager()->get(\Magento\Indexer\Model\Indexer::class);
107+
$indexer->load(\Magento\Catalog\Model\Indexer\Product\Price\Processor::INDEXER_ID);
108+
109+
try {
110+
$currentMode = $input->getArgument(self::INPUT_KEY_MODE);
111+
$previousMode = $this->configReader->getValue(ModeSwitcher::XML_PATH_PRICE_DIMENSIONS_MODE) ?:
112+
DimensionModeConfiguration::DIMENSION_NONE;
113+
114+
if ($previousMode !== $currentMode) {
115+
//Create new tables and move data
116+
$this->modeSwitcher->createTables($currentMode);
117+
$this->modeSwitcher->moveData($currentMode, $previousMode);
118+
119+
//Change config options
120+
$this->configWriter->saveConfig(ModeSwitcher::XML_PATH_PRICE_DIMENSIONS_MODE, $currentMode);
121+
$this->cacheTypeList->cleanType('config');
122+
$indexer->invalidate();
123+
124+
//Delete old tables
125+
$this->modeSwitcher->dropTables($previousMode);
126+
127+
$output->writeln(
128+
'Dimensions mode for indexer ' . $indexer->getTitle() . ' was changed from \''
129+
. $previousMode . '\' to \'' . $currentMode . '\''
130+
);
131+
} else {
132+
$output->writeln('Dimensions mode for indexer ' . $indexer->getTitle() . ' has not been changed');
133+
}
134+
} catch (LocalizedException $e) {
135+
$output->writeln($e->getMessage() . PHP_EOL);
136+
// we must have an exit code higher than zero to indicate something was wrong
137+
$returnValue = \Magento\Framework\Console\Cli::RETURN_FAILURE;
138+
} catch (\Exception $e) {
139+
$output->writeln($indexer->getTitle() . " indexer process unknown error:" . PHP_EOL);
140+
$output->writeln($e->getMessage() . PHP_EOL);
141+
// we must have an exit code higher than zero to indicate something was wrong
142+
$returnValue = \Magento\Framework\Console\Cli::RETURN_FAILURE;
143+
}
144+
145+
return $returnValue;
146+
}
147+
148+
/**
149+
* Get list of arguments for the command
150+
*
151+
* @return InputOption[]
152+
*/
153+
public function getInputList(): array
154+
{
155+
$modeOptions[] = new InputArgument(
156+
self::INPUT_KEY_MODE,
157+
InputArgument::REQUIRED,
158+
'Indexer dimensions mode ['. DimensionModeConfiguration::DIMENSION_NONE
159+
. '|' . DimensionModeConfiguration::DIMENSION_WEBSITE
160+
. '|' . DimensionModeConfiguration::DIMENSION_CUSTOMER_GROUP
161+
. '|' . DimensionModeConfiguration::DIMENSION_WEBSITE_AND_CUSTOMER_GROUP .']'
162+
);
163+
return $modeOptions;
164+
}
165+
166+
/**
167+
* Check if all admin options are provided
168+
*
169+
* @param InputInterface $input
170+
* @return string[]
171+
*/
172+
public function validate(InputInterface $input): array
173+
{
174+
$errors = [];
175+
176+
$acceptedModeValues = ' Accepted values for ' . self::INPUT_KEY_MODE . ' are \''
177+
. DimensionModeConfiguration::DIMENSION_NONE . '\', \''
178+
. DimensionModeConfiguration::DIMENSION_WEBSITE . '\', \''
179+
. DimensionModeConfiguration::DIMENSION_CUSTOMER_GROUP . '\', \''
180+
. DimensionModeConfiguration::DIMENSION_WEBSITE_AND_CUSTOMER_GROUP . '\'';
181+
182+
$inputMode = $input->getArgument(self::INPUT_KEY_MODE);
183+
if (!$inputMode) {
184+
$errors[] = 'Missing argument \'' . self::INPUT_KEY_MODE .'\'.' . $acceptedModeValues;
185+
} elseif (!in_array(
186+
$inputMode,
187+
[
188+
DimensionModeConfiguration::DIMENSION_NONE,
189+
DimensionModeConfiguration::DIMENSION_WEBSITE,
190+
DimensionModeConfiguration::DIMENSION_CUSTOMER_GROUP,
191+
DimensionModeConfiguration::DIMENSION_WEBSITE_AND_CUSTOMER_GROUP
192+
]
193+
)) {
194+
$errors[] = $acceptedModeValues;
195+
}
196+
return $errors;
197+
}
198+
}

‎app/code/Magento/Catalog/Model/Indexer/Category/Product/Plugin/StoreGroup.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ protected function validate(AbstractModel $group)
9797
public function afterDelete(AbstractDb $subject, AbstractDb $objectResource, AbstractModel $storeGroup)
9898
{
9999
foreach ($storeGroup->getStores() as $store) {
100-
$this->tableMaintainer->dropTablesForStore($store->getId());
100+
$this->tableMaintainer->dropTablesForStore((int)$store->getId());
101101
}
102102
return $objectResource;
103103
}

‎app/code/Magento/Catalog/Model/Indexer/Category/Product/Plugin/StoreView.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public function afterSave(AbstractDb $subject, AbstractDb $objectResource, Abstr
5151
*/
5252
public function afterDelete(AbstractDb $subject, AbstractDb $objectResource, AbstractModel $store)
5353
{
54-
$this->tableMaintainer->dropTablesForStore($store->getId());
54+
$this->tableMaintainer->dropTablesForStore((int)$store->getId());
5555
return $objectResource;
5656
}
5757
}

‎app/code/Magento/Catalog/Model/Indexer/Category/Product/Plugin/Website.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ public function __construct(
3939
public function afterDelete(AbstractDb $subject, AbstractDb $objectResource, AbstractModel $website)
4040
{
4141
foreach ($website->getStoreIds() as $storeId) {
42-
$this->tableMaintainer->dropTablesForStore($storeId);
42+
$this->tableMaintainer->dropTablesForStore((int)$storeId);
4343
}
4444
return $objectResource;
4545
}

0 commit comments

Comments
 (0)