Slices

src/app/code/community/IntegerNet/Solr/Model/Bridge/ProductRepository.php

@@ -66,12 +66,22 @@ function($parentId, $childrenIds) {
     }
 
     /**
+     * @param null|int $sliceId
+     * @param null|int $totalNumberSlices
      * @return int[]
      */
-    public function getAllProductIds()
+    public function getAllProductIds($sliceId = null, $totalNumberSlices = null)
     {
         /** @var $productCollection Mage_Catalog_Model_Resource_Product_Collection */
         $productCollection = Mage::getResourceModel('catalog/product_collection');
+
+        if ((!is_null($sliceId)) && (!is_null($totalNumberSlices))) {
+            if ($sliceId == $totalNumberSlices) {
+                $sliceId = 0;
+            }
+            $productCollection->getSelect()->where('e.entity_id % ' . intval($totalNumberSlices) . ' = ' . intval($sliceId));
+        }
+
         return $productCollection->getAllIds();
     }
 

src/app/code/community/IntegerNet/Solr/Test/Model/Result.php

@@ -95,7 +95,7 @@ public function shouldUseParametersBasedOnToolbar()
             )
         );
         $logMock->expects($this->at(4))->method('debug')->with(
-            'Filter Query: store_id:1 AND is_visible_in_search_i:1');
+            'Filter Query: content_type:product AND store_id:1 AND is_visible_in_search_i:1');
 
         /* @var Mage_Core_Block_Text $toolbar Not using actual toolbar block which reads from session */
         $toolbar = $this->app()->getLayout()->createBlock('core/text', 'product_list_toolbar');

src/shell/integernet-solr.php

@@ -67,6 +67,73 @@ public function run()
                 echo $e->getMessage() . "\n";
             }
 
+        } else if ($this->getArg('reindex_slice')) {
+            $storeIdentifiers = $this->getArg('stores');
+            if (!$storeIdentifiers) {
+                $storeIdentifiers = 'all';
+            }
+            $storeIds = $this->_getStoreIds($storeIdentifiers);
+
+            $autoloader = new IntegerNet_Solr_Helper_Autoloader();
+            $autoloader->createAndRegister();
+
+            try {
+                $sliceArg = $this->getArg('slice');
+                $this->_checkSliceArgument($sliceArg);
+                list($sliceId, $totalNumberSlices) = explode('/', $sliceArg);
+
+                $indexer = Mage::helper('integernet_solr')->factory()->getProductIndexer();
+
+                if ($this->getArg('use_swap_core')) {
+                    $indexer->activateSwapCore();
+                }
+                $indexer->reindex(null, false, $storeIds, $sliceId, $totalNumberSlices);
+                if ($this->getArg('use_swap_core')) {
+                    $indexer->deactivateSwapCore();
+                }
+
+                $storeIdsString = implode(', ', $storeIds);
+                echo "Solr product index rebuilt for Stores {$storeIdsString}.\n";
+                echo '(Slice ' . $sliceId . ' of ' . $totalNumberSlices . ')' . "\n";
+            } catch (Exception $e) {
+                echo $e->getMessage() . "\n";
+            }
+
+        } else if ($this->getArg('clear')) {
+            $storeIdentifiers = $this->getArg('stores');
+            if (!$storeIdentifiers) {
+                $storeIdentifiers = 'all';
+            }
+            $storeIds = $this->_getStoreIds($storeIdentifiers);
+            $indexer = Mage::helper('integernet_solr')->factory()->getProductIndexer();
+            if ($this->getArg('use_swap_core')) {
+                $indexer->activateSwapCore();
+            }
+            foreach($storeIds as $storeId) {
+                $indexer->clearIndex($storeId);
+            }
+            if ($this->getArg('use_swap_core')) {
+                $indexer->deactivateSwapCore();
+            }
+            $storeIdsString = implode(', ', $storeIds);
+            echo "Solr product index cleared for Stores {$storeIdsString}.\n";
+
+        } else if ($this->getArg('swap_cores')) {
+            $storeIdentifiers = $this->getArg('stores');
+            if (!$storeIdentifiers) {
+                $storeIdentifiers = 'all';
+            }
+            $storeIds = $this->_getStoreIds($storeIdentifiers);
+            $indexer = Mage::helper('integernet_solr')->factory()->getProductIndexer();
+            try {
+                $indexer->checkSwapCoresConfiguration($storeIds);
+                $indexer->swapCores($storeIds);
+                $storeIdsString = implode(', ', $storeIds);
+                echo "Solr cores swapped for Stores {$storeIdsString}.\n";
+            } catch (Exception $e) {
+                echo $e->getMessage() . "\n";
+            }
+
         } else {
             echo $this->usageHelp();
         }
@@ -82,12 +149,27 @@ public function usageHelp()
 Usage:  php -f integernet-solr.php -- [options]
         php -f integernet-solr.php -- reindex --stores de
         php -f integernet-solr.php -- reindex --stores all --emptyindex
+        php -f integernet-solr.php -- reindex --stores 1 --slice 1/5 --use_swap_core
+        php -f integernet-solr.php -- clear --stores 1
 
-  reindex           reindex solr for given stores (see "stores" param)
-  --stores <stores> reindex given stores (can be store id, store code, comma seperated. Or "all".) If not set, reindex all stores.
+  reindex           Reindex solr for given stores (see "stores" param)
+  --stores <stores> Reindex given stores (can be store id, store code, comma seperated. Or "all".) If not set, reindex all stores.
   --emptyindex      Force emptying the solr index for the given store(s). If not set, configured value is used.
   --noemptyindex    Force not emptying the solr index for the given store(s). If not set, configured value is used.
   --types <types>   Restrict indexing to certain entity types, i.e. "product", "category" or "page" (comma separated). Or "all". If not set, reindex products.
+
+  reindex_slice     Reindex solr for given stores (see "stores" param). Use this if you want to index only a part of the products, i.e. for letting indexing run in parallel (for products only).
+  --slice <number>/<total_number>, i.e. "1/5" or "2/5". 
+  --stores <stores> Reindex given stores (can be store id, store code, comma seperated. Or "all".) If not set, reindex all stores.
+  --use_swap_core   Use swap core for indexing instead of live solr core (only if configured correctly).
+
+  clear             Clear solr product index for given stores (see "stores" param and "use_swap_core" param)
+  --stores <stores> Reindex given stores (can be store id, store code, comma seperated. Or "all".) If not set, reindex all stores.
+  --use_swap_core   Use swap core for clearing instead of live solr core (only if configured correctly).
+
+  swap_cores        Swap cores. This is useful if using slices (see above) after indexing with the "--use_swap_core" param; it's not needed otherwise.
+  --stores <stores> Reindex given stores (can be store id, store code, comma seperated. Or "all".) If not set, reindex all stores.
+
   help              This help
 
 USAGE;
@@ -149,6 +231,32 @@ protected function _useCmsIndexer()
     {
         return Mage::helper('core')->isModuleEnabled('IntegerNet_SolrPro') && Mage::getStoreConfigFlag('integernet_solr/cms/is_active');
     }
+
+    /**
+     * @param string $sliceArg
+     * @throws InvalidArgumentException
+     */
+    protected function _checkSliceArgument($sliceArg)
+    {
+        if (!strlen($sliceArg)) {
+            throw new InvalidArgumentException('The "slice" argument must be given.');
+        }
+        if (strpos($sliceArg, '/') < 1) {
+            throw new InvalidArgumentException('The "slice" argument must be of format "1/5" or "20/20"');
+        }
+        list($sliceId, $totalNumberSlices) = explode('/', $sliceArg);
+        $sliceId = intval($sliceId);
+        $totalNumberSlices = intval($totalNumberSlices);
+        if (!is_integer($sliceId) || !is_integer($totalNumberSlices)) {
+            throw new InvalidArgumentException('The "slice" argument must be of format "1/5" or "20/20", only containing integer numbers before/after the slash.');
+        }
+        if ($totalNumberSlices < 2) {
+            throw new InvalidArgumentException('The "slice" argument must be of format "1/5" or "20/20". The second number must be higher than 1.');
+        }
+        if ($sliceId < 1 || $sliceId > $totalNumberSlices) {
+            throw new InvalidArgumentException('The "slice" argument must be of format "1/5" or "20/20". The first number is invalid, should be between 1 and the second number (including those).');
+        }
+    }
 }
 
 $shell = new IntegerNet_Solr_Shell();