From 5925b31333658b737faa8d9839f82a6ce5b7d199 Mon Sep 17 00:00:00 2001 From: Vadym Korovin Date: Wed, 27 Jun 2018 20:42:19 +0300 Subject: [PATCH 1/3] Changed shopProductsCollection::categoryPrepare() queries, up to 20 times faster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Немного изменена логика запроса в БД. Теперь, вместо того чтобы JOIN-ть большие таблицы, мы сначала делаем выборку товаров, присутствующих в нужных категориях, а затем эту выборку JOIN-им с shop_product таблицей. Что это нам дает: * отпимизатор больше не использует using_temporary, using_filesort, из-за этого снижается дисковая нагрузка * обычный запрос (без ручной сортировки) выполняется на 95% быстрее (20-кратное ускорение), если раньше на 80к товаров это было 1.84 сек, сейчас - 0.09 сек. * также ускорился COUNT запрос * в ручной сортировке также немного ускорился запрос, т.к. DISTINCT делается в простом подзапросе без джойнов, когда результирующий размер таблицы ещё не такой большой и влезает в буфер Также был проделан небольшой рефакторинг, например, получение алиаса таблицы вынесено в отдельный метод. ВНИМАНИЕ: так как код не покрыт тестами, нет никаких гарантий что где-то что-то не всплывет. Поэтому, нужно тщательно проверить. --- lib/classes/shopProductsCollection.class.php | 156 +++++++++++++++---- 1 file changed, 128 insertions(+), 28 deletions(-) diff --git a/lib/classes/shopProductsCollection.class.php b/lib/classes/shopProductsCollection.class.php index 668899f0a..3d926f4f6 100644 --- a/lib/classes/shopProductsCollection.class.php +++ b/lib/classes/shopProductsCollection.class.php @@ -2,6 +2,7 @@ class shopProductsCollection { + protected $hash; protected $info = array(); @@ -464,22 +465,39 @@ protected function categoryPrepare($id, $auto_title = true) if ($this->info['type'] == shopCategoryModel::TYPE_STATIC) { - $alias = $this->addJoin('shop_category_products'); - if (true - /* && wa()->getEnv() == 'frontend'*/ - && $this->info['include_sub_categories'] - ) { - $this->info['subcategories'] = $category_model->descendants($this->info, true)->where('type = '.shopCategoryModel::TYPE_STATIC)->fetchAll('id'); + $scp = new shopCategoryProductsModel(); + $shop_category_products_builder = new waDbQuery($scp); + $subquery_categories = array( + $id, + ); + + $this->info['subcategories'] = $category_model->descendants($this->info, true)->where('type = '.shopCategoryModel::TYPE_STATIC)->fetchAll('id'); + + if ($this->info['include_sub_categories'] && $this->info['subcategories']) { $descendant_ids = array_keys($this->info['subcategories']); if ($descendant_ids) { - $this->where[] = $alias.".category_id IN(".implode(',', $descendant_ids).")"; + $subquery_categories = array_merge($subquery_categories, $descendant_ids); } - } else { - $this->where[] = $alias.".category_id = ".(int)$id; } + + $shop_category_products_builder->where('category_id IN (i:ids)', array('ids' => $subquery_categories)); + if ((empty($this->info['sort_products']) && !waRequest::get('sort')) || waRequest::get('sort') == 'sort') { + $shop_category_products_builder->select('DISTINCT product_id, sort')->order('sort DESC'); + $subquery = $shop_category_products_builder->getSQL(); + + $alias = $this->addJoinSubquery($scp, $subquery, ':table.product_id = p.id', true); + $this->order_by = $alias.'.sort ASC'; + } else { + // the sort field is in shop_products table, so we dont need to use JOIN + DISTINCT + // this improvement gives us x20 faster query + $shop_category_products_builder->select('product_id'); + $subquery = $shop_category_products_builder->getSQL(); + + $this->idIn($subquery); } + } else { $hash = $this->hash; $this->setHash('/search/'.$this->info['conditions']); @@ -545,6 +563,18 @@ protected function setPrepare($id, $auto_title = true) } } + /** + * Specify subquery for id IN + * + * In some cases this can speed up query, to avoid JOIN + DISTINCT queries + * + * @param string $subquery not correlated subquery + */ + protected function idIn($subquery) + { + $this->where[] = "p.id IN ({$subquery})"; + } + protected function idPrepare($ids_str) { $this->info = array( @@ -1226,13 +1256,22 @@ public function getSQL() if ($this->joins) { foreach ($this->joins as $join) { - $alias = isset($join['alias']) ? $join['alias'] : ''; - if (isset($join['on'])) { - $on = $join['on']; + if (isset($join['table'])) { + $alias = isset($join['alias']) ? $join['alias'] : ''; + if (isset($join['on'])) { + $on = $join['on']; + } else { + $on = "p.id = ".($alias ? $alias : $join['table']).".product_id"; + } + $sql .= "\n\t".(isset($join['type']) ? $join['type'].' ' : '')."JOIN ".$join['table']." ".$alias."\n\t\tON ".$on; + } elseif (isset($join['subquery'])) { + // $join['with'] === self::JOIN_SUBQUERY + $sql .= "\n\t".(isset($join['type']) ? $join['type'].' ' : ''); + $sql .= "JOIN (" . $join['subquery'] . ") " . $join['alias'] . "\n\t\tON ". $join['on']; } else { - $on = "p.id = ".($alias ? $alias : $join['table']).".product_id"; + // skip join } - $sql .= "\n\t".(isset($join['type']) ? $join['type'].' ' : '')."JOIN ".$join['table']." ".$alias."\n\t\tON ".$on; + } } @@ -1263,6 +1302,7 @@ public function count() } else { $sql = "SELECT COUNT(".($this->joins ? 'DISTINCT ' : '')."p.id) ".$sql; } + $count = (int)$this->getModel()->query($sql)->fetchField(); if ($this->hash[0] == 'category' && !empty($this->info['id']) && $this->info['type'] == shopCategoryModel::TYPE_DYNAMIC) { @@ -1318,8 +1358,22 @@ public function getProducts($fields = "*", $offset = 0, $limit = null, $escape = $limit = $this->count - $offset; } } - - $distinct = $this->joins && !$this->group_by ? 'DISTINCT ' : ''; + + $distinct = ''; + if ($this->joins && !$this->group_by) { + foreach ($this->joins as $join) { + if (isset($join['subquery']) && isset($join['distinct']) && $join['distinct'] === true) { + // If we joins subquery (not table) and result set already unique + // we dont need to make DISTINCT again + // + // see self::addJoinSubquery() method for more details + continue; + } else { + $distinct = 'DISTINCT '; + break; + } + } + } $sql = "SELECT " . $distinct . $this->getFields($fields) . "\n"; $sql .= $from_and_where; @@ -2316,6 +2370,62 @@ public function addWhere($condition) return $this; } + + /** + * @param waModel $table model of table to join + * @param string $subquery query builder to join + * @param string|null + * @return self + */ + public function addJoinSubquery(waModel $model, $subquery, $on, $distinct = false, $type = null) + { + $table = $model->getTableName(); + + $alias = $this->generateAliasForTable($table); + + if (!isset($this->join_index[$alias])) { + $this->join_index[$alias] = 1; + } else { + $this->join_index[$alias]++; + } + + $alias .= $this->join_index[$alias]; + + $join = array( + // If join subquery has DISTINCT, but join type is LEFT/RIGHT JOIN + // we still need to do distinct in the main query + 'distinct' => $distinct && $type === null || $type === "INNER", + 'alias' => $alias, + 'type' => $type, + 'on' => str_replace(':table', $alias, $on), + 'subquery' => $subquery, + ); + + $this->joins[] = $join; + + return $alias; + } + + + protected function generateAliasForTable($table) + { + $alias = ''; + $t = explode('_', $table); + + foreach ($t as $tp) { + if ($tp == 'shop') { + continue; + } + $alias .= substr($tp, 0, 1); + } + + if (!$alias) { + $alias = $table; + } + + return $alias; + } + /** * Adds a simple JOIN clause to product selection query. * @@ -2337,18 +2447,8 @@ public function addJoin($table, $on = null, $where = null) } $table = $table['table']; } - $t = explode('_', $table); - $alias = ''; - foreach ($t as $tp) { - if ($tp == 'shop') { - continue; - } - $alias .= substr($tp, 0, 1); - } - - if (!$alias) { - $alias = $table; - } + + $alias = $this->generateAliasForTable($table); if (!isset($this->join_index[$alias])) { $this->join_index[$alias] = 1; From 27232834cd6eb2af743acbd54ea62ef0b19219dd Mon Sep 17 00:00:00 2001 From: Vadym Korovin Date: Thu, 28 Jun 2018 23:39:40 +0300 Subject: [PATCH 2/3] Add DISTINCT in queries with no 'sort' param --- lib/classes/shopProductsCollection.class.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/classes/shopProductsCollection.class.php b/lib/classes/shopProductsCollection.class.php index 3d926f4f6..3b56819d8 100644 --- a/lib/classes/shopProductsCollection.class.php +++ b/lib/classes/shopProductsCollection.class.php @@ -490,9 +490,9 @@ protected function categoryPrepare($id, $auto_title = true) $this->order_by = $alias.'.sort ASC'; } else { - // the sort field is in shop_products table, so we dont need to use JOIN + DISTINCT + // the sort field is in shop_products table, so we dont need to use JOIN // this improvement gives us x20 faster query - $shop_category_products_builder->select('product_id'); + $shop_category_products_builder->select('DISTINCT product_id'); $subquery = $shop_category_products_builder->getSQL(); $this->idIn($subquery); From cbc5455c92182813a943c88bf241f9bfd3303c45 Mon Sep 17 00:00:00 2001 From: Vadym Korovin Date: Sat, 30 Jun 2018 12:31:56 +0300 Subject: [PATCH 3/3] Fix $distinct param in addJoinSubquery, fix PHPDoc --- lib/classes/shopProductsCollection.class.php | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/classes/shopProductsCollection.class.php b/lib/classes/shopProductsCollection.class.php index 3b56819d8..930089835 100644 --- a/lib/classes/shopProductsCollection.class.php +++ b/lib/classes/shopProductsCollection.class.php @@ -2372,9 +2372,12 @@ public function addWhere($condition) /** - * @param waModel $table model of table to join - * @param string $subquery query builder to join - * @param string|null + * @param waModel $table waModel instance that will return table name + * @param string $subquery Builded subquery + * @param string $on ON statement used to join two tables, where :table will be replaced with alias + * @param bool $distinct If subquery has DISTINCT statement, we will not use DISTINCT in main query. + * It is important to set this param accordingly to gain performance. + * @param $type string JOIN type, e.g. INNER, LEFT, RIGHT, CROSS. * @return self */ public function addJoinSubquery(waModel $model, $subquery, $on, $distinct = false, $type = null) @@ -2392,9 +2395,7 @@ public function addJoinSubquery(waModel $model, $subquery, $on, $distinct = fals $alias .= $this->join_index[$alias]; $join = array( - // If join subquery has DISTINCT, but join type is LEFT/RIGHT JOIN - // we still need to do distinct in the main query - 'distinct' => $distinct && $type === null || $type === "INNER", + 'distinct' => $distinct, 'alias' => $alias, 'type' => $type, 'on' => str_replace(':table', $alias, $on),