常见sql深入优化
Order by 与 Group by 优化
Case 1:
explain select * from employees where name = 'SAN' and position = 'dev' order by age desc;
分析:利用最左前缀法则:中间字段不能断,因此查询用到了name索引,从key_len=74也能看出,age索引列用在排序过程中,因为Extra字段里没有using filesort
Case 2:
explain select * from employees where name = 'SAN' order by position;
分析: 从explain的执行结果来看:key_len=74,查询使用了name索引,由于用了position进行排序,跳过了age,出现了Using filesort。
Case 3:
explain select * from employees where name = 'SAN' order by age, position;
分析: 查找只用到索引name,age和position用于排序,无Using filesort。
Case 4:
explain select * from employees where name = 'SAN' order by position, age;
分析: 和Case 3中explain的执行结果一样,但是出现了Using filesort,因为索引的创建顺序为name,age,position,但是排序的时候age和position颠倒位置了。
Case 5:
explain select * from employees where name = 'SAN' and age = 30 order by position, age;
分析: 与Case 4对比,在Extra中并未出现Using filesort,因为age为常量,在排序中被优化,所以索引未颠倒,不会出现Using filesort。
Case 6:
explain select * from employees where name = 'SAN' order by age desc ,position asc;
分析: 虽然排序的字段列与索引顺序一样,且order by默认升序,这里position desc变成了降序,导致与索引的排序方式不同,从而产生Using filesort。Mysql8以上版本有降序索引可以支持该种查询方式。
Case 7:
explain select * from employees where name in ('SAN', 'SI', 'WU') order by age ,position;
分析: 对于排序来说,多个相等条件也是范围查询
Case 8:
explain select * from employees where name > 'a' order by name;
可以用覆盖索引优化
explain select name, age, position from employees where name > 'a' order by name;
优化总结:
1、MySQL支持两种方式的排序filesort和index,Using index是指MySQL扫描索引本身完成排序。index效率高,filesort效率低。
2、order by满足两种情况会使用Using index。
- order by语句使用索引最左前列。
- 使用where子句与order by子句条件列组合满足索引最左前列。
3、尽量在索引列上完成排序,遵循索引建立(索引创建的顺序)时的最左前缀法则。
4、如果order by的条件不在索引列上,就会产生Using filesort。
5、能用覆盖索引尽量用覆盖索引
6、group by与order by很类似,其实质是先排序后分组,遵照索引创建顺序的最左前缀法则。对于group by的优化如果不需要排序的可以加上order by null禁止排序。注意,where高于having,能写在where中的限定条件就不要去having限定了。
Using filesort 文件排序原理详解
filesort 文件排序方式
- 单路排序:是一次性取出满足条件行的所有字段,然后在sort buffer中进行排序;用trace工具可以看到sort_mode信息里显示< sort_key, additional_fields >或者< sort_key, packed_additional_fields >
- 双路排序(又叫回表排序模式):是首先根据相应的条件取出相应的排序字段和可以直接定位行数据的行 ID,然后在 sort buffer 中进行排序,排序完后需要再次取回其它需要的字段;用trace工具可以看到sort_mode信息里显示< sort_key, rowid >
- MySQL 通过比较系统变量 **max_length_for_sort_data(默认1024字节) **的大小和需要查询的字段总大小来判断使用哪种排序模式。
- 如果 字段的总长度小于max_length_for_sort_data ,那么使用 单路排序模式;
- 如果 字段的总长度大于max_length_for_sort_data ,那么使用 双路排序模式。
示例验证下各种排序方式:
查看下这条sql对应trace结果如下(只展示排序部分):
set session optimizer_trace="enabled=on",end_markers_in_json=on; --开启trace select * from employees where name = 'SAN' order by position; select * from information_schema.OPTIMIZER_TRACE; trace排序部分结果: { "steps": [ { "join_preparation": { "select#": 1, "steps": [ { "expanded_query": "/* select#1 */ select `employees`.`id` AS `id`,`employees`.`name` AS `name`,`employees`.`age` AS `age`,`employees`.`position` AS `position`,`employees`.`hire_time` AS `hire_time` from `employees` where (`employees`.`name` > 'a') order by `employees`.`position`" } ] /* steps */ } /* join_preparation */ }, { "join_optimization": { "select#": 1, "steps": [ { "condition_processing": { "condition": "WHERE", "original_condition": "(`employees`.`name` > 'a')", "steps": [ { "transformation": "equality_propagation", "resulting_condition": "(`employees`.`name` > 'a')" }, { "transformation": "constant_propagation", "resulting_condition": "(`employees`.`name` > 'a')" }, { "transformation": "trivial_condition_removal", "resulting_condition": "(`employees`.`name` > 'a')" } ] /* steps */ } /* condition_processing */ }, { "substitute_generated_columns": { } /* substitute_generated_columns */ }, { "table_dependencies": [ { "table": "`employees`", "row_may_be_null": false, "map_bit": 0, "depends_on_map_bits": [ ] /* depends_on_map_bits */ } ] /* table_dependencies */ }, { "ref_optimizer_key_uses": [ ] /* ref_optimizer_key_uses */ }, { "rows_estimation": [ { "table": "`employees`", "range_analysis": { "table_scan": { "rows": 120085, "cost": 24372 } /* table_scan */, "potential_range_indexes": [ { "index": "PRIMARY", "usable": false, "cause": "not_applicable" }, { "index": "idx_name_age_position", "usable": true, "key_parts": [ "name", "age", "position", "id" ] /* key_parts */ } ] /* potential_range_indexes */, "setup_range_conditions": [ ] /* setup_range_conditions */, "group_index_range": { "chosen": false, "cause": "not_group_by_or_distinct" } /* group_index_range */, "analyzing_range_alternatives": { "range_scan_alternatives": [ { "index": "idx_name_age_position", "ranges": [ "a < name" ] /* ranges */, "index_dives_for_eq_ranges": true, "rowid_ordered": false, "using_mrr": false, "index_only": false, "rows": 60042, "cost": 72051, "chosen": false, "cause": "cost" } ] /* range_scan_alternatives */, "analyzing_roworder_intersect": { "usable": false, "cause": "too_few_roworder_scans" } /* analyzing_roworder_intersect */ } /* analyzing_range_alternatives */ } /* range_analysis */ } ] /* rows_estimation */ }, { "considered_execution_plans": [ { "plan_prefix": [ ] /* plan_prefix */, "table": "`employees`", "best_access_path": { "considered_access_paths": [ { "rows_to_scan": 120085, "access_type": "scan", "resulting_rows": 120085, "cost": 24370, "chosen": true, "use_tmp_table": true } ] /* considered_access_paths */ } /* best_access_path */, "condition_filtering_pct": 100, "rows_for_plan": 120085, "cost_for_plan": 24370, "sort_cost": 120085, "new_cost_for_plan": 144455, "chosen": true } ] /* considered_execution_plans */ }, { "attaching_conditions_to_tables": { "original_condition": "(`employees`.`name` > 'a')", "attached_conditions_computation": [ ] /* attached_conditions_computation */, "attached_conditions_summary": [ { "table": "`employees`", "attached": "(`employees`.`name` > 'a')" } ] /* attached_conditions_summary */ } /* attaching_conditions_to_tables */ }, { "clause_processing": { "clause": "ORDER BY", "original_clause": "`employees`.`position`", "items": [ { "item": "`employees`.`position`" } ] /* items */, "resulting_clause_is_simple": true, "resulting_clause": "`employees`.`position`" } /* clause_processing */ }, { "reconsidering_access_paths_for_index_ordering": { "clause": "ORDER BY", "index_order_summary": { "table": "`employees`", "index_provides_order": false, "order_direction": "undefined", "index": "unknown", "plan_changed": false } /* index_order_summary */ } /* reconsidering_access_paths_for_index_ordering */ }, { "refine_plan": [ { "table": "`employees`" } ] /* refine_plan */ } ] /* steps */ } /* join_optimization */ }, { "join_execution": { --SQL 执行阶段 "select#": 1, "steps": [ { "filesort_information": [ { "direction": "asc", "table": "`employees`", "field": "position" } ] /* filesort_information */, "filesort_priority_queue_optimization": { "usable": false, "cause": "not applicable (no LIMIT)" } /* filesort_priority_queue_optimization */, "filesort_execution": [ ] /* filesort_execution */, "filesort_summary": { "rows": 120003, --文件排序信息 "examined_rows": 120003, --预计扫描行数 "number_of_tmp_files": 34, --参与排序的行 "sort_buffer_size": 262056, --使用临时文件的个数,如果这个值为 9 代表全部使用的 sort_buffer 内存排序,否则使用的磁盘文件排序 "sort_mode": "<sort_key, packed_additional_fields>" --排序方式,这里采用的是单路排序 } /* filesort_summary */ } ] /* steps */ } /* join_execution */ } ] /* steps */ } set max_length_for_sort_data = 2; --employees表所有字段长度总和肯定大于2字节 select * from employees where name = 'SAN' order by position; select * from information_schema.OPTIMIZER_TRACE; trace排序部分结果: { "steps": [ { "join_preparation": { "select#": 1, "steps": [ { "expanded_query": "/* select#1 */ select `employees`.`id` AS `id`,`employees`.`name` AS `name`,`employees`.`age` AS `age`,`employees`.`position` AS `position`,`employees`.`hire_time` AS `hire_time` from `employees` where (`employees`.`name` = 'SAN') order by `employees`.`position`" } ] /* steps */ } /* join_preparation */ }, { "join_optimization": { "select#": 1, "steps": [ { "condition_processing": { "condition": "WHERE", "original_condition": "(`employees`.`name` = 'SAN')", "steps": [ { "transformation": "equality_propagation", "resulting_condition": "(`employees`.`name` = 'SAN')" }, { "transformation": "constant_propagation", "resulting_condition": "(`employees`.`name` = 'SAN')" }, { "transformation": "trivial_condition_removal", "resulting_condition": "(`employees`.`name` = 'SAN')" } ] /* steps */ } /* condition_processing */ }, { "substitute_generated_columns": { } /* substitute_generated_columns */ }, { "table_dependencies": [ { "table": "`employees`", "row_may_be_null": false, "map_bit": 0, "depends_on_map_bits": [ ] /* depends_on_map_bits */ } ] /* table_dependencies */ }, { "ref_optimizer_key_uses": [ { "table": "`employees`", "field": "name", "equals": "'SAN'", "null_rejecting": false } ] /* ref_optimizer_key_uses */ }, { "rows_estimation": [ { "table": "`employees`", "range_analysis": { "table_scan": { "rows": 120085, "cost": 24372 } /* table_scan */, "potential_range_indexes": [ { "index": "PRIMARY", "usable": false, "cause": "not_applicable" }, { "index": "idx_name_age_position", "usable": true, "key_parts": [ "name", "age", "position", "id" ] /* key_parts */ } ] /* potential_range_indexes */, "setup_range_conditions": [ ] /* setup_range_conditions */, "group_index_range": { "chosen": false, "cause": "not_group_by_or_distinct" } /* group_index_range */, "analyzing_range_alternatives": { "range_scan_alternatives": [ { "index": "idx_name_age_position", "ranges": [ "SAN <= name <= SAN" ] /* ranges */, "index_dives_for_eq_ranges": true, "rowid_ordered": false, "using_mrr": false, "index_only": false, "rows": 1, "cost": 2.21, "chosen": true } ] /* range_scan_alternatives */, "analyzing_roworder_intersect": { "usable": false, "cause": "too_few_roworder_scans" } /* analyzing_roworder_intersect */ } /* analyzing_range_alternatives */, "chosen_range_access_summary": { "range_access_plan": { "type": "range_scan", "index": "idx_name_age_position", "rows": 1, "ranges": [ "SAN <= name <= SAN" ] /* ranges */ } /* range_access_plan */, "rows_for_plan": 1, "cost_for_plan": 2.21, "chosen": true } /* chosen_range_access_summary */ } /* range_analysis */ } ] /* rows_estimation */ }, { "considered_execution_plans": [ { "plan_prefix": [ ] /* plan_prefix */, "table": "`employees`", "best_access_path": { "considered_access_paths": [ { "access_type": "ref", "index": "idx_name_age_position", "rows": 1, "cost": 1.2, "chosen": true }, { "access_type": "range", "range_details": { "used_index": "idx_name_age_position" } /* range_details */, "chosen": false, "cause": "heuristic_index_cheaper" } ] /* considered_access_paths */ } /* best_access_path */, "condition_filtering_pct": 100, "rows_for_plan": 1, "cost_for_plan": 1.2, "chosen": true } ] /* considered_execution_plans */ }, { "attaching_conditions_to_tables": { "original_condition": "(`employees`.`name` = 'SAN')", "attached_conditions_computation": [ ] /* attached_conditions_computation */, "attached_conditions_summary": [ { "table": "`employees`", "attached": null } ] /* attached_conditions_summary */ } /* attaching_conditions_to_tables */ }, { "clause_processing": { "clause": "ORDER BY", "original_clause": "`employees`.`position`", "items": [ { "item": "`employees`.`position`" } ] /* items */, "resulting_clause_is_simple": true, "resulting_clause": "`employees`.`position`" } /* clause_processing */ }, { "added_back_ref_condition": "((`employees`.`name` <=> 'SAN'))" }, { "reconsidering_access_paths_for_index_ordering": { "clause": "ORDER BY", "index_order_summary": { "table": "`employees`", "index_provides_order": false, "order_direction": "undefined", "index": "idx_name_age_position", "plan_changed": false } /* index_order_summary */ } /* reconsidering_access_paths_for_index_ordering */ }, { "refine_plan": [ { "table": "`employees`", "pushed_index_condition": "(`employees`.`name` <=> 'SAN')", "table_condition_attached": null } ] /* refine_plan */ } ] /* steps */ } /* join_optimization */ }, { "join_execution": { "select#": 1, "steps": [ { "filesort_information": [ { "direction": "asc", "table": "`employees`", "field": "position" } ] /* filesort_information */, "filesort_priority_queue_optimization": { "usable": false, "cause": "not applicable (no LIMIT)" } /* filesort_priority_queue_optimization */, "filesort_execution": [ ] /* filesort_execution */, "filesort_summary": { "rows": 0, "examined_rows": 0, "number_of_tmp_files": 0, "sort_buffer_size": 262136, "sort_mode": "<sort_key, rowid>" --排序方式,双路排序 } /* filesort_summary */ } ] /* steps */ } /* join_execution */ } ] /* steps */ } set session optimizer_trace="enabled=off"; --关闭trace