一、product-es准备
P128
ES在内存中，所以在检索中优于mysql。ES也支持集群，数据分片存储。
需求：
- 上架的商品才可以在网站展示。
- 上架的商品需要可以被检索。
分析sku在es中如何存储
商品mapping
分析：商品上架在es中是存sku还是spu？
1）、检索的时候输入名字，是需要按照sku的title进行全文检索的
2）、检素使用商品规格，规格是spu的公共属性，每个spu是一样的
3）、按照分类id进去的都是直接列出spu的，还可以切换。
4〕、我们如果将sku的全量信息保存到es中（包括spu属性〕就太多字段了
方案1：
{ skuId:1 spuId:11 skyTitile:华为xx price:999 saleCount:99 attr:[ {尺寸:5}, {CPU:高通945}, {分辨率:全高清} ] 缺点：如果每个sku都存储规格参数(如尺寸)，会有冗余存储，因为每个sku对应的spu的规格参数都一样
方案2：
sku索引 { spuId:1 skuId:11 } attr索引 { skuId:11 attr:[ {尺寸:5}, {CPU:高通945}, {分辨率:全高清} ] } 先找到4000个符合要求的spu，再根据4000个spu查询对应的属性，封装了4000个id，long 8B*4000=32000B=32KB 1K个人检索，就是32MB 结论：如果将规格参数单独建立索引，会出现检索时出现大量数据传输的问题，会引起网络网络
🚩 因此选用方案1，以空间换时间
建立product索引
最终选用的数据模型：
- { “type”: “keyword” }, # 保持数据精度问题，可以检索，但不分词
- “analyzer”: “ik_smart” # 中文分词器
- “index”: false, # 不可被检索，不生成index
- “doc_values”: false # 默认为true，不可被聚合，es就不会维护一些聚合的信息
PUT product { "mappings":{ "properties": { "skuId":{ "type": "long" }, "spuId":{ "type": "keyword" }, # 不可分词 "skuTitle": { "type": "text", "analyzer": "ik_smart" # 中文分词器 }, "skuPrice": { "type": "keyword" }, # 保证精度问题 "skuImg" : { "type": "keyword" }, # 视频中有false "saleCount":{ "type":"long" }, "hasStock": { "type": "boolean" }, "hotScore": { "type": "long" }, "brandId": { "type": "long" }, "catalogId": { "type": "long" }, "brandName": {"type": "keyword"}, # 视频中有false "brandImg":{ "type": "keyword", "index": false, # 不可被检索，不生成index，只用做页面使用 "doc_values": false # 不可被聚合，默认为true }, "catalogName": {"type": "keyword" }, # 视频里有false "attrs": { "type": "nested", "properties": { "attrId": {"type": "long" }, "attrName": { "type": "keyword", "index": false, "doc_values": false }, "attrValue": {"type": "keyword" } } } } } }
如果检索不到商品，自己用postman测试一下，可能有的字段需要更改，你也可以把没必要的"keyword"去掉
冗余存储的字段：不用来检索，也不用来分析，节省空间
库存是bool。
检索品牌id，但是不检索品牌名字、图片
用skuTitle检索
二、nested嵌入式对象
属性是"type": “nested”,因为是内部的属性进行检索
数组类型的对象会被扁平化处理（对象的每个属性会分别存储到一起）
user.name=["aaa","bbb"] user.addr=["ccc","ddd"] 这种存储方式，可能会发生如下错误： 错误检索到{aaa,ddd}，这个组合是不存在的
数组的扁平化处理会使检索能检索到本身不存在的，为了解决这个问题，就采用了嵌入式属性，数组里是对象时用嵌入式属性（不是对象无需用嵌入式属性）
三、商品上架（ES保存）
@Override // SpuInfoServiceImpl public void up(Long spuId) { // 1 组装数据 查出当前spuId对应的所有sku信息 List<SkuInfoEntity> skus = skuInfoService.getSkusBySpuId(spuId); // 查询这些sku是否有库存 List<Long> skuids = skus.stream().map(sku -> sku.getSkuId()).collect(Collectors.toList()); // 2 封装每个sku的信息 // 3.查询当前sku所有可以被用来检索的规格属性 List<ProductAttrValueEntity> baseAttrs = attrValueService.baseAttrListForSpu(spuId); // 得到基本属性id List<Long> attrIds = baseAttrs.stream().map(attr -> attr.getAttrId()).collect(Collectors.toList()); // 过滤出可被检索的基本属性id，即search_type = 1 [数据库中目前 4、5、6、11不可检索] Set<Long> ids = new HashSet<>(attrService.selectSearchAttrIds(attrIds)); // 可被检索的属性封装到SkuEsModel.Attrs中 List<SkuEsModel.Attrs> attrs = baseAttrs.stream() .filter(item -> ids.contains(item.getAttrId())) .map(item -> { SkuEsModel.Attrs attr = new SkuEsModel.Attrs(); BeanUtils.copyProperties(item, attr); return attr; }).collect(Collectors.toList()); // 每件skuId是否有库存 Map<Long, Boolean> stockMap = null; try { // 3.1 远程调用库存系统 查询该sku是否有库存 R hasStock = wareFeignService.getSkuHasStock(skuids); // 构造器受保护 所以写成内部类对象 stockMap = hasStock.getData(new TypeReference<List<SkuHasStockVo>>() {}) .stream() .collect(Collectors.toMap(SkuHasStockVo::getSkuId, item -> item.getHasStock())); log.warn("服务调用成功" + hasStock); } catch (Exception e) { log.error("库存服务调用失败: 原因{}", e); } Map<Long, Boolean> finalStockMap = stockMap;//防止lambda中改变 // 开始封装es List<SkuEsModel> skuEsModels = skus.stream().map(sku -> { SkuEsModel esModel = new SkuEsModel(); BeanUtils.copyProperties(sku, esModel); esModel.setSkuPrice(sku.getPrice()); esModel.setSkuImg(sku.getSkuDefaultImg()); // 4 设置库存，只查是否有库存，不查有多少 if (finalStockMap == null) { esModel.setHasStock(true); } else { esModel.setHasStock(finalStockMap.get(sku.getSkuId())); } // TODO 1.热度评分 刚上架是0 esModel.setHotScore(0L); // 设置品牌信息 BrandEntity brandEntity = brandService.getById(esModel.getBrandId()); esModel.setBrandName(brandEntity.getName()); esModel.setBrandImg(brandEntity.getLogo()); // 查询分类信息 CategoryEntity categoryEntity = categoryService.getById(esModel.getCatalogId()); esModel.setCatalogName(categoryEntity.getName()); // 保存商品的属性， 查询当前sku的所有可以被用来检索的规格属性，同一spu都一样，在外面查一遍即可 esModel.setAttrs(attrs); return esModel; }).collect(Collectors.toList()); // 5.发给ES进行保存 gulimall-search R r = searchFeignService.productStatusUp(skuEsModels); if (r.getCode() == 0) { // 远程调用成功 baseMapper.updateSpuStatus(spuId, ProductConstant.StatusEnum.SPU_UP.getCode()); } else { // 远程调用失败 TODO 接口幂等性 重试机制 /** * Feign 的调用流程 Feign有自动重试机制 * 1. 发送请求执行 * 2. */ } }
@Slf4j @Service public class ProductSaveServiceImpl implements ProductSaveService { @Resource private RestHighLevelClient client; /** * 将数据保存到ES * 用bulk代替index，进行批量保存 * BulkRequest bulkRequest, RequestOptions options */ @Override // ProductSaveServiceImpl public boolean productStatusUp(List<SkuEsModel> skuEsModels) throws IOException { // 1. 批量保存 BulkRequest bulkRequest = new BulkRequest(); // 2.构造保存请求 for (SkuEsModel esModel : skuEsModels) { // 设置es索引 gulimall_product IndexRequest indexRequest = new IndexRequest(EsConstant.PRODUCT_INDEX); // 设置索引id indexRequest.id(esModel.getSkuId().toString()); // json格式 String jsonString = JSON.toJSONString(esModel); indexRequest.source(jsonString, XContentType.JSON); // 添加到文档 bulkRequest.add(indexRequest); } // bulk批量保存 BulkResponse bulk = client.bulk(bulkRequest, GuliESConfig.COMMON_OPTIONS); // TODO 是否拥有错误 boolean hasFailures = bulk.hasFailures(); if(hasFailures){ List<String> collect = Arrays.stream(bulk.getItems()).map(item -> item.getId()).collect(Collectors.toList()); log.error("商品上架错误：{}",collect); } return hasFailures; } }
PUT product { "mappings": { "properties": { "skuId":{ "type": "long" }, "spuId":{ "type": "keyword" }, "skuTitle":{ "type": "text", "analyzer": "ik_smart" }, "skuPrice":{ "type": "keyword" }, "skuImg":{ "type": "keyword", "index": false, "doc_values": false }, "saleCount":{ "type": "long" }, "hasStock":{ "type": "boolean" }, "hotScore":{ "type": "long" }, "brandId":{ "type": "long" }, "catalogId":{ "type": "long" }, "brandName":{ "type":"keyword", "index": false, "doc_values": false }, "brandImg":{ "type": "keyword", "index": false, "doc_values": false }, "catalogName":{ "type": "keyword", "index": false, "doc_values": false }, "attrs":{ "type": "nested", "properties": { "attrId":{ "type":"long" }, "attrName":{ "type":"keyword", "index":false, "doc_values": false }, "attrValue":{ "type":"keyword" } } } } } }
四、检索服务
package com.atguigu.gulimall.search.service.impl; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.TypeReference; import com.atguigu.common.to.es.SkuEsModel; import com.atguigu.common.utils.R; import com.atguigu.gulimall.search.config.GuliESConfig; import com.atguigu.gulimall.search.constant.EsConstant; import com.atguigu.gulimall.search.feign.ProductFeignService; import com.atguigu.gulimall.search.service.SearchService; import com.atguigu.gulimall.search.vo.AttrResponseVo; import com.atguigu.gulimall.search.vo.BrandVo; import com.atguigu.gulimall.search.vo.SearchParam; import com.atguigu.gulimall.search.vo.SearchResult; import lombok.extern.slf4j.Slf4j; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.NestedQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested; import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms; import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; import org.elasticsearch.search.sort.SortOrder; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import javax.annotation.Resource; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; /** * <p>Title: MallServiceImpl</p> * Description： * date：2020/6/12 23:06 */ @Slf4j @Service public class SearchServiceImpl implements SearchService { @Resource private RestHighLevelClient restHighLevelClient; @Resource private ProductFeignService productFeignService; @Override public SearchResult search(SearchParam Param) { SearchResult result = null; // 1.准备检索请求 SearchRequest searchRequest = buildSearchRequest(Param); try { // 2.执行检索请求 SearchResponse response = restHighLevelClient.search(searchRequest, GuliESConfig.COMMON_OPTIONS); // 3.分析响应数据 result = buildSearchResult(response, Param); } catch (IOException e) { e.printStackTrace(); } return result; } /** * 准备检索请求 [构建查询语句] */ private SearchRequest buildSearchRequest(SearchParam Param) { // 帮我们构建DSL语句的 SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); // 1. 模糊匹配 过滤(按照属性、分类、品牌、价格区间、库存) 先构建一个布尔Query // 1.1 must BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); if(!StringUtils.isEmpty(Param.getKeyword())){ boolQuery.must(QueryBuilders.matchQuery("skuTitle",Param.getKeyword())); } // 1.2 bool - filter Catalog3Id if(StringUtils.isEmpty(Param.getCatalog3Id() != null)){ boolQuery.filter(QueryBuilders.termQuery("catalogId", Param.getCatalog3Id())); } // 1.2 bool - brandId [集合] if(Param.getBrandId() != null && Param.getBrandId().size() > 0){ boolQuery.filter(QueryBuilders.termsQuery("brandId", Param.getBrandId())); } // 属性查询 if(Param.getAttrs() != null && Param.getAttrs().size() > 0){ for (String attrStr : Param.getAttrs()) { BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); String[] s = attrStr.split("_"); // 检索的id 属性检索用的值 String attrId = s[0]; String[] attrValue = s[1].split(":"); boolQueryBuilder.must(QueryBuilders.termQuery("attrs.attrId", attrId)); boolQueryBuilder.must(QueryBuilders.termsQuery("attrs.attrValue", attrValue)); // 构建一个嵌入式Query 每一个必须都得生成嵌入的 nested 查询 NestedQueryBuilder attrsQuery = QueryBuilders.nestedQuery("attrs", boolQueryBuilder, ScoreMode.None); boolQuery.filter(attrsQuery); } } // 1.2 bool - filter [库存] if(Param.getHasStock() != null){ boolQuery.filter(QueryBuilders.termQuery("hasStock",Param.getHasStock() == 1)); } // 1.2 bool - filter [价格区间] if(!StringUtils.isEmpty(Param.getSkuPrice())){ RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery("skuPrice"); String[] s = Param.getSkuPrice().split("_"); if(s.length == 2){ // 有三个值 就是区间 rangeQuery.gte(s[0]).lte(s[1]); }else if(s.length == 1){ // 单值情况 if(Param.getSkuPrice().startsWith("_")){ rangeQuery.lte(s[0]); } if(Param.getSkuPrice().endsWith("_")){ rangeQuery.gte(s[0]); } } boolQuery.filter(rangeQuery); } // 把以前所有条件都拿来进行封装 sourceBuilder.query(boolQuery); // 1.排序 if(!StringUtils.isEmpty(Param.getSort())){ String sort = Param.getSort(); // sort=hotScore_asc/desc String[] s = sort.split("_"); SortOrder order = s[1].equalsIgnoreCase("asc") ? SortOrder.ASC : SortOrder.DESC; sourceBuilder.sort(s[0], order); } // 2.分页 pageSize ： 5 sourceBuilder.from((Param.getPageNum()-1) * EsConstant.PRODUCT_PASIZE); sourceBuilder.size(EsConstant.PRODUCT_PASIZE); // 3.高亮 if(!StringUtils.isEmpty(Param.getKeyword())){ HighlightBuilder builder = new HighlightBuilder(); builder.field("skuTitle"); builder.preTags("<b style='color:red'>"); builder.postTags("</b>"); sourceBuilder.highlighter(builder); } // 聚合分析 // TODO 1.品牌聚合 TermsAggregationBuilder brand_agg = AggregationBuilders.terms("brand_agg"); brand_agg.field("brandId").size(50); // 品牌聚合的子聚合 brand_agg.subAggregation(AggregationBuilders.terms("brand_name_agg").field("brandName").size(1)); brand_agg.subAggregation(AggregationBuilders.terms("brand_img_agg").field("brandImg").size(1)); // 将品牌聚合加入 sourceBuilder sourceBuilder.aggregation(brand_agg); // TODO 2.分类聚合 TermsAggregationBuilder catalog_agg = AggregationBuilders.terms("catalog_agg").field("catalogId").size(20); catalog_agg.subAggregation(AggregationBuilders.terms("catalog_name_agg").field("catalogName").size(1)); // 将分类聚合加入 sourceBuilder sourceBuilder.aggregation(catalog_agg); // TODO 3.属性聚合 attr_agg 构建嵌入式聚合 NestedAggregationBuilder attr_agg = AggregationBuilders.nested("attr_agg", "attrs"); // 3.1 聚合出当前所有的attrId TermsAggregationBuilder attrIdAgg = AggregationBuilders.terms("attr_id_agg").field("attrs.attrId"); // 3.1.1 聚合分析出当前attrId对应的attrName attrIdAgg.subAggregation(AggregationBuilders.terms("attr_name_agg").field("attrs.attrName").size(1)); // 3.1.2 聚合分析出当前attrId对应的所有可能的属性值attrValue 这里的属性值可能会有很多 所以写50 attrIdAgg.subAggregation(AggregationBuilders.terms("attr_value_agg").field("attrs.attrValue").size(50)); // 3.2 将这个子聚合加入嵌入式聚合 attr_agg.subAggregation(attrIdAgg); sourceBuilder.aggregation(attr_agg); log.info("\n构建语句：->\n" + sourceBuilder.toString()); SearchRequest searchRequest = new SearchRequest(new String[]{EsConstant.PRODUCT_INDEX}, sourceBuilder); return searchRequest; } /** * 构建结果数据 指定catalogId 、brandId、attrs.attrId、嵌入式查询、倒序、0-6000、每页显示两个、高亮skuTitle、聚合分析 */ private SearchResult buildSearchResult(SearchResponse response, SearchParam Param) { SearchResult result = new SearchResult(); // 1.返回的所有查询到的商品 SearchHits hits = response.getHits(); List<SkuEsModel> esModels = new ArrayList<>(); if(hits.getHits() != null && hits.getHits().length > 0){ for (SearchHit hit : hits.getHits()) { String sourceAsString = hit.getSourceAsString(); // ES中检索得到的对象 SkuEsModel esModel = JSON.parseObject(sourceAsString, SkuEsModel.class); if(!StringUtils.isEmpty(Param.getKeyword())){ // 1.1 获取标题的高亮属性 HighlightField skuTitle = hit.getHighlightFields().get("skuTitle"); String highlightFields = skuTitle.getFragments()[0].string(); // 1.2 设置文本高亮 esModel.setSkuTitle(highlightFields); } esModels.add(esModel); } } result.setProducts(esModels); // 2.当前所有商品涉及到的所有属性信息 ArrayList<SearchResult.AttrVo> attrVos = new ArrayList<>(); ParsedNested attr_agg = response.getAggregations().get("attr_agg"); ParsedLongTerms attr_id = attr_agg.getAggregations().get("attr_id_agg"); for (Terms.Bucket bucket : attr_id.getBuckets()) { SearchResult.AttrVo attrVo = new SearchResult.AttrVo(); // 2.1 得到属性的id attrVo.setAttrId(bucket.getKeyAsNumber().longValue()); // 2.2 得到属性的名字 String attr_name = ((ParsedStringTerms) bucket.getAggregations().get("attr_name_agg")).getBuckets().get(0).getKeyAsString(); attrVo.setAttrName(attr_name); // 2.3 得到属性的所有值 List<String> attr_value = ((ParsedStringTerms) bucket.getAggregations().get("attr_value_agg")).getBuckets().stream().map(item -> item.getKeyAsString()).collect(Collectors.toList()); attrVo.setAttrValue(attr_value); attrVos.add(attrVo); } result.setAttrs(attrVos); // 3.当前所有商品涉及到的所有品牌信息 ArrayList<SearchResult.BrandVo> brandVos = new ArrayList<>(); ParsedLongTerms brand_agg = response.getAggregations().get("brand_agg"); for (Terms.Bucket bucket : brand_agg.getBuckets()) { SearchResult.BrandVo brandVo = new SearchResult.BrandVo(); // 3.1 得到品牌的id long brnadId = bucket.getKeyAsNumber().longValue(); brandVo.setBrandId(brnadId); // 3.2 得到品牌的名 String brand_name = ((ParsedStringTerms) bucket.getAggregations().get("brand_name_agg")).getBuckets().get(0).getKeyAsString(); brandVo.setBrandName(brand_name); // 3.3 得到品牌的图片 String brand_img = ((ParsedStringTerms) (bucket.getAggregations().get("brand_img_agg"))).getBuckets().get(0).getKeyAsString(); brandVo.setBrandImg(brand_img); brandVos.add(brandVo); } result.setBrands(brandVos); // 4.当前商品所有涉及到的分类信息 ParsedLongTerms catalog_agg = response.getAggregations().get("catalog_agg"); List<SearchResult.CatalogVo> catalogVos = new ArrayList<>(); for (Terms.Bucket bucket : catalog_agg.getBuckets()) { // 设置分类id SearchResult.CatalogVo catalogVo = new SearchResult.CatalogVo(); catalogVo.setCatalogId(Long.parseLong(bucket.getKeyAsString())); // 得到分类名 ParsedStringTerms catalog_name_agg = bucket.getAggregations().get("catalog_name_agg"); String catalog_name = catalog_name_agg.getBuckets().get(0).getKeyAsString(); catalogVo.setCatalogName(catalog_name); catalogVos.add(catalogVo); } result.setCatalogs(catalogVos); // ================以上信息从聚合信息中获取 // 5.分页信息-页码 result.setPageNum(Param.getPageNum()); // 总记录数 long total = hits.getTotalHits().value; result.setTotal(total); // 总页码：计算得到 int totalPages = (int)(total / EsConstant.PRODUCT_PASIZE + 0.999999999999); result.setTotalPages(totalPages); // 设置导航页 ArrayList<Integer> pageNavs = new ArrayList<>(); for (int i = 1;i <= totalPages; i++){ pageNavs.add(i); } result.setPageNavs(pageNavs); // 6.构建面包屑导航功能 if(Param.getAttrs() != null){ List<SearchResult.NavVo> navVos = Param.getAttrs().stream().map(attr -> { SearchResult.NavVo navVo = new SearchResult.NavVo(); String[] s = attr.split("_"); navVo.setNavValue(s[1]); R r = productFeignService.getAttrsInfo(Long.parseLong(s[0])); // 将已选择的请求参数添加进去 前端页面进行排除 result.getAttrIds().add(Long.parseLong(s[0])); if(r.getCode() == 0){ AttrResponseVo data = r.getData(new TypeReference<AttrResponseVo>(){}); navVo.setName(data.getAttrName()); }else{ // 失败了就拿id作为名字 navVo.setName(s[0]); } // 拿到所有查询条件 替换查询条件 String replace = replaceQueryString(Param, attr, "attrs"); navVo.setLink("http://search.gulimall.com/list.html?" + replace); return navVo; }).collect(Collectors.toList()); result.setNavs(navVos); } // 品牌、分类 if(Param.getBrandId() != null && Param.getBrandId().size() > 0){ List<SearchResult.NavVo> navs = result.getNavs(); SearchResult.NavVo navVo = new SearchResult.NavVo(); navVo.setName("品牌"); // TODO 远程查询所有品牌 R r = productFeignService.brandInfo(Param.getBrandId()); if(r.getCode() == 0){ List<BrandVo> brand = r.getData("data", new TypeReference<List<BrandVo>>() {}); StringBuffer buffer = new StringBuffer(); // 替换所有品牌ID String replace = ""; for (BrandVo brandVo : brand) { buffer.append(brandVo.getBrandName() + ";"); replace = replaceQueryString(Param, brandVo.getBrandId() + "", "brandId"); } navVo.setNavValue(buffer.toString()); navVo.setLink("http://search.gulimall.com/list.html?" + replace); } navs.add(navVo); } return result; } /** * 替换字符 * key ：需要替换的key */ private String replaceQueryString(SearchParam Param, String value, String key) { String encode = null; try { encode = URLEncoder.encode(value,"UTF-8"); // 浏览器对空格的编码和java的不一样 encode = encode.replace("+","%20"); encode = encode.replace("%28", "(").replace("%29",")"); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } return Param.get_queryString().replace("&" + key + "=" + encode, ""); } }
