问题背景
数据集成同步不同分区的数据报错:Exception when job run com.alibaba.datax.common.exception.DataXException: Code:DATAX_R_ODPS_018:分区不存在, Solution:[请修改配置的分区值]
排查过程
1、准备分区表
CREATETABLE IF NOT EXISTS sale_detail( shop_name STRING, customer_id STRING, total_price DOUBLE, customer_name STRING, education BIGINT, customer_name1 STRING COMMENT '客户', education1 BIGINT COMMENT '教育') PARTITIONED BY(sale_date STRING,region STRING) STORED AS ALIORC;
2、准备数据
insertinto sale_detail partition (sale_date='2014', region='china')values('s1','c1',99,'c1',99,'c1',99),('s2','c2',100.2,'c1',99,'c1',99),('s3','c3',100.3,'c1',99,'c1',99);
3、脚本模式创建同步任务
{ "type": "job", "version": "2.0", "steps": [ { "stepType": "odps", "parameter": { "partition": "sale_date=*/region=china", "indexes": [], "datasource": "odps_first", "envType": 1, "isSupportThreeModel": false, "column": [ "shop_name", "customer_id", "total_price", "customer_name", "education", "customer_name1", "education1" ], "tableComment": "", "table": "sale_detail" }, "name": "Reader", "category": "reader" }, { "stepType": "odps", "parameter": { "partition": "sale_date=${bizdate},region=china", "truncate": true, "indexes": [], "datasource": "public_odps", "envType": 1, "isSupportThreeModel": false, "column": [ "shop_name", "customer_id", "total_price", "customer_name", "education", "customer_name1", "education1" ], "emptyAsNull": false, "tableComment": "", "table": "sale_detail" }, "name": "Writer", "category": "writer" }, { "name": "Processor", "stepType": null, "category": "processor", "copies": 1, "parameter": { "nodes": [], "edges": [], "groups": [], "version": "2.0" } } ], "setting": { "executeMode": null, "errorLimit": { "record": "" }, "speed": { "concurrent": 2, "throttle": false } }, "order": { "hops": [ { "from": "Reader", "to": "Writer" } ] } }
4、验证数据5、复现问题
问题原因
配置的分区不存在
解决方案
根据实际有的分区配置,以下两种方式均可:
- "partition": "sale_date=${bizdate},region=china"
- "partition": "sale_date=${bizdate}/region=china"
适用范围
大数据开发治理平台DataWorks