开发者社区> 问答> 正文

ODPSMR本地可以运行,上传资源使用ODPS却执行报错

odps:yangsw_test> jar -resources word_count.jar -classpath word_count.jar com.thomas.odps.WordCount word_count word_count_result
Exception in thread "main" java.lang.RuntimeException: java.lang.NoSuchMethodException: com.thomas.odps.WordCount.main([Ljava.lang.String;)
        at com.aliyun.odps.mapred.cli.Cli.main(Cli.java:47)
Caused by: java.lang.NoSuchMethodException: com.thomas.odps.WordCount.main([Ljava.lang.String;)
        at java.lang.Class.getMethod(Unknown Source)
        at com.aliyun.odps.mapred.cli.Cli.main(Cli.java:35)


代码是官方的WordCount示例代码:
package com.aliyun.odps.mapred.open.example;

import java.io.IOException;
import java.util.Iterator;

import com.aliyun.odps.data.Record;
import com.aliyun.odps.data.TableInfo;
import com.aliyun.odps.mapred.JobClient;
import com.aliyun.odps.mapred.MapperBase;
import com.aliyun.odps.mapred.ReducerBase;
import com.aliyun.odps.mapred.TaskContext;
import com.aliyun.odps.mapred.conf.JobConf;
import com.aliyun.odps.mapred.utils.InputUtils;
import com.aliyun.odps.mapred.utils.OutputUtils;
import com.aliyun.odps.mapred.utils.SchemaUtils;

public class WordCount {

   public static class TokenizerMapper extends MapperBase {
     private Record word;
     private Record one;

     @Override
     public void setup(TaskContext context) throws IOException {
       word = context.createMapOutputKeyRecord();
       one = context.createMapOutputValueRecord();
       one.set(new Object[] { 1L });
       System.out.println("TaskID:" + context.getTaskID().toString());
     }

     @Override
     public void map(long recordNum, Record record, TaskContext context)
         throws IOException {
       for (int i = 0; i < record.getColumnCount(); i++) {
         word.set(new Object[] { record.get(i).toString() });
         context.write(word, one);
       }
     }
   }

   /**
    * A combiner class that combines map output by sum them.
    **/
   public static class SumCombiner extends ReducerBase {
     private Record count;

     @Override
     public void setup(TaskContext context) throws IOException {
       count = context.createMapOutputValueRecord();
     }

     @Override
     public void reduce(Record key, Iterator<Record> values, TaskContext context)
         throws IOException {
       long c = 0;
       while (values.hasNext()) {
         Record val = values.next();
         c += (Long) val.get(0);
       }
       count.set(0, c);
       context.write(key, count);
     }
   }

   /**
    * A reducer class that just emits the sum of the input values.
    **/
   public static class SumReducer extends ReducerBase {
     private Record result = null;

     @Override
     public void setup(TaskContext context) throws IOException {
       result = context.createOutputRecord();
     }

     @Override
     public void reduce(Record key, Iterator<Record> values, TaskContext context)
         throws IOException {
       long count = 0;
       while (values.hasNext()) {
         Record val = values.next();
         count += (Long) val.get(0);
       }
       result.set(0, key.get(0));
       result.set(1, count);
       context.write(result);
     }
   }

   public static void main(String[] args) throws Exception {
     if (args.length != 2) {
       System.err.println("Usage: WordCount <in_table> <out_table>");
       System.exit(2);
     }

     JobConf job = new JobConf();

     job.setMapperClass(TokenizerMapper.class);
     job.setCombinerClass(SumCombiner.class);
     job.setReducerClass(SumReducer.class);

     job.setMapOutputKeySchema(SchemaUtils.fromString("word:string"));
     job.setMapOutputValueSchema(SchemaUtils.fromString("count:bigint"));

     InputUtils.addTable(TableInfo.builder().tableName(args[0]).build(), job);
     OutputUtils.addTable(TableInfo.builder().tableName(args[1]).build(), job);

     JobClient.runJob(job);
   }

}


展开
收起
yangsw15406 2014-11-14 09:44:35 13870 0
2 条回答
写回答
取消 提交回答
  • 回1楼啊里新人的帖子
    把ODPS的CLT的jar包加入eclipse,本地运行
    WordCount.java右键,“Run As”,“Run Configurations …”:

    “Main”,”Project”:
    ODPS_WORD_COUNT
    “Main”,”Main class”:
    com.thomas.odps.WordCount

    “Arguments”,”Program arguments”:
    word_count word_count_result
    “Arguments”,”VM arguments”:
    -Dodps.runner.mode=local
    -Dodps.project.name=yangsw_test
    -Dodps.end.point=http://service.odps.aliyun.com/api
    -Dodps.access.id=**********
    -Dodps.access.key=**********
    配置好后点击“Run”,启动本地测试

    -------------------------

    ReODPSMR本地可以运行,上传资源使用ODPS却执行报错
    就这个问题在ODPS用户中心提交了一个工单,反馈的是ODPS MR功能还没有正式开放,需要等到12月底。
    另外 -classpath要指向本地,而不是指向已经上传的资源。
    2014-11-17 10:08:13
    赞同 展开评论 打赏
  • 支持本地了?
    2014-11-17 09:57:51
    赞同 展开评论 打赏
问答排行榜
最热
最新

相关电子书

更多
DataWorks数据集成实时同步最佳实践(含内测邀请)-2020飞天大数据平台实战应用第一季 立即下载
DataWorks调度任务迁移最佳实践-2020飞天大数据平台实战应用第一季 立即下载
DataWorks商业化资源组省钱秘籍-2020飞天大数据平台实战应用第一季 立即下载