开发者社区> 问答> 正文

Hadoop自定义partition运行失败?报错

如下代码在伪分布模式下运行报错,各位大神有什么相应的解决办法吗?

package preview;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class SortNum extends Configured implements Tool {

        public static class MapClass extends Mapper<Object, Text, IntWritable, IntWritable>{
                private static IntWritable data = new IntWritable();

                /**
                 * map 将输入中的value转化成IntWritable类型,作为输出的Key
                 */
                @Override
                protected void map(Object key, Text value, Context context)
                                throws IOException, InterruptedException {
                        String line = value.toString();
                        //data保存实际值,通过map的key输出
                        data.set(Integer.parseInt(line));
                        context.write(data, new IntWritable(1));
                }
        }
        
        public static class Reduce extends Reducer<IntWritable, IntWritable, IntWritable, IntWritable>{
                private static IntWritable lineNum = new IntWritable(1);

                @Override
                protected void reduce(IntWritable key, Iterable<IntWritable> values,
                                Context context)
                                throws IOException, InterruptedException {
                        for (IntWritable value : values) {
                                context.write(lineNum, key);
                                lineNum = new IntWritable(lineNum.get()+1);
                        }
                }
        }
        
        public static class Partition extends Partitioner<IntWritable, IntWritable>{

                @Override
                public int getPartition(IntWritable key, IntWritable value, int numPartitions) {
                        int Maxnumber = 65223;
                        int bound = Maxnumber / numPartitions + 1;
                        int keynumber = key.get();
                        for (int i = 0; i < numPartitions; i++) {
                                if(keynumber < bound*i && keynumber >= bound*(i-1))
                                        return i-1;
                        }
                        return -1;
                }
        }
        
        public int run(String[] args) throws Exception {
                
                Configuration conf = getConf();
                Job job = new Job(conf);
                job.setJobName("SortNum");
                job.setJarByClass(SortNum.class);
                
                Path in = new Path(args[0]);
                Path out = new Path(args[1]);
                FileInputFormat.setInputPaths(job, in);
                FileOutputFormat.setOutputPath(job, out);
                
                job.setMapperClass(MapClass.class);
                job.setReducerClass(Reduce.class);
                job.setPartitionerClass(Partition.class);
                
                job.setInputFormatClass(TextInputFormat.class);
                job.setOutputFormatClass(TextOutputFormat.class);
                job.setOutputKeyClass(Text.class);
                job.setOutputValueClass(Text.class);
                
                System.exit(job.waitForCompletion(true)?0:1);
                return 0;
        }

        public static void main(String[] args) {
                int ret = 0;
                try {
                        ret = ToolRunner.run(new Configuration(), new SortNum(), args);
                } catch (Exception e) {
                        e.printStackTrace();
                }
                System.exit(ret);
        }
}
报错信息:

Exception in thread "main" java.io.IOException: Error opening job jar: wordcount.jar
        at org.apache.hadoop.util.RunJar.main(RunJar.java:90)
Caused by: java.util.zip.ZipException: error in opening zip file
        at java.util.zip.ZipFile.open(Native Method)
        at java.util.zip.ZipFile.<init>(ZipFile.java:127)
        at java.util.jar.JarFile.<init>(JarFile.java:135)
        at java.util.jar.JarFile.<init>(JarFile.java:72)
        at org.apache.hadoop.util.RunJar.main(RunJar.java:88)
还有就是,在Hadoop环境下有什么比较好的程序调试工具吗?可以像eclipse debug那样跟踪代码的。跪谢!

展开
收起
爱吃鱼的程序员 2020-06-22 21:10:34 568 0
1 条回答
写回答
取消 提交回答
  • https://developer.aliyun.com/profile/5yerqm5bn5yqg?spm=a2c6h.12873639.0.0.6eae304abcjaIB

    有eclipse的hadoop插件。根据hadoop版本,自己编译一个。程序没错,是部署<spanstyle="font-family:Consolas,'BitstreamVeraSansMono','CourierNew',Courier,monospace;line-height:14.65625px;background-color:#FFFFFF;">wordcount.jar的路径问题。这种错误以前我也遇到过。你尝试把绝对路径加进去再看看

    2020-06-22 21:10:52
    赞同 展开评论 打赏
问答排行榜
最热
最新

相关电子书

更多
《构建Hadoop生态批流一体的实时数仓》 立即下载
零基础实现hadoop 迁移 MaxCompute 之 数据 立即下载
CIO 指南:如何在SAP软件架构中使用Hadoop 立即下载

相关实验场景

更多