环境要求：
1、分布式/伪分布式的hadoop环境【hadoop2.7.3】
2、win10本地Java环境【jdk8】
3、win10本地hadoop环境【2.7.3】
4、win10本地Maven环境【Maven3】
5、idea开发工具【IntelliJ IDEA 2020.1.3 x64】
1、项目创建：
需要建立【maven】项目，建立过程中
项目名称无所谓：
2、修改Maven
点击【file】->【Settings】后如下图操作，选择本地的Maven配置文件以及仓库地址。
引入【hadoop-client】
<dependencies> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <version>2.7.3</version> </dependency> </dependencies>
3、编码
在【src的java】下创建【com.item.test】包，以及【MapAction】【ReduceAction】【Action】的类文件。
【MapAction】编码
package com.item.test; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; import java.io.IOException; public class MapAction extends Mapper<LongWritable, Text, Text, LongWritable> { @Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { char[] split = value.toString().toCharArray(); for (char s : split) { context.write(new Text(s+""), new LongWritable(1)); } } }
【ReduceAction】编码
package com.item.test; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Reducer; import java.io.IOException; public class ReduceAction extends Reducer<Text, LongWritable, Text, LongWritable> { @Override protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException { long count = 0; for (LongWritable value : values) { count += value.get(); } context.write(key, new LongWritable(count)); } }
【Action】编码
package com.item.test; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import java.io.IOException; public class Action { public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf); job.setJarByClass(Action.class); job.setMapperClass(MapAction.class); job.setReducerClass(ReduceAction.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); //本地测试 FileInputFormat.setInputPaths(job,new Path("D:\\info.txt")); FileOutputFormat.setOutputPath(job,new Path("D:\\infos")); boolean b = job.waitForCompletion(true); System.exit(b?0:1); } }
