Hadoop学习---7、OutputFormat数据输出、MapReduce内核源码解析、Join应用、数据清洗、MapReduce开发总结（一）

package org.example._08outputformat;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
/**
 * @ClassName LogMapper
 * @Description TODO
 * @Author Zouhuiming
 * @Date 2023/5/22 9:26
 * @Version 1.0
 */
public class LogMapper extends Mapper<LongWritable, Text,Text, NullWritable> {
    @Override
    protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, NullWritable>.Context context) throws IOException, InterruptedException {
        context.write(value,NullWritable.get());
    }
}

package org.example._08outputformat;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
/**
 * @ClassName LogReducer
 * @Description TODO
 * @Author Zouhuiming
 * @Date 2023/5/22 9:30
 * @Version 1.0
 */
public class LogReducer extends Reducer<Text, NullWritable,Text,NullWritable> {
    @Override
    protected void reduce(Text key, Iterable<NullWritable> values, Reducer<Text, NullWritable, Text, NullWritable>.Context context) throws IOException, InterruptedException {
        for (NullWritable value : values) {
            context.write(key,value);
        }
    }
}

package org.example._08outputformat;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
/**
 * @ClassName LogOutFormat
 * @Description TODO
 * @Author Zouhuiming
 * @Date 2023/5/22 9:31
 * @Version 1.0
 */
public class LogOutputFormat extends FileOutputFormat<Text, NullWritable> {
    @Override
    public RecordWriter<Text, NullWritable> getRecordWriter(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
        //自定义一个RecordWriter返回
        LogRecordWriter logRecordWriter=new LogRecordWriter(taskAttemptContext);
        return logRecordWriter;
    }
}

package org.example._08outputformat;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import java.io.IOException;
/**
 * @ClassName LogRecordWriter
 * @Description TODO
 * @Author Zouhuiming
 * @Date 2023/5/22 9:33
 * @Version 1.0
 */
public class LogRecordWriter extends RecordWriter<Text, NullWritable> {
    private FSDataOutputStream zhmOut;
    private FSDataOutputStream otherOut;
    public LogRecordWriter(TaskAttemptContext job) {
        try {
            //获取文件系统对象
            FileSystem fileSystem=FileSystem.get(job.getConfiguration());
            //利用文件系统对象创建两个输出流对应不同的目录
            zhmOut=fileSystem.create(new Path("E:\\test\\output7\\zhm.log"));
            otherOut=fileSystem.create(new Path("E:\\test\\output7\\other.log"));
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
    @Override
    public void write(Text text, NullWritable nullWritable) throws IOException, InterruptedException {
        String log=text.toString();
        //根据一行的log数据是否包含atguigu,判段两条输出输入流输出的内容
        if (log.contains("atguigu")){
            zhmOut.writeBytes(log+"\n");
        }else {
            otherOut.writeBytes(log+"\n");
        }
    }
    @Override
    public void close(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
        IOUtils.closeStream(zhmOut);
        IOUtils.closeStream(otherOut);
    }
}

package org.example._08outputformat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
/**
 * @ClassName LogDriver
 * @Description TODO
 * @Author Zouhuiming
 * @Date 2023/5/22 9:50
 * @Version 1.0
 */
public class LogDriver {
    public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
        Configuration configuration=new Configuration();
        Job job=Job.getInstance();
        job.setJarByClass(LogDriver.class);
        job.setMapperClass(LogMapper.class);
        job.setReducerClass(LogReducer.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(NullWritable.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);
        //设置自定义的outputformat
        job.setOutputFormatClass(LogOutputFormat.class);
        FileInputFormat.setInputPaths(job,new Path("E:\\test\\input3"));
        //虽然我们自定义了outputformat，但是因为我们的outputformat继承自fileOutputformat
        //而fileoutputformat要输出一个_success文件，所以在这还得指定一个输出目录
        FileOutputFormat.setOutputPath(job,new Path("E:\\test\\output7"));
        System.exit(job.waitForCompletion(true)?0:1);
    }
}

job.setNumReduceTasks(4);//这样就设置了4个ReduceTask