2. 查看读写Avro文件结果
a. 写Avro文件
b. 读Avro文件
3. 编码实现读写Avro文件（HDFS）
a. 引入所需要的jar包
<dependency> <groupId>org.apache.avro</groupId> <artifactId>avro-mapred</artifactId> <version>1.8.0</version> </dependency>
b. 写Avro文件到HDFS完整代码
package com.shaonaiyi.hadoop.filetype.avro; import org.apache.avro.mapred.AvroKey; import org.apache.avro.mapreduce.AvroJob; import org.apache.avro.mapreduce.AvroKeyOutputFormat; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.*; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.task.JobContextImpl; import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import java.io.IOException; /** * @Author shaonaiyi@163.com * @Date 2019/12/17 17:15 * @Description 编码实现写Avro文件到HDFS */ public class MRAvroFileWriter { public static void main(String[] args) throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, InterruptedException { //1 构建一个job实例 Configuration hadoopConf = new Configuration(); Job job = Job.getInstance(hadoopConf); //2 设置job的相关属性 // job.setOutputKeyClass(NullWritable.class); // job.setOutputValueClass(Text.class); // job.setOutputFormatClass(TextOutputFormat.class); //job.setOutputKeyClass(AvroKey.class); //job.setOutputValueClass(Person.class); job.setOutputFormatClass(AvroKeyOutputFormat.class); //AvroJob.setOutputKeySchema(job, Schema.create(Schema.Type.INT)); AvroJob.setOutputKeySchema(job, Person.SCHEMA$); //3 设置输出路径 FileOutputFormat.setOutputPath(job, new Path("hdfs://master:9999/user/hadoop-sny/mr/filetype/avro")); //FileOutputFormat.setCompressOutput(job, true); //FileOutputFormat.setOutputCompressorClass(job, GzipCodec.class); //4 构建JobContext JobID jobID = new JobID("jobId", 123); JobContext jobContext = new JobContextImpl(job.getConfiguration(), jobID); //5 构建taskContext TaskAttemptID attemptId = new TaskAttemptID("jobTrackerId", 123, TaskType.REDUCE, 0, 0); TaskAttemptContext hadoopAttemptContext = new TaskAttemptContextImpl(job.getConfiguration(), attemptId); //6 构建OutputFormat实例 OutputFormat format = job.getOutputFormatClass().newInstance(); //7 设置OutputCommitter OutputCommitter committer = format.getOutputCommitter(hadoopAttemptContext); committer.setupJob(jobContext); committer.setupTask(hadoopAttemptContext); //8 获取writer写数据，写完关闭writer RecordWriter<AvroKey, Person> writer = format.getRecordWriter(hadoopAttemptContext); // writer.write(null, new Text("shao")); // writer.write(null, new Text("nai")); // writer.write(null, new Text("yi")); // writer.write(null, new Text("bigdata-man")); Person person = new Person(); person.setName("jeffy"); person.setAge(20); person.setFavoriteNumber(10); person.setFavoriteColor("red"); writer.write(new AvroKey(person), null); writer.close(hadoopAttemptContext); //9 committer提交job和task committer.commitTask(hadoopAttemptContext); committer.commitJob(jobContext); } }
与写Text格式（文章链接跳转：Hadoop支持的文件格式之Text）时类似，主要不同如下：
c. 从HDFS上读Avro文件完整代码
package com.shaonaiyi.hadoop.filetype.avro; import org.apache.avro.mapred.AvroKey; import org.apache.avro.mapreduce.AvroKeyInputFormat; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.*; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.task.JobContextImpl; import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import java.io.IOException; import java.util.List; import java.util.function.Consumer; /** * @Author shaonaiyi@163.com * @Date 2019/12/17 17:29 * @Description 编码实现从HDFS上读Avro文件 */ public class MRAvroFileReader { public static void main(String[] args) throws IOException, IllegalAccessException, InstantiationException { //1 构建一个job实例 Configuration hadoopConf = new Configuration(); Job job = Job.getInstance(hadoopConf); //2 设置需要读取的文件全路径 FileInputFormat.setInputPaths(job, "hdfs://master:9999/user/hadoop-sny/mr/filetype/avro"); //3 获取读取文件的格式 // TextInputFormat inputFormat = TextInputFormat.class.newInstance(); AvroKeyInputFormat inputFormat = AvroKeyInputFormat.class.newInstance(); //4 获取需要读取文件的数据块的分区信息 //4.1 获取文件被分成多少数据块了 JobID jobID = new JobID("jobId", 123); JobContext jobContext = new JobContextImpl(job.getConfiguration(), jobID); List<InputSplit> inputSplits = inputFormat.getSplits(jobContext); //读取每一个数据块的数据 inputSplits.forEach(new Consumer<InputSplit>() { @Override public void accept(InputSplit inputSplit) { TaskAttemptID attemptId = new TaskAttemptID("jobTrackerId", 123, TaskType.MAP, 0, 0); TaskAttemptContext hadoopAttemptContext = new TaskAttemptContextImpl(job.getConfiguration(), attemptId); // RecordReader reader = inputFormat.createRecordReader(inputSplit, hadoopAttemptContext); RecordReader<AvroKey<Person>, NullWritable> reader = null; try { // reader.initialize(inputSplit, hadoopAttemptContext); // System.out.println("<key,value>"); // System.out.println("-----------"); // while (reader.nextKeyValue()) { // System.out.println("<"+reader.getCurrentKey() + "," + reader.getCurrentValue()+ ">" ); // } reader = inputFormat.createRecordReader(inputSplit, hadoopAttemptContext); reader.initialize(inputSplit, hadoopAttemptContext); while (reader.nextKeyValue()) { Person person = reader.getCurrentKey().datum(); System.out.println("key=>" + person); System.out.println("value=>" + reader.getCurrentValue()); } reader.close(); } catch (IOException e) { e.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } } }); } }
