1 import org.apache.hadoop.conf.Configuration;
2 import org.apache.hadoop.fs.*;
3 import org.apache.hadoop.io.IOUtils;
4
5 import java.io.ByteArrayOutputStream;
6 import java.io.IOException;
7 import java.io.InputStream;
8 import java.io.OutputStream;
9
10 /**
11 * HDFS工具类
12 * Author: 菩提树下的杨过(http://yjmyzz.cnblogs.com)
13 * Since: 2015-05-21
14 */
15 public class HDFSUtil {
16
17
18 private HDFSUtil() {
19
20 }
21
22 /**
23 * 判断路径是否存在
24 *
25 * @param conf
26 * @param path
27 * @return
28 * @throws IOException
29 */
30 public static boolean exits(Configuration conf, String path) throws IOException {
31 FileSystem fs = FileSystem.get(conf);
32 return fs.exists(new Path(path));
33 }
34
35 /**
36 * 创建文件
37 *
38 * @param conf
39 * @param filePath
40 * @param contents
41 * @throws IOException
42 */
43 public static void createFile(Configuration conf, String filePath, byte[] contents) throws IOException {
44 FileSystem fs = FileSystem.get(conf);
45 Path path = new Path(filePath);
46 FSDataOutputStream outputStream = fs.create(path);
47 outputStream.write(contents);
48 outputStream.close();
49 fs.close();
50 }
51
52 /**
53 * 创建文件
54 *
55 * @param conf
56 * @param filePath
57 * @param fileContent
58 * @throws IOException
59 */
60 public static void createFile(Configuration conf, String filePath, String fileContent) throws IOException {
61 createFile(conf, filePath, fileContent.getBytes());
62 }
63
64 /**
65 * @param conf
66 * @param localFilePath
67 * @param remoteFilePath
68 * @throws IOException
69 */
70 public static void copyFromLocalFile(Configuration conf, String localFilePath, String remoteFilePath) throws IOException {
71 FileSystem fs = FileSystem.get(conf);
72 Path localPath = new Path(localFilePath);
73 Path remotePath = new Path(remoteFilePath);
74 fs.copyFromLocalFile(true, true, localPath, remotePath);
75 fs.close();
76 }
77
78 /**
79 * 删除目录或文件
80 *
81 * @param conf
82 * @param remoteFilePath
83 * @param recursive
84 * @return
85 * @throws IOException
86 */
87 public static boolean deleteFile(Configuration conf, String remoteFilePath, boolean recursive) throws IOException {
88 FileSystem fs = FileSystem.get(conf);
89 boolean result = fs.delete(new Path(remoteFilePath), recursive);
90 fs.close();
91 return result;
92 }
93
94 /**
95 * 删除目录或文件(如果有子目录,则级联删除)
96 *
97 * @param conf
98 * @param remoteFilePath
99 * @return
100 * @throws IOException
101 */
102 public static boolean deleteFile(Configuration conf, String remoteFilePath) throws IOException {
103 return deleteFile(conf, remoteFilePath, true);
104 }
105
106 /**
107 * 文件重命名
108 *
109 * @param conf
110 * @param oldFileName
111 * @param newFileName
112 * @return
113 * @throws IOException
114 */
115 public static boolean renameFile(Configuration conf, String oldFileName, String newFileName) throws IOException {
116 FileSystem fs = FileSystem.get(conf);
117 Path oldPath = new Path(oldFileName);
118 Path newPath = new Path(newFileName);
119 boolean result = fs.rename(oldPath, newPath);
120 fs.close();
121 return result;
122 }
123
124 /**
125 * 创建目录
126 *
127 * @param conf
128 * @param dirName
129 * @return
130 * @throws IOException
131 */
132 public static boolean createDirectory(Configuration conf, String dirName) throws IOException {
133 FileSystem fs = FileSystem.get(conf);
134 Path dir = new Path(dirName);
135 boolean result = fs.mkdirs(dir);
136 fs.close();
137 return result;
138 }
139
140 /**
141 * 列出指定路径下的所有文件(不包含目录)
142 *
143 * @param conf
144 * @param basePath
145 * @param recursive
146 */
147 public static RemoteIterator<LocatedFileStatus> listFiles(FileSystem fs, String basePath, boolean recursive) throws IOException {
148
149 RemoteIterator<LocatedFileStatus> fileStatusRemoteIterator = fs.listFiles(new Path(basePath), recursive);
150
151 return fileStatusRemoteIterator;
152 }
153
154 /**
155 * 列出指定路径下的文件（非递归）
156 *
157 * @param conf
158 * @param basePath
159 * @return
160 * @throws IOException
161 */
162 public static RemoteIterator<LocatedFileStatus> listFiles(Configuration conf, String basePath) throws IOException {
163 FileSystem fs = FileSystem.get(conf);
164 RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(new Path(basePath), false);
165 fs.close();
166 return remoteIterator;
167 }
168
169 /**
170 * 列出指定目录下的文件\子目录信息（非递归）
171 *
172 * @param conf
173 * @param dirPath
174 * @return
175 * @throws IOException
176 */
177 public static FileStatus[] listStatus(Configuration conf, String dirPath) throws IOException {
178 FileSystem fs = FileSystem.get(conf);
179 FileStatus[] fileStatuses = fs.listStatus(new Path(dirPath));
180 fs.close();
181 return fileStatuses;
182 }
183
184
185 /**
186 * 读取文件内容
187 *
188 * @param conf
189 * @param filePath
190 * @return
191 * @throws IOException
192 */
193 public static String readFile(Configuration conf, String filePath) throws IOException {
194 String fileContent = null;
195 FileSystem fs = FileSystem.get(conf);
196 Path path = new Path(filePath);
197 InputStream inputStream = null;
198 ByteArrayOutputStream outputStream = null;
199 try {
200 inputStream = fs.open(path);
201 outputStream = new ByteArrayOutputStream(inputStream.available());
202 IOUtils.copyBytes(inputStream, outputStream, conf);
203 fileContent = outputStream.toString();
204 } finally {
205 IOUtils.closeStream(inputStream);
206 IOUtils.closeStream(outputStream);
207 fs.close();
208 }
209 return fileContent;
210 }
211 }
