开发者社区> 问答> 正文

跨账号的OSS对拷程序代码(JAVA)

有个把一个账号下的OSS下所有bucket拷贝到另一个账号下的需求,于是就写了下面的代码供大家参考。
前提:考虑到速度问题,我建议要有一个与OSS相同地域的ECS,这样可以方便使用内网模式。
代码:

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import com.aliyun.openservices.oss.OSSClient;
import com.aliyun.openservices.oss.OSSException;
import com.aliyun.openservices.oss.model.Bucket;
import com.aliyun.openservices.oss.model.ListObjectsRequest;
import com.aliyun.openservices.oss.model.OSSObject;
import com.aliyun.openservices.oss.model.OSSObjectSummary;
import com.aliyun.openservices.oss.model.ObjectListing;
import com.aliyun.openservices.oss.model.ObjectMetadata;

public class OSSCopy {

    public static void main(String[] args) throws Exception {
        String fAccessKey = args[0];
        String fAccessKeySecret = args[1];
        String tAccessKey = args[2];
        String tAccessKeySecret = args[3];

        System.out.println("Creating source client...");
        final OSSClient fClient = new OSSClient("http://oss-cn-hangzhou-internal.aliyuncs.com", fAccessKey, fAccessKeySecret); // TODO 源OSS
        System.out.println("Creating target client...");
        final OSSClient tClient = new OSSClient("http://oss-cn-hangzhou-internal.aliyuncs.com", tAccessKey, tAccessKeySecret); // TODO 目标OSS

        // ExecutorService es = Executors.newFixedThreadPool(10); // 一个accesskey也只有10个buckets
        
        // 列出所有Buckets
        List<Bucket> buckets = fClient.listBuckets();
        for (Bucket b : buckets) {
            final Bucket bucket = b;
            // es.execute(new Runnable() {
            //     public void run() {
                    String tBucketName = "ylr" + bucket.getName().substring("yunluru".length()); // TODO 这里需要修改一下前缀,不然会提示bucket重名
                    if (!tClient.doesBucketExist(tBucketName)) {
                        System.out.println("Creating bucket..." + tBucketName);
                        tClient.createBucket(tBucketName);
                    } else {
                        System.out.println("Entering bucket..." + tBucketName);
                    }

                    // 在Source OSS列表文件
                    Map<String, String> sFileList = new HashMap<String, String>();
                    ListObjectsRequest sRequest = new ListObjectsRequest(bucket.getName());
                    sRequest.setMaxKeys(1000);
                    ObjectListing list;
                    do {
                        list = fClient.listObjects(sRequest);
                        for (OSSObjectSummary os : list.getObjectSummaries()) {
                            sFileList.put(os.getKey(), os.getETag());
                        }
                        sRequest.setMarker(list.getNextMarker());
                    } while (list.getNextMarker() != null);
                    System.out.println("Listing source files in " + bucket.getName() + "..." + sFileList.size());
                    
                    // 在Target OSS列表文件
                    Map<String, String> tFileList = new HashMap<String, String>();
                    ListObjectsRequest tRequest = new ListObjectsRequest(tBucketName);
                    tRequest.setMaxKeys(1000);
                    do {
                        list = tClient.listObjects(tRequest);
                        for (OSSObjectSummary os : list.getObjectSummaries()) {
                            tFileList.put(os.getKey(), os.getETag());
                        }
                        tRequest.setMarker(list.getNextMarker());
                    } while (list.getNextMarker() != null);
                    System.out.println("Listing target files in " + tBucketName + "..." + tFileList.size());
                    
                    // 列表
                    Set<String> cFileList = new HashSet<String>(); // 拷贝
                    Set<String> dFileList = new HashSet<String>(); // 删除
                    
                    String[] sFileNames = new String[sFileList.size()];
                    sFileList.keySet().toArray(sFileNames);
                    for(String sFileName: sFileNames){
                        if(sFileName.endsWith("/")){
                            continue;
                        }
                        String sETag = sFileList.get(sFileName);
                        String tETag = tFileList.get(sFileName);
                        if(sETag != null && tETag != null){
                            if(sETag.equals(tETag)){
                                // 相同就忽略
                            }else{
                                // 不同就复制
                                cFileList.add(sFileName);
                            }
                        }else if(sETag != null){
                            // S有T无,复制
                            cFileList.add(sFileName);
                        }else if(tETag != null){
                            // S无T有,删除
                            dFileList.add(sFileName);
                        }
                    }
                    
                    String[] tFileNames = new String[tFileList.size()];
                    tFileList.keySet().toArray(tFileNames);
                    for(String tFileName: tFileNames){
                        if(tFileName.endsWith("/")){
                            continue;
                        }
                        String sETag = sFileList.get(tFileName);
                        String tETag = tFileList.get(tFileName);
                        if(sETag != null && tETag != null){
                            if(sETag.equals(tETag)){
                                // 相同就忽略
                            }else{
                                // 不同就复制
                                cFileList.add(tFileName);
                            }
                        }else if(sETag != null){
                            // S有T无,复制
                            cFileList.add(tFileName);
                        }else if(tETag != null){
                            // S无T有,删除
                            dFileList.add(tFileName);
                        }
                    }

                    sFileList.clear();
                    sFileList = null;
                    tFileList.clear();
                    tFileList = null;
                    
                    // 复制
                    for (String path : cFileList) {
                        
                        // 读取数据
                        try {
                            OSSObject fObject = fClient.getObject(bucket.getName(), path);
                            ObjectMetadata fmd = fObject.getObjectMetadata();
                            InputStream is = fObject.getObjectContent();
                            ByteArrayOutputStream os = new ByteArrayOutputStream((int) fmd.getContentLength());
                            byte[] cache = new byte[4096];
                            for (int i = is.read(cache); i != -1; i = is.read(cache)) {
                                os.write(cache, 0, i);
                            }
                            os.flush();
                            os.close();
                            is.close();

                            // 写入数据
                            tClient.putObject(tBucketName, path, new ByteArrayInputStream(os.toByteArray()), fmd);
                            System.out.println(" " + path + "...rwd");
                        } catch (IOException e) {
                            System.out.println(" " + path + "...err");
                            continue;
                        }
                    }

                    // 删除
                    for (String path : dFileList) {
                        tClient.deleteObject(tBucketName, path);
                        System.out.println(" " + path + "...del");
                    }
            //     }
            // });
        }
        
        // es.shutdown();
    }
}


展开
收起
yipsilon 2014-10-18 10:12:45 10713 0
1 条回答
写回答
取消 提交回答
  • 更新啦,去掉注释的代码可以实现并行处理,充分利用多核CPU资源。
    2014-10-21 23:26:34
    赞同 展开评论 打赏
问答排行榜
最热
最新

相关电子书

更多
Spring Cloud Alibaba - 重新定义 Java Cloud-Native 立即下载
The Reactive Cloud Native Arch 立即下载
JAVA开发手册1.5.0 立即下载