DataWorks OpenAPI 示例(元数据模块)-阿里云开发者社区

开发者社区> DataWorks> 正文
登录阅读全文

DataWorks OpenAPI 示例(元数据模块)

简介: DataWorks OpenAPI 示例

DataWorks OpenAPI 元数据demo

Client初始化

    String region = "cn-shanghai";
    IClientProfile profile = DefaultProfile.getProfile(region, "xxx", "xxx");
    DefaultProfile.addEndpoint(region,"dataworks-public", String.format("dataworks.%s.aliyuncs.com", region));
    IAcsClient client = new DefaultAcsClient(profile);

GetMetaDBInfo

  • 获取odps引擎详情
    GetMetaDBInfoRequest request = new GetMetaDBInfoRequest();
    //odps app guid,格式 odps.{projectName}
    request.setAppGuid("odps.project1");
    //类型为odps
    request.setDataSourceType("odps");
    GetMetaDBInfoResponse res = client.getAcsResponse(request);
    GetMetaDBInfoResponse.Data data = res.getData();
  • 获取emr库详情

    GetMetaDBInfoRequest request = new GetMetaDBInfoRequest();
    request.setDataSourceType("emr");
    //emr 集群id
    request.setClusterId("C-C206DE8CACxxxxx");
    //emr db名称
    request.setDatabaseName("default");
    
    GetMetaDBInfoResponse res = client.getAcsResponse(request);
    GetMetaDBInfoResponse.Data data = res.getData();

ListMetaDB

  • 获取emr 数据库列表

    ListMetaDBRequest request = new ListMetaDBRequest();
    //数据类型
    request.setDataSourceType("emr");
    //dataworks 项目id
    request.setProjectId(110L);
    request.setPageSize(1);
    request.setPageNum(10);
    
    ListMetaDBResponse res = client.getAcsResponse(request);
    //总条数
    res.getDatabaseInfo().getTotalCount();
    //数据
    res.getDatabaseInfo().getDbList();

GetMetaDBTableList

  • 获取odps引擎关联表
    GetMetaDBTableListRequest request = new GetMetaDBTableListRequest();
    //app guid
    request.setAppGuid("odps.project1");
    request.setDataSourceType("odps");
    //第1页
    request.setPageNumber(1);
    //每页大小
    request.setPageSize(10);
    GetMetaDBTableListResponse res =  client.getAcsResponse(request);
    //数据总大小
    Long totalCount = res.getData().getTotalCount();
    //当前页数据列表
    List<GetMetaDBTableListResponse.Data.TableEntityListItem>  tableEntityListItems = res.getData().getTableEntityList();
  • 获取emr 数据库下的表

    GetMetaDBTableListRequest request = new GetMetaDBTableListRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CAC0xxxxx");
    //emr 数据库名称
    request.setDatabaseName("default");
    
    GetMetaDBTableListResponse res = client.getAcsResponse(request);
    //数据总大小
    Long totalCount = res.getData().getTotalCount();
    //当前页数据列表
    List<GetMetaDBTableListResponse.Data.TableEntityListItem>  tableEntityListItems = res.getData().getTableEntityList();

CheckMetaTable

  • 检查odps表是否存在
    CheckMetaTableRequest request = new CheckMetaTableRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //资源类型
    request.setDataSourceType("odps");
    CheckMetaTableResponse res = client.getAcsResponse(request);
    //表是否存在
    Boolean exists = res.getData();
  • 检查emr表是否存在

    CheckMetaTableRequest request = new CheckMetaTableRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 数据库名称
    request.setDatabaseName("default");
    //表名称
    request.setTableName("table1");
    
    CheckMetaTableResponse res = client.getAcsResponse(request);

CheckMetaPartition

  • 检查odps分区是否存在

    CheckMetaPartitionRequest request = new CheckMetaPartitionRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //资源类型
    request.setDataSourceType("odps");
    //分区
    request.setPartition("dt_time=20200419");
    
    CheckMetaPartitionResponse res = client.getAcsResponse(request);
    //分区是否存在
    Boolean exists = res.getData();
  • 检查emr分区是否存在

    CheckMetaPartitionRequest request = new CheckMetaPartitionRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 数据库名称
    request.setDatabaseName("default");
    //表名称
    request.setTableName("table1");
    
    request.setPartition("ds=202001");
    
    CheckMetaPartitionResponse res = client.getAcsResponse(request);
    //分区是否存在
    Boolean exists = res.getData();

SearchMetaTables

  • 搜索odps表

    SearchMetaTablesRequest request = new SearchMetaTablesRequest();
    //odps app guid,格式 odps.{projectName}
    request.setAppGuid("odps.p1");
    request.setDataSourceType("odps");
    //表类型,0:table, 1:view  默认全部
    request.setEntityType(0);
    //关键字
    request.setKeyword("test");
    //分页
    request.setPageNumber(1);
    request.setPageSize(10);
    
    SearchMetaTablesResponse res = client.getAcsResponse(request);
    //数据总条数
    Long totalCount = res.getData().getTotalCount();
    //当前页表列表
    res.getData().getDataEntityList();
  • 搜索emr表

    SearchMetaTablesRequest request = new SearchMetaTablesRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    request.setKeyword("test");
    request.setPageNumber(1);
    request.setPageSize(10);
    
    SearchMetaTablesResponse res = client.getAcsResponse(request);
    //数据总条数
    Long totalCount = res.getData().getTotalCount();
    //当前页表列表
    res.getData().getDataEntityList();

GetMetaTableBasicInfo

  • 获取odps表基本信息

    GetMetaTableBasicInfoRequest request = new GetMetaTableBasicInfoRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //资源类型
    request.setDataSourceType("odps");
    
    GetMetaTableBasicInfoResponse res = client.getAcsResponse(request);
    //表数据
    GetMetaTableBasicInfoResponse.Data info = res.getData();
  • 获取emr表基础信息

    GetMetaTableBasicInfoRequest request = new GetMetaTableBasicInfoRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 数据库名称
    request.setDatabaseName("default");
    //表名称
    request.setTableName("table1");
    
    GetMetaTableBasicInfoResponse res = client.getAcsResponse(request);
    //表数据
    GetMetaTableBasicInfoResponse.Data info = res.getData();

GetMetaTableColumn

  • 获取odps column列表
    
    GetMetaTableColumnRequest request = new GetMetaTableColumnRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //资源类型
    request.setDataSourceType("odps");
    //分页
    request.setPageNum(1);
    request.setPageSize(10);
    GetMetaTableColumnResponse res = client.getAcsResponse(request);
    //数据总条数
    Long totalCount = res.getData().getTotalCount();
    //当前页数据列表
    List<GetMetaTableColumnResponse.Data.ColumnListItem> columnList = res.getData().getColumnList();
* 获取emr表column信息
```java
    GetMetaTableColumnRequest request = new GetMetaTableColumnRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 数据库名称
    request.setDatabaseName("default");
    //表名称
    request.setTableName("table1");
    request.setPageNum(1);
    request.setPageSize(10);
    GetMetaTableColumnResponse res = client.getAcsResponse(request);
    //数据总条数
    Long totalCount = res.getData().getTotalCount();
    //当前页数据列表
    List<GetMetaTableColumnResponse.Data.ColumnListItem> columnList = res.getData().getColumnList();

GetMetaTableFullInfo

  • 获取odps表详情,包含基础信息和column信息

    GetMetaTableFullInfoRequest request = new GetMetaTableFullInfoRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //资源类型
    request.setDataSourceType("odps");
    //column 分页
    request.setPageNum(1);
    //column 分页大小
    request.setPageSize(10);
    
    GetMetaTableFullInfoResponse res = client.getAcsResponse(request);
    //表数据,包含column
    GetMetaTableFullInfoResponse.Data info = res.getData();
    //column数据总条数
    Long columnTotalCount = res.getData().getTotalColumnCount();
    //当前页数据列表
    List<GetMetaTableFullInfoResponse.Data.ColumnListItem> columnList = res.getData().getColumnList();
  • 获取emr表详情,包含基础信息和column信息

    GetMetaTableFullInfoRequest request = new GetMetaTableFullInfoRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 数据库名称
    request.setDatabaseName("default");
    //表名称
    request.setTableName("table1");
    request.setPageNum(1);
    request.setPageSize(10);
    
    GetMetaTableFullInfoResponse res = client.getAcsResponse(request);
    //表数据,包含column
    GetMetaTableFullInfoResponse.Data info = res.getData();
    //column数据总条数
    Long columnTotalCount = res.getData().getTotalColumnCount();
    //当前页数据列表
    List<GetMetaTableFullInfoResponse.Data.ColumnListItem> columnList = res.getData().getColumnList();

GetMetaTablePartition

  • 获取odps表分区
    GetMetaTablePartitionRequest request = new GetMetaTablePartitionRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //资源类型
    request.setDataSourceType("odps");
    //分页
    request.setPageNumber(1);
    request.setPageSize(10);
    GetMetaTablePartitionResponse res = client.getAcsResponse(request);
    //数据总条数
    res.getData().getTotalCount();
    //分区列表
    res.getData().getDataEntityList();
  • 获取emr表分区

    GetMetaTablePartitionRequest request = new GetMetaTablePartitionRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 数据库名称
    request.setDatabaseName("default");
    //表名称
    request.setTableName("table1");
    request.setPageNumber(1);
    request.setPageSize(10);
    
    GetMetaTablePartitionResponse res = client.getAcsResponse(request);
    //数据总条数
    res.getData().getTotalCount();
    //分区列表
    res.getData().getDataEntityList();

GetMetaTableOutput

  • 获取odps表输出
    GetMetaTableOutputRequest request = new GetMetaTableOutputRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //output开始时间
    request.setStartDate("2020-05-27");
    //output结束时间
    request.setEndDate("2020-06-12");
    request.setPageNumber(1);
    request.setPageSize(10);
    GetMetaTableOutputResponse res = client.getAcsResponse(request);
    //数据总条数
    res.getData().getTotalCount();
    //输出列表
    res.getData().getDataEntityList();

GetMetaTableChangeLog

  • 获取odps表变更信息
    GetMetaTableChangeLogRequest request = new GetMetaTableChangeLogRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //开始时间,格式yyyy-MM-dd HH:mm:ss
    request.setStartDate("2020-01-01 01:01:00");
    //格式yyyy-MM-dd HH:mm:ss
    request.setEndDate("2020-01-02 01:01:00");
    /**
     * 类型:
     * CREATE_TABLE
     * ALTER_TABLE
     * DROP_TABLE
     * ADD_PARTITION
     * DROP_PARTITION
     * ALTER_OWNER_ID
     * ALTER_LIFE_CYCLE_TIME
     */
    request.setChangeType("CREATE_TABLE");
    //对象 类型 TABLE,PARTITION
    request.setObjectType("TABLE");
    //分页
    request.setPageNumber(1);
    request.setPageSize(10);
    GetMetaTableChangeLogResponse res = client.getAcsResponse(request);
    //数据总条数
    res.getData().getTotalCount();
    //数据
    res.getData().getDataEntityList();

GetMetaTableIntroWiki

  • 获取wiki
    GetMetaTableIntroWikiRequest request = new GetMetaTableIntroWikiRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //wiki版本
    request.setWikiVersion(1L);
    GetMetaTableIntroWikiResponse res = client.getAcsResponse(request);
    //数据
    res.getData();

GetMetaTableLineage

  • 获取odps表血缘

    GetMetaTableLineageRequest request = new GetMetaTableLineageRequest();
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    request.setDataSourceType("odps");
    //方向,up:上游 down:下游
    request.setDirection("up");
    //每页大小
    request.setPageSize(10);
    //下一页的起始key,null的时候从最开始查询
    String nextPrimaryKey = null;
    //是否有下一页
    Boolean hasNext = false;
    
    List<GetMetaTableLineageResponse.Data.DataEntityListItem> dataEntityListItems = new ArrayList<>();
    
    do {
        //分页起始key,返回数据中获取
        request.setNextPrimaryKey(nextPrimaryKey);
    
        GetMetaTableLineageResponse res = client.getAcsResponse(request);
        List<GetMetaTableLineageResponse.Data.DataEntityListItem> items = res.getData().getDataEntityList();
        dataEntityListItems.addAll(items);
        //是否有下一页
        hasNext = res.getData().getHasNext();
        //下一页起始key
        nextPrimaryKey = res.getData().getNextPrimaryKey();
    
    } while(hasNext);
  • 获取emr表血缘

    GetMetaTableLineageRequest request = new GetMetaTableLineageRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 数据库名称
    request.setDatabaseName("default");
    //表名称
    request.setTableName("table1");
    //下一次请求开始id,参考 GetMetaTableLineageResponse.getData().getNextPrimaryKey()
    request.setDirection("up");
    request.setPageSize(10);
    //下一页的起始key,null的时候从最开始查询
    String nextPrimaryKey = null;
    //是否有下一页
    Boolean hasNext = false;
    
    List<GetMetaTableLineageResponse.Data.DataEntityListItem> dataEntityListItems = new ArrayList<>();
    
    do {
        //分页起始key,返回数据中获取
        request.setNextPrimaryKey(nextPrimaryKey);
    
        GetMetaTableLineageResponse res = client.getAcsResponse(request);
        List<GetMetaTableLineageResponse.Data.DataEntityListItem> items = res.getData().getDataEntityList();
        dataEntityListItems.addAll(items);
        //是否有下一页
        hasNext = res.getData().getHasNext();
        //下一页起始key
        nextPrimaryKey = res.getData().getNextPrimaryKey();
    
    } while(hasNext);

GetMetaColumnLineage

  • 获取odps字段血缘
    GetMetaColumnLineageRequest request = new GetMetaColumnLineageRequest();
    //column guid,格式 odps.{projectName}.{tableName}.{columnName}
    request.setColumnGuid("odps.project1.t1");
    request.setDataSourceType("odps");
    //方向,up:上游 down:下游
    request.setDirection("down");
    //分页
    request.setPageNum(1);
    request.setPageSize(10);
    GetMetaColumnLineageResponse res = client.getAcsResponse(request);
    //数据总条数
    res.getData().getTotalCount();
    //数据
    res.getData().getDataEntityList();
  • 获取emr字段血缘
    GetMetaColumnLineageRequest request = new GetMetaColumnLineageRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 数据库名称
    request.setDatabaseName("default");
    //表名称
    request.setTableName("table1");
    request.setDirection("up");
    request.setColumnName("eid");
    request.setPageNum(1);
    request.setPageSize(10);
    GetMetaColumnLineageResponse res = client.getAcsResponse(request);
    //数据总条数
    res.getData().getTotalCount();
    //数据
    res.getData().getDataEntityList();

CreateMetaCategory

  • 创建类目

    CreateMetaCategoryRequest request = new CreateMetaCategoryRequest();
    //类目名称
    request.setName("test1");
    //注释
    request.setComment("test");
    //父类目id,0表示没有父类目
    request.setParentId(0L);
    
    CreateMetaCategoryResponse res = client.getAcsResponse(request);
    //生成的类目id
    res.getData().getCategoryId()

UpdateMetaCategory

  • 更新类目
    UpdateMetaCategoryRequest request = new UpdateMetaCategoryRequest();
    request.setCategoryId(53664L);
    request.setName("t93233");
    request.setComment("the com");
    UpdateMetaCategoryResponse res = client.getAcsResponse(request);
    //是否成功
    res.getData();

DeleteMetaCategory

  • 删除类目
    DeleteMetaCategoryRequest request = new DeleteMetaCategoryRequest();
    request.setCategoryId(53664L);
    DeleteMetaCategoryResponse res = client.getAcsResponse(request);
    //是否成功
    res.getData();

GetMetaCategory

  • 获取子类目列表
    GetMetaCategoryRequest request = new GetMetaCategoryRequest();
    //父类目id
    request.setParentCategoryId(0L);
    request.setPageNum(1);
    request.setPageSize(10);
    GetMetaCategoryResponse res = client.getAcsResponse(request);
    //总数据
    res.getData().getTotalCount();
    //数据列表
    res.getData().getDataEntityList();

GetMetaTableListByCategory

  • 获取类目关联表
    GetMetaTableListByCategoryRequest request = new GetMetaTableListByCategoryRequest();
    request.setCategoryId(53664L);
    request.setPageNumber(1);
    request.setPageSize(10);
    GetMetaTableListByCategoryResponse res = client.getAcsResponse(request);

AddToMetaCategory

  • 增加类目、表关联

    AddToMetaCategoryRequest request = new AddToMetaCategoryRequest();
    request.setCategoryId(53664L);
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    
    AddToMetaCategoryResponse res = client.getAcsResponse(request);

DeleteFromMetaCategory

  • 删除类目、表关联
    DeleteFromMetaCategoryRequest request = new DeleteFromMetaCategoryRequest();
    request.setCategoryId(53642L);
    //odps table  guid,格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    DeleteFromMetaCategoryResponse res = client.getAcsResponse(request);

版权声明:本文内容由阿里云实名注册用户自发贡献,版权归原作者所有,阿里云开发者社区不拥有其著作权,亦不承担相应法律责任。具体规则请查看《阿里云开发者社区用户服务协议》和《阿里云开发者社区知识产权保护指引》。如果您发现本社区中有涉嫌抄袭的内容,填写侵权投诉表单进行举报,一经查实,本社区将立刻删除涉嫌侵权内容。

分享:
DataWorks
使用钉钉扫一扫加入圈子
+ 订阅

DataWorks作为飞天大数据平台操作系统,对接各种大数据计算引擎,以all in one box的方式提供专业高效、安全可靠的全域智能大数据平台,高效率完成数据全链路研发流程,建设企业数据治理体系。 从2009年飞天大数据平台写下第一行代码开始,DataWorks历经10年发展,形成一套成熟的产品功能体系,满足企业数据中台搭建需求。

官方博客
DataWorks产品官网