OSS提供的分片上传(Multipart Upload)功能,将要上传的较大文件(Object)分成多个分片(Part)来分别上传,上传完成后再调用CompleteMultipartUpload接口将这些Part组合成一个Object来达到断点续传的效果。
分片上传流程
分片上传(Multipart Upload)分为以下三个步骤:
- 初始化一个分片上传事件。
调用oss_init_multipart_upload方法返回OSS创建的全局唯一的uploadId。
- 上传分片。
调用oss_upload_part_from_file方法上传分片数据。
说明- 对于同一个uploadId,分片号(PartNumber)标识了该分片在整个文件内的相对位置。如果使用同一个分片号上传了新的数据,则OSS上该分片已有的数据将会被覆盖。
- OSS将收到的分片数据的MD5值放在ETag头内返回给用户。
- OSS计算上传数据的MD5值,并与SDK计算的MD5值比较,如果不一致则返回InvalidDigest错误码。
- 完成分片上传。
所有分片上传完成后,调用oss_complete_multipart_upload方法将所有分片合并成完整的文件。
分片上传完整示例
以下通过一个完整的示例对分片上传的流程进行逐步解析:
#include "oss_api.h" #include "aos_http_io.h" #include <sys/stat.h> const char *endpoint = "<yourEndpoint>"; const char *access_key_id = "<yourAccessKeyId>"; const char *access_key_secret = "<yourAccessKeySecret>"; const char *bucket_name = "<yourBucketName>"; const char *object_name = "<yourObjectName>"; const char *local_filename = "<yourLocalFilename>"; void init_options(oss_request_options_t *options) { options->config = oss_config_create(options->pool); /* 用char*类型的字符串初始化aos_string_t类型。*/ aos_str_set(&options->config->endpoint, endpoint); aos_str_set(&options->config->access_key_id, access_key_id); aos_str_set(&options->config->access_key_secret, access_key_secret); /* 是否使用CNAME访问OSS服务。0表示不使用。*/ options->config->is_cname = 0; /* 设置网络相关参数,比如超时时间等。*/ options->ctl = aos_http_controller_create(options->pool, 0); } int64_t get_file_size(const char *file_path) { int64_t filesize = -1; struct stat statbuff; if(stat(file_path, &statbuff) < 0){ return filesize; } else { filesize = statbuff.st_size; } return filesize; } int main(int argc, char *argv[]) { /* 在程序入口调用aos_http_io_initialize方法来初始化网络、内存等全局资源。*/ if (aos_http_io_initialize(NULL, 0) != AOSE_OK) { exit(1); } /* 用于内存管理的内存池(pool),等价于apr_pool_t。其实现代码在apr库中。*/ aos_pool_t *pool; /* 重新创建一个内存池,第二个参数是NULL,表示没有继承其它内存池。*/ aos_pool_create(&pool, NULL); /* 创建并初始化options,该参数包括endpoint、access_key_id、acces_key_secret、is_cname、curl等全局配置信息。*/ oss_request_options_t *oss_client_options; /* 在内存池中分配内存给options。*/ oss_client_options = oss_request_options_create(pool); /* 初始化Client的选项oss_client_options。*/ init_options(oss_client_options); /* 初始化参数。*/ aos_string_t bucket; aos_string_t object; oss_upload_file_t *upload_file = NULL; aos_string_t upload_id; aos_table_t *headers = NULL; aos_table_t *complete_headers = NULL; aos_table_t *resp_headers = NULL; aos_status_t *resp_status = NULL; aos_str_set(&bucket, bucket_name); aos_str_set(&object, object_name); aos_str_null(&upload_id); headers = aos_table_make(pool, 1); complete_headers = aos_table_make(pool, 1); int part_num = 1; /* 初始化分片上传,获取一个上传ID(upload_id)。*/ resp_status = oss_init_multipart_upload(oss_client_options, &bucket, &object, &upload_id, headers, &resp_headers); /* 判断分片上传初始化是否成功。*/ if (aos_status_is_ok(resp_status)) { printf("Init multipart upload succeeded, upload_id:%.*s\n", upload_id.len, upload_id.data); } else { printf("Init multipart upload failed, upload_id:%.*s\n", upload_id.len, upload_id.data); } /* 上传分片。*/ int64_t file_length = 0; int64_t pos = 0; aos_list_t complete_part_list; oss_complete_part_content_t* complete_content = NULL; char* part_num_str = NULL; char* etag = NULL; aos_list_init(&complete_part_list); file_length = get_file_size(local_filename); while(pos < file_length) { upload_file = oss_create_upload_file(pool); aos_str_set(&upload_file->filename, local_filename); upload_file->file_pos = pos; pos += 100 * 1024; upload_file->file_last = pos < file_length ? pos : file_length; resp_status = oss_upload_part_from_file(oss_client_options, &bucket, &object, &upload_id, part_num++, upload_file, &resp_headers); /* 保存分片号和ETag。*/ complete_content = oss_create_complete_part_content(pool); part_num_str = apr_psprintf(pool, "%d", part_num-1); aos_str_set(&complete_content->part_number, part_num_str); etag = apr_pstrdup(pool, (char*)apr_table_get(resp_headers, "ETag")); aos_str_set(&complete_content->etag, etag); aos_list_add_tail(&complete_content->node, &complete_part_list); if (aos_status_is_ok(resp_status)) { printf("Multipart upload part from file succeeded\n"); } else { printf("Multipart upload part from file failed\n"); } } /* 完成分片上传。*/ resp_status = oss_complete_multipart_upload(oss_client_options, &bucket, &object, &upload_id, &complete_part_list, complete_headers, &resp_headers); /* 判断分片上传是否完成。*/ if (aos_status_is_ok(resp_status)) { printf("Complete multipart upload from file succeeded, upload_id:%.*s\n", upload_id.len, upload_id.data); } else { printf("Complete multipart upload from file failed\n"); } /* 释放内存池,相当于释放了请求过程中各资源分配的内存。*/ aos_pool_destroy(pool); /* 释放之前分配的全局资源。*/ aos_http_io_deinitialize(); return 0; }
获取已经上传的分片,调用oss_complete_multipart_upload接口时需要每个分片的ETag值。您可以通过以下两种方式获取ETag值:
- 上传每个分片时,返回结果中会包含这个分片的ETag值,供您直接保存使用。
- 通过调用oss_list_upload_part接口获取已经上传的分片的ETag值。上述示例采用的此种方式。
取消分片上传事件
以下代码用于取消分片上传事件。
#include "oss_api.h" #include "aos_http_io.h" const char *endpoint = "<yourEndpoint>"; const char *access_key_id = "<yourAccessKeyId>"; const char *access_key_secret = "<yourAccessKeySecret>"; const char *bucket_name = "<yourBucketName>"; const char *object_name = "<yourObjectName>"; void init_options(oss_request_options_t *options) { options->config = oss_config_create(options->pool); /* 用char*类型的字符串初始化aos_string_t类型。*/ aos_str_set(&options->config->endpoint, endpoint); aos_str_set(&options->config->access_key_id, access_key_id); aos_str_set(&options->config->access_key_secret, access_key_secret); /* 是否使用CNAME访问OSS服务。0表示不使用。*/ options->config->is_cname = 0; /* 设置网络相关参数,比如超时时间等。*/ options->ctl = aos_http_controller_create(options->pool, 0); } int main(int argc, char *argv[]) { /* 在程序入口调用aos_http_io_initialize方法来初始化网络、内存等全局资源。*/ if (aos_http_io_initialize(NULL, 0) != AOSE_OK) { exit(1); } /* 用于内存管理的内存池(pool),等价于apr_pool_t。其实现代码在apr库中。*/ aos_pool_t *pool; /* 重新创建一个内存池,第二个参数是NULL,表示没有继承其它内存池。*/ aos_pool_create(&pool, NULL); /* 创建并初始化options,该参数包括endpoint、access_key_id、acces_key_secret、is_cname、curl等全局配置信息。*/ oss_request_options_t *oss_client_options; /* 在内存池中分配内存给options。*/ oss_client_options = oss_request_options_create(pool); /* 初始化Client的选项oss_client_options。*/ init_options(oss_client_options); /* 初始化参数。*/ aos_string_t bucket; aos_string_t object; aos_string_t upload_id; aos_table_t *headers = NULL; aos_table_t *resp_headers = NULL; aos_status_t *resp_status = NULL; aos_str_set(&bucket, bucket_name); aos_str_set(&object, object_name); aos_str_null(&upload_id); /* 初始化分片上传,获取一个上传ID(upload_id)。*/ resp_status = oss_init_multipart_upload(oss_client_options, &bucket, &object, &upload_id, headers, &resp_headers); /* 判断是否初始化分片上传成功。 */ if (aos_status_is_ok(resp_status)) { printf("Init multipart upload succeeded, upload_id:%.*s\n", upload_id.len, upload_id.data); } else { printf("Init multipart upload failed, upload_id:%.*s\n", upload_id.len, upload_id.data); } /* 取消这次分片上传。*/ resp_status = oss_abort_multipart_upload(oss_client_options, &bucket, &object, &upload_id, &resp_headers); /* 判断取消分片上传是否成功。*/ if (aos_status_is_ok(resp_status)) { printf("Abort multipart upload succeeded, upload_id::%.*s\n", upload_id.len, upload_id.data); } else { printf("Abort multipart upload failed\n"); } /* 释放内存池,相当于释放了请求过程中各资源分配的内存。*/ aos_pool_destroy(pool); /* 释放之前分配的全局资源。*/ aos_http_io_deinitialize(); return 0; }
说明 当一个分片上传事件被取消后,就不能再使用upload_id做任何操作,已经上传的分片数据也会被删除。