mirror/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-01-15 19:42:28 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions
mariadb/extra/mariabackup/xbcloud.cc
Sergei Golubchik d046aca0c7 cleanup: CREATE_TYPELIB_FOR() helper
2024-11-05 14:00:47 -08:00

2721 lines
68 KiB
C++
Raw Permalink Blame History

/******************************************************
Copyright (c) 2014 Percona LLC and/or its affiliates.
The xbstream utility: serialize/deserialize files in the XBSTREAM format.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA
*******************************************************/
#include <my_global.h>
#include <my_default.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <curl/curl.h>
#include <ev.h>
#include <unistd.h>
#include <errno.h>
#include <gcrypt.h>
#include <assert.h>
#include <my_sys.h>
#include <my_dir.h>
#include <my_getopt.h>
#include <algorithm>
#include <map>
#include <string>
#include <jsmn.h>
#include "xbstream.h"
using std::min;
using std::max;
using std::map;
using std::string;
#define XBCLOUD_VERSION "1.0"
#define SWIFT_MAX_URL_SIZE 8192
#define SWIFT_MAX_HDR_SIZE 8192
#define SWIFT_CHUNK_SIZE 11 * 1024 * 1024
#if ((LIBCURL_VERSION_MAJOR >= 7) && (LIBCURL_VERSION_MINOR >= 16))
#define OLD_CURL_MULTI 0
#else
#define OLD_CURL_MULTI 1
#endif
/*****************************************************************************/
typedef struct swift_auth_info_struct swift_auth_info;
typedef struct connection_info_struct connection_info;
typedef struct socket_info_struct socket_info;
typedef struct global_io_info_struct global_io_info;
typedef struct slo_chunk_struct slo_chunk;
typedef struct container_list_struct container_list;
typedef struct object_info_struct object_info;
struct swift_auth_info_struct {
char url[SWIFT_MAX_URL_SIZE];
char token[SWIFT_MAX_HDR_SIZE];
};
struct global_io_info_struct {
struct ev_loop *loop;
struct ev_io input_event;
struct ev_timer timer_event;
CURLM *multi;
int still_running;
int eof;
curl_socket_t input_fd;
connection_info **connections;
long chunk_no;
connection_info *current_connection;
const char *url;
const char *container;
const char *token;
const char *backup_name;
};
struct socket_info_struct {
curl_socket_t sockfd;
CURL *easy;
int action;
long timeout;
struct ev_io ev;
int evset;
global_io_info *global;
};
struct connection_info_struct {
CURL *easy;
global_io_info *global;
char *buffer;
size_t buffer_size;
size_t filled_size;
size_t upload_size;
bool chunk_uploaded;
bool chunk_acked;
char error[CURL_ERROR_SIZE];
struct curl_slist *slist;
char *name;
size_t name_len;
char hash[33];
size_t chunk_no;
bool magic_verified;
size_t chunk_path_len;
xb_chunk_type_t chunk_type;
size_t payload_size;
size_t chunk_size;
int retry_count;
bool upload_started;
ulong global_idx;
};
struct slo_chunk_struct {
char name[SWIFT_MAX_URL_SIZE];
char md5[33];
int idx;
size_t size;
};
struct object_info_struct {
char hash[33];
char name[SWIFT_MAX_URL_SIZE];
size_t bytes;
};
struct container_list_struct {
size_t content_length;
size_t content_bufsize;
char *content_json;
size_t object_count;
size_t idx;
object_info *objects;
bool final;
};
enum {SWIFT, S3};
const char *storage_names[] =
{ "SWIFT", "S3", NullS};
static my_bool opt_verbose = 0;
static ulong opt_storage = SWIFT;
static const char *opt_swift_user = NULL;
static const char *opt_swift_user_id = NULL;
static const char *opt_swift_password = NULL;
static const char *opt_swift_tenant = NULL;
static const char *opt_swift_tenant_id = NULL;
static const char *opt_swift_project = NULL;
static const char *opt_swift_project_id = NULL;
static const char *opt_swift_domain = NULL;
static const char *opt_swift_domain_id = NULL;
static const char *opt_swift_region = NULL;
static const char *opt_swift_container = NULL;
static const char *opt_swift_storage_url = NULL;
static const char *opt_swift_auth_url = NULL;
static const char *opt_swift_key = NULL;
static const char *opt_swift_auth_version = NULL;
static const char *opt_name = NULL;
static const char *opt_cacert = NULL;
static ulong opt_parallel = 1;
static my_bool opt_insecure = 0;
static enum {MODE_GET, MODE_PUT, MODE_DELETE} opt_mode;
static char **file_list = NULL;
static int file_list_size = 0;
TYPELIB storage_typelib = CREATE_TYPELIB_FOR(storage_names);
enum {
OPT_STORAGE = 256,
OPT_SWIFT_CONTAINER,
OPT_SWIFT_AUTH_URL,
OPT_SWIFT_KEY,
OPT_SWIFT_USER,
OPT_SWIFT_USER_ID,
OPT_SWIFT_PASSWORD,
OPT_SWIFT_TENANT,
OPT_SWIFT_TENANT_ID,
OPT_SWIFT_PROJECT,
OPT_SWIFT_PROJECT_ID,
OPT_SWIFT_DOMAIN,
OPT_SWIFT_DOMAIN_ID,
OPT_SWIFT_REGION,
OPT_SWIFT_STORAGE_URL,
OPT_SWIFT_AUTH_VERSION,
OPT_PARALLEL,
OPT_CACERT,
OPT_INSECURE,
OPT_VERBOSE
};
static struct my_option my_long_options[] =
{
{"help", '?', "Display this help and exit.",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"storage", OPT_STORAGE, "Specify storage type S3/SWIFT.",
&opt_storage, &opt_storage, &storage_typelib,
GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"swift-auth-version", OPT_SWIFT_AUTH_VERSION,
"Swift authentication verison to use.",
&opt_swift_auth_version, &opt_swift_auth_version, 0,
GET_STR_ALLOC, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"swift-container", OPT_SWIFT_CONTAINER,
"Swift container to store backups into.",
&opt_swift_container, &opt_swift_container, 0,
GET_STR_ALLOC, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"swift-user", OPT_SWIFT_USER,
"Swift user name.",
&opt_swift_user, &opt_swift_user, 0, GET_STR_ALLOC, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"swift-user-id", OPT_SWIFT_USER_ID,
"Swift user ID.",
&opt_swift_user_id, &opt_swift_user_id, 0, GET_STR_ALLOC, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"swift-auth-url", OPT_SWIFT_AUTH_URL,
"Base URL of SWIFT authentication service.",
&opt_swift_auth_url, &opt_swift_auth_url, 0,
GET_STR_ALLOC, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"swift-storage-url", OPT_SWIFT_STORAGE_URL,
"URL of object-store endpoint. Usually received from authentication "
"service. Specify to override this value.",
&opt_swift_storage_url, &opt_swift_storage_url, 0,
GET_STR_ALLOC, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"swift-key", OPT_SWIFT_KEY,
"Swift key.",
&opt_swift_key, &opt_swift_key, 0, GET_STR_ALLOC, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"swift-tenant", OPT_SWIFT_TENANT,
"The tenant name. Both the --swift-tenant and --swift-tenant-id "
"options are optional, but should not be specified together.",
&opt_swift_tenant, &opt_swift_tenant, 0, GET_STR_ALLOC, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"swift-tenant-id", OPT_SWIFT_TENANT_ID,
"The tenant ID. Both the --swift-tenant and --swift-tenant-id "
"options are optional, but should not be specified together.",
&opt_swift_tenant_id, &opt_swift_tenant_id, 0,
GET_STR_ALLOC, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"swift-project", OPT_SWIFT_PROJECT,
"The project name.",
&opt_swift_project, &opt_swift_project, 0, GET_STR_ALLOC, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"swift-project-id", OPT_SWIFT_PROJECT_ID,
"The project ID.",
&opt_swift_project_id, &opt_swift_project_id, 0,
GET_STR_ALLOC, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"swift-domain", OPT_SWIFT_DOMAIN,
"The domain name.",
&opt_swift_domain, &opt_swift_domain, 0, GET_STR_ALLOC, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"swift-domain-id", OPT_SWIFT_DOMAIN_ID,
"The domain ID.",
&opt_swift_domain_id, &opt_swift_domain_id, 0,
GET_STR_ALLOC, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"swift-password", OPT_SWIFT_PASSWORD,
"The password of the user.",
&opt_swift_password, &opt_swift_password, 0,
GET_STR_ALLOC, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"swift-region", OPT_SWIFT_REGION,
"The region object-store endpoint.",
&opt_swift_region, &opt_swift_region, 0,
GET_STR_ALLOC, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"parallel", OPT_PARALLEL,
"Number of parallel chunk uploads.",
&opt_parallel, &opt_parallel, 0, GET_ULONG, REQUIRED_ARG,
1, 0, 0, 0, 0, 0},
{"cacert", OPT_CACERT,
"CA certificate file.",
&opt_cacert, &opt_cacert, 0, GET_STR_ALLOC, REQUIRED_ARG,
0, 0, 0, 0, 0, 0},
{"insecure", OPT_INSECURE,
"Do not verify server SSL certificate.",
&opt_insecure, &opt_insecure, 0, GET_BOOL, NO_ARG,
0, 0, 0, 0, 0, 0},
{"verbose", OPT_VERBOSE,
"Turn ON cURL tracing.",
&opt_verbose, &opt_verbose, 0, GET_BOOL, NO_ARG,
0, 0, 0, 0, 0, 0},
{0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
};
/* The values of these arguments should be masked
on the command line */
static const char * const masked_args[] = {
"--swift-password",
"--swift-key",
"--swift-auth-url",
"--swift-storage-url",
"--swift-container",
"--swift-user",
"--swift-tenant",
"--swift-user-id",
"--swift-tenant-id",
0
};
static map<string, ulonglong> file_chunk_count;
static
void
print_version()
{
printf("%s Ver %s for %s (%s)\n", my_progname, XBCLOUD_VERSION,
SYSTEM_TYPE, MACHINE_TYPE);
}
static
void
usage()
{
print_version();
puts("Copyright (C) 2015 Percona LLC and/or its affiliates.");
puts("This software comes with ABSOLUTELY NO WARRANTY. "
"This is free software,\nand you are welcome to modify and "
"redistribute it under the GPL license.\n");
puts("Manage backups on Cloud services.\n");
puts("Usage: ");
printf(" %s -c put [OPTIONS...] <NAME> upload backup from STDIN into "
"the cloud service with given name.\n", my_progname);
printf(" %s -c get [OPTIONS...] <NAME> [FILES...] stream specified "
"backup or individual files from cloud service into STDOUT.\n",
my_progname);
puts("\nOptions:");
my_print_help(my_long_options);
}
static
my_bool
get_one_option(int optid, const struct my_option *opt __attribute__((unused)),
char *argument __attribute__((unused)))
{
switch (optid) {
case '?':
usage();
exit(0);
}
return(FALSE);
}
static const char *load_default_groups[]=
{ "xbcloud", 0 };
/*********************************************************************//**
mask sensitive values on the command line */
static
void
mask_args(int argc, char **argv)
{
int i;
for (i = 0; i < argc-1; i++) {
int j = 0;
if (argv[i]) while (masked_args[j]) {
char *p;
if ((p = strstr(argv[i], masked_args[j]))) {
p += strlen(masked_args[j]);
while (*p && *p != '=') {
p++;
}
if (*p == '=') {
p++;
while (*p) {
*p++ = 'x';
}
}
}
j++;
}
}
}
static
int parse_args(int argc, char **argv)
{
const char *command;
if (argc < 2) {
fprintf(stderr, "Command isn't specified. "
"Supported commands are put and get\n");
usage();
exit(EXIT_FAILURE);
}
command = argv[1];
argc--; argv++;
if (strcasecmp(command, "put") == 0) {
opt_mode = MODE_PUT;
} else if (strcasecmp(command, "get") == 0) {
opt_mode = MODE_GET;
} else if (strcasecmp(command, "delete") == 0) {
opt_mode = MODE_DELETE;
} else {
fprintf(stderr, "Unknown command %s. "
"Supported commands are put and get\n", command);
usage();
exit(EXIT_FAILURE);
}
load_defaults_or_exit("my", load_default_groups, &argc, &argv);
if (handle_options(&argc, &argv, my_long_options, get_one_option)) {
exit(EXIT_FAILURE);
}
/* make sure name is specified */
if (argc < 1) {
fprintf(stderr, "Backup name is required argument\n");
exit(EXIT_FAILURE);
}
opt_name = argv[0];
argc--; argv++;
/* validate arguments */
if (opt_storage == SWIFT) {
if (opt_swift_user == NULL) {
fprintf(stderr, "Swift user is not specified\n");
exit(EXIT_FAILURE);
}
if (opt_swift_container == NULL) {
fprintf(stderr,
"Swift container is not specified\n");
exit(EXIT_FAILURE);
}
if (opt_swift_auth_url == NULL) {
fprintf(stderr, "Swift auth URL is not specified\n");
exit(EXIT_FAILURE);
}
} else {
fprintf(stderr, "Swift is only supported storage API\n");
}
if (argc > 0) {
file_list = argv;
file_list_size = argc;
}
return(0);
}
static char *hex_md5(const unsigned char *hash, char *out)
{
enum { hash_len = 16 };
char *p;
int i;
for (i = 0, p = out; i < hash_len; i++, p+=2) {
sprintf(p, "%02x", hash[i]);
}
return out;
}
/* If header starts with prefix it's value will be copied into output buffer */
static
int get_http_header(const char *prefix, const char *buffer,
char *out, size_t out_size)
{
const char *beg, *end;
size_t len, prefix_len;
prefix_len = strlen(prefix);
if (strncasecmp(buffer, prefix, prefix_len) == 0) {
beg = buffer + prefix_len;
end = strchr(beg, '\r');
len = min<size_t>(end - beg, out_size - 1);
strncpy(out, beg, len);
out[len] = 0;
return 1;
}
return 0;
}
static
size_t swift_auth_header_read_cb(char *ptr, size_t size, size_t nmemb,
void *data)
{
swift_auth_info *info = (swift_auth_info*)(data);
get_http_header("X-Storage-Url: ", ptr,
info->url, array_elements(info->url));
get_http_header("X-Auth-Token: ", ptr,
info->token, array_elements(info->token));
return nmemb * size;
}
/*********************************************************************//**
Authenticate against Swift TempAuth. Fills swift_auth_info struct.
Uses creadentials privided as global variables.
@returns true if access is granted and token received. */
static
bool
swift_temp_auth(const char *auth_url, swift_auth_info *info)
{
CURL *curl;
CURLcode res;
long http_code;
char *hdr_buf = NULL;
struct curl_slist *slist = NULL;
if (opt_swift_user == NULL) {
fprintf(stderr, "Swift user must be specified for TempAuth.\n");
return(false);
}
if (opt_swift_key == NULL) {
fprintf(stderr, "Swift key must be specified for TempAuth.\n");
return(false);
}
curl = curl_easy_init();
if (curl != NULL) {
hdr_buf = (char *)(calloc(14 + max(strlen(opt_swift_user),
strlen(opt_swift_key)), 1));
if (!hdr_buf) {
res = CURLE_FAILED_INIT;
goto cleanup;
}
sprintf(hdr_buf, "X-Auth-User: %s", opt_swift_user);
slist = curl_slist_append(slist, hdr_buf);
sprintf(hdr_buf, "X-Auth-Key: %s", opt_swift_key);
slist = curl_slist_append(slist, hdr_buf);
curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose);
curl_easy_setopt(curl, CURLOPT_URL, auth_url);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION,
swift_auth_header_read_cb);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, info);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
if (opt_cacert != NULL)
curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert);
if (opt_insecure)
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE);
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr, "error: authentication failed: "
"curl_easy_perform(): %s\n",
curl_easy_strerror(res));
goto cleanup;
}
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
if (http_code != 200 &&
http_code != 204) {
fprintf(stderr, "error: authentication failed "
"with response code: %ld\n", http_code);
res = CURLE_LOGIN_DENIED;
goto cleanup;
}
} else {
res = CURLE_FAILED_INIT;
fprintf(stderr, "error: curl_easy_init() failed\n");
goto cleanup;
}
cleanup:
if (hdr_buf) {
free(hdr_buf);
}
if (slist) {
curl_slist_free_all(slist);
}
if (curl) {
curl_easy_cleanup(curl);
}
if (res == CURLE_OK) {
/* check that we received token and storage URL */
if (*info->url == 0) {
fprintf(stderr, "error: malformed response: "
"X-Storage-Url is missing\n");
return(false);
}
if (*info->token == 0) {
fprintf(stderr, "error: malformed response: "
"X-Auth-Token is missing\n");
return(false);
}
return(true);
}
return(false);
}
static
size_t
write_null_cb(char *buffer, size_t size, size_t nmemb, void *stream)
{
return fwrite(buffer, size, nmemb, stderr);
}
static
size_t
read_null_cb(char *ptr, size_t size, size_t nmemb, void *data)
{
return 0;
}
static
int
swift_create_container(swift_auth_info *info, const char *name)
{
char url[SWIFT_MAX_URL_SIZE];
char auth_token[SWIFT_MAX_HDR_SIZE];
CURLcode res;
long http_code;
CURL *curl;
struct curl_slist *slist = NULL;
snprintf(url, array_elements(url), "%s/%s", info->url, name);
snprintf(auth_token, array_elements(auth_token), "X-Auth-Token: %s",
info->token);
curl = curl_easy_init();
if (curl != NULL) {
slist = curl_slist_append(slist, auth_token);
slist = curl_slist_append(slist, "Content-Length: 0");
curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose);
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_null_cb);
curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_null_cb);
curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0L);
curl_easy_setopt(curl, CURLOPT_PUT, 1L);
if (opt_cacert != NULL)
curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert);
if (opt_insecure)
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE);
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr,
"error: curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
goto cleanup;
}
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
if (http_code != 201 && /* created */
http_code != 202 /* accepted (already exists) */) {
fprintf(stderr, "error: request failed "
"with response code: %ld\n", http_code);
res = CURLE_LOGIN_DENIED;
goto cleanup;
}
} else {
res = CURLE_FAILED_INIT;
fprintf(stderr, "error: curl_easy_init() failed\n");
goto cleanup;
}
cleanup:
if (slist) {
curl_slist_free_all(slist);
}
if (curl) {
curl_easy_cleanup(curl);
}
return res;
}
/*********************************************************************//**
Delete object with given url.
@returns true if object deleted successfully. */
static
bool
swift_delete_object(swift_auth_info *info, const char *url)
{
char auth_token[SWIFT_MAX_HDR_SIZE];
CURLcode res;
long http_code;
CURL *curl;
struct curl_slist *slist = NULL;
bool ret = false;
snprintf(auth_token, array_elements(auth_token), "X-Auth-Token: %s",
info->token);
curl = curl_easy_init();
if (curl != NULL) {
slist = curl_slist_append(slist, auth_token);
curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose);
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
if (opt_cacert != NULL)
curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert);
if (opt_insecure)
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE);
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr,
"error: curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
goto cleanup;
}
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
if (http_code != 200 && /* OK */
http_code != 204 /* no content */) {
fprintf(stderr, "error: request failed "
"with response code: %ld\n", http_code);
goto cleanup;
}
ret = true;
} else {
fprintf(stderr, "error: curl_easy_init() failed\n");
goto cleanup;
}
cleanup:
if (slist) {
curl_slist_free_all(slist);
}
if (curl) {
curl_easy_cleanup(curl);
}
return ret;
}
static int conn_upload_init(connection_info *conn);
static void conn_buffer_updated(connection_info *conn);
static connection_info *conn_new(global_io_info *global, ulong global_idx);
static void conn_cleanup(connection_info *conn);
static void conn_upload_retry(connection_info *conn);
/* Check for completed transfers, and remove their easy handles */
static void check_multi_info(global_io_info *g)
{
char *eff_url;
CURLMsg *msg;
int msgs_left;
connection_info *conn;
CURL *easy;
while ((msg = curl_multi_info_read(g->multi, &msgs_left))) {
if (msg->msg == CURLMSG_DONE) {
easy = msg->easy_handle;
curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn);
curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL,
&eff_url);
curl_multi_remove_handle(g->multi, easy);
curl_easy_cleanup(easy);
conn->easy = NULL;
if (conn->chunk_acked) {
conn->chunk_uploaded = true;
fprintf(stderr, "%s is done\n", conn->hash);
} else {
fprintf(stderr, "error: chunk %zu '%s' %s "
"is not uploaded, but socket closed "
"(%zu bytes of %zu left to upload)\n",
conn->chunk_no,
conn->name,
conn->hash,
conn->chunk_size - conn->upload_size,
conn->chunk_size);
conn_upload_retry(conn);
}
}
}
}
/* Die if we get a bad CURLMcode somewhere */
static void mcode_or_die(const char *where, CURLMcode code)
{
if (code != CURLM_OK)
{
const char *s;
switch (code)
{
case CURLM_BAD_HANDLE:
s = "CURLM_BAD_HANDLE";
break;
case CURLM_BAD_EASY_HANDLE:
s = "CURLM_BAD_EASY_HANDLE";
break;
case CURLM_OUT_OF_MEMORY:
s = "CURLM_OUT_OF_MEMORY";
break;
case CURLM_INTERNAL_ERROR:
s = "CURLM_INTERNAL_ERROR";
break;
case CURLM_UNKNOWN_OPTION:
s = "CURLM_UNKNOWN_OPTION";
break;
case CURLM_LAST:
s = "CURLM_LAST";
break;
default:
s = "CURLM_unknown";
break;
case CURLM_BAD_SOCKET:
s = "CURLM_BAD_SOCKET";
fprintf(stderr, "error: %s returns (%d) %s\n",
where, code, s);
/* ignore this error */
return;
}
fprintf(stderr, "error: %s returns (%d) %s\n",
where, code, s);
assert(0);
}
}
/* Called by libev when we get action on a multi socket */
static void event_cb(EV_P_ struct ev_io *w, int revents)
{
global_io_info *global = (global_io_info*)(w->data);
CURLMcode rc;
#if !(OLD_CURL_MULTI)
int action = (revents & EV_READ ? CURL_POLL_IN : 0) |
(revents & EV_WRITE ? CURL_POLL_OUT : 0);
do {
rc = curl_multi_socket_action(global->multi, w->fd, action,
&global->still_running);
} while (rc == CURLM_CALL_MULTI_PERFORM);
#else
do {
rc = curl_multi_socket(global->multi, w->fd,
&global->still_running);
} while (rc == CURLM_CALL_MULTI_PERFORM);
#endif
mcode_or_die("error: event_cb: curl_multi_socket_action", rc);
check_multi_info(global);
if (global->still_running <= 0) {
ev_timer_stop(global->loop, &global->timer_event);
}
}
static void remsock(curl_socket_t s, socket_info *fdp, global_io_info *global)
{
if (fdp) {
if (fdp->evset) {
ev_io_stop(global->loop, &fdp->ev);
}
free(fdp);
}
}
static void setsock(socket_info *fdp, curl_socket_t s, CURL *easy, int action,
global_io_info *global)
{
int kind = (action & CURL_POLL_IN ? (int)(EV_READ) : 0) |
(action & CURL_POLL_OUT ? (int)(EV_WRITE) : 0);
fdp->sockfd = s;
fdp->action = action;
fdp->easy = easy;
if (fdp->evset)
ev_io_stop(global->loop, &fdp->ev);
ev_io_init(&fdp->ev, event_cb, fdp->sockfd, kind);
fdp->ev.data = global;
fdp->evset = 1;
ev_io_start(global->loop, &fdp->ev);
}
static void addsock(curl_socket_t s, CURL *easy, int action,
global_io_info *global)
{
socket_info *fdp = (socket_info *)(calloc(sizeof(socket_info), 1));
fdp->global = global;
setsock(fdp, s, easy, action, global);
curl_multi_assign(global->multi, s, fdp);
}
static int sock_cb(CURL *easy, curl_socket_t s, int what, void *cbp,
void *sockp)
{
global_io_info *global = (global_io_info*)(cbp);
socket_info *fdp = (socket_info*)(sockp);
if (what == CURL_POLL_REMOVE) {
remsock(s, fdp, global);
} else {
if (!fdp) {
addsock(s, easy, what, global);
} else {
setsock(fdp, s, easy, what, global);
}
}
return 0;
}
/* Called by libev when our timeout expires */
static void timer_cb(EV_P_ struct ev_timer *w, int revents)
{
global_io_info *io_global = (global_io_info*)(w->data);
CURLMcode rc;
#if !(OLD_CURL_MULTI)
do {
rc = curl_multi_socket_action(io_global->multi,
CURL_SOCKET_TIMEOUT, 0,
&io_global->still_running);
} while (rc == CURLM_CALL_MULTI_PERFORM);
#else
do {
rc = curl_multi_socket_all(io_global->multi,
&io_global->still_running);
} while (rc == CURLM_CALL_MULTI_PERFORM);
#endif
mcode_or_die("timer_cb: curl_multi_socket_action", rc);
check_multi_info(io_global);
}
static connection_info *get_current_connection(global_io_info *global)
{
connection_info *conn = global->current_connection;
ulong i;
if (conn && conn->filled_size < conn->chunk_size)
return conn;
for (i = 0; i < opt_parallel; i++) {
conn = global->connections[i];
if (conn->chunk_uploaded || conn->filled_size == 0) {
global->current_connection = conn;
conn_upload_init(conn);
return conn;
}
}
return NULL;
}
/* This gets called whenever data is received from the input */
static void input_cb(EV_P_ struct ev_io *w, int revents)
{
global_io_info *io_global = (global_io_info *)(w->data);
connection_info *conn = get_current_connection(io_global);
if (conn == NULL)
return;
if (conn->filled_size < conn->chunk_size) {
if (revents & EV_READ) {
ssize_t nbytes = read(io_global->input_fd,
conn->buffer + conn->filled_size,
conn->chunk_size -
conn->filled_size);
if (nbytes > 0) {
conn->filled_size += nbytes;
conn_buffer_updated(conn);
} else if (nbytes < 0) {
if (errno != EAGAIN && errno != EINTR) {
char error[200];
my_strerror(error, sizeof(error),
errno);
fprintf(stderr, "error: failed to read "
"input stream (%s)\n", error);
/* failed to read input */
exit(1);
}
} else {
io_global->eof = 1;
ev_io_stop(io_global->loop, w);
}
}
}
assert(conn->filled_size <= conn->chunk_size);
}
static int swift_upload_read_cb(char *ptr, size_t size, size_t nmemb,
void *data)
{
size_t realsize;
connection_info *conn = (connection_info*)(data);
if (conn->filled_size == conn->upload_size &&
conn->upload_size < conn->chunk_size && !conn->global->eof) {
ssize_t nbytes;
assert(conn->global->current_connection == conn);
do {
nbytes = read(conn->global->input_fd,
conn->buffer + conn->filled_size,
conn->chunk_size - conn->filled_size);
} while (nbytes == -1 && errno == EAGAIN);
if (nbytes > 0) {
conn->filled_size += nbytes;
conn_buffer_updated(conn);
} else {
conn->global->eof = 1;
}
}
realsize = min(size * nmemb, conn->filled_size - conn->upload_size);
memcpy(ptr, conn->buffer + conn->upload_size, realsize);
conn->upload_size += realsize;
assert(conn->filled_size <= conn->chunk_size);
assert(conn->upload_size <= conn->filled_size);
return realsize;
}
static
size_t upload_header_read_cb(char *ptr, size_t size, size_t nmemb,
void *data)
{
connection_info *conn = (connection_info *)(data);
char etag[33];
if (get_http_header("Etag: ", ptr, etag, array_elements(etag))) {
if (strcmp(conn->hash, etag) != 0) {
fprintf(stderr, "error: ETag mismatch\n");
exit(EXIT_FAILURE);
}
fprintf(stderr, "acked chunk %s\n", etag);
conn->chunk_acked = true;
}
return nmemb * size;
}
static int conn_upload_init(connection_info *conn)
{
conn->filled_size = 0;
conn->upload_size = 0;
conn->chunk_uploaded = false;
conn->chunk_acked = false;
conn->chunk_size = CHUNK_HEADER_CONSTANT_LEN;
conn->magic_verified = false;
conn->chunk_path_len = 0;
conn->chunk_type = XB_CHUNK_TYPE_UNKNOWN;
conn->payload_size = 0;
conn->upload_started = false;
conn->retry_count = 0;
if (conn->name != NULL) {
conn->name[0] = 0;
}
if (conn->easy != NULL) {
conn->easy = 0;
}
if (conn->slist != NULL) {
curl_slist_free_all(conn->slist);
conn->slist = NULL;
}
return 0;
}
static void conn_upload_prepare(connection_info *conn)
{
gcry_md_hd_t md5;
gcry_md_open(&md5, GCRY_MD_MD5, 0);
gcry_md_write(md5, conn->buffer, conn->chunk_size);
hex_md5(gcry_md_read(md5, GCRY_MD_MD5), conn->hash);
gcry_md_close(md5);
}
static int conn_upload_start(connection_info *conn)
{
char token_header[SWIFT_MAX_HDR_SIZE];
char object_url[SWIFT_MAX_URL_SIZE];
char content_len[200], etag[200];
global_io_info *global;
CURLMcode rc;
global = conn->global;
fprintf(stderr, "uploading chunk %s/%s/%s.%020zu "
"(md5: %s, size: %zu)\n",
global->container, global->backup_name, conn->name,
conn->chunk_no, conn->hash, conn->chunk_size);
snprintf(object_url, array_elements(object_url), "%s/%s/%s/%s.%020zu",
global->url, global->container, global->backup_name,
conn->name, conn->chunk_no);
snprintf(content_len, sizeof(content_len), "Content-Length: %lu",
(ulong)(conn->chunk_size));
snprintf(etag, sizeof(etag), "ETag: %s", conn->hash);
snprintf(token_header, array_elements(token_header),
"X-Auth-Token: %s", global->token);
conn->slist = curl_slist_append(conn->slist, token_header);
conn->slist = curl_slist_append(conn->slist,
"Connection: keep-alive");
conn->slist = curl_slist_append(conn->slist,
"Content-Type: "
"application/octet-stream");
conn->slist = curl_slist_append(conn->slist, content_len);
conn->slist = curl_slist_append(conn->slist, etag);
conn->easy = curl_easy_init();
if (!conn->easy) {
fprintf(stderr, "error: curl_easy_init() failed\n");
return 1;
}
curl_easy_setopt(conn->easy, CURLOPT_URL, object_url);
curl_easy_setopt(conn->easy, CURLOPT_READFUNCTION,
swift_upload_read_cb);
curl_easy_setopt(conn->easy, CURLOPT_READDATA, conn);
curl_easy_setopt(conn->easy, CURLOPT_VERBOSE, opt_verbose);
curl_easy_setopt(conn->easy, CURLOPT_ERRORBUFFER, conn->error);
curl_easy_setopt(conn->easy, CURLOPT_PRIVATE, conn);
curl_easy_setopt(conn->easy, CURLOPT_NOPROGRESS, 1L);
curl_easy_setopt(conn->easy, CURLOPT_LOW_SPEED_TIME, 5L);
curl_easy_setopt(conn->easy, CURLOPT_LOW_SPEED_LIMIT, 1024L);
curl_easy_setopt(conn->easy, CURLOPT_PUT, 1L);
curl_easy_setopt(conn->easy, CURLOPT_HTTPHEADER, conn->slist);
curl_easy_setopt(conn->easy, CURLOPT_HEADERFUNCTION,
upload_header_read_cb);
curl_easy_setopt(conn->easy, CURLOPT_HEADERDATA, conn);
curl_easy_setopt(conn->easy, CURLOPT_INFILESIZE,
(long) conn->chunk_size);
if (opt_cacert != NULL)
curl_easy_setopt(conn->easy, CURLOPT_CAINFO, opt_cacert);
if (opt_insecure)
curl_easy_setopt(conn->easy, CURLOPT_SSL_VERIFYPEER, FALSE);
rc = curl_multi_add_handle(conn->global->multi, conn->easy);
mcode_or_die("conn_upload_init: curl_multi_add_handle", rc);
#if (OLD_CURL_MULTI)
do {
rc = curl_multi_socket_all(global->multi,
&global->still_running);
} while(rc == CURLM_CALL_MULTI_PERFORM);
#endif
conn->upload_started = true;
return 0;
}
static void conn_cleanup(connection_info *conn)
{
if (conn) {
free(conn->name);
free(conn->buffer);
if (conn->slist) {
curl_slist_free_all(conn->slist);
conn->slist = NULL;
}
if (conn->easy) {
curl_easy_cleanup(conn->easy);
conn->easy = NULL;
}
}
free(conn);
}
static void conn_upload_retry(connection_info *conn)
{
/* already closed by cURL */
conn->easy = NULL;
if (conn->slist != NULL) {
curl_slist_free_all(conn->slist);
conn->slist = NULL;
}
if (conn->retry_count++ > 3) {
fprintf(stderr, "error: retry count limit reached\n");
exit(EXIT_FAILURE);
}
fprintf(stderr, "warning: retrying to upload chunk %zu of '%s'\n",
conn->chunk_no, conn->name);
conn->upload_size = 0;
conn_upload_start(conn);
}
static connection_info *conn_new(global_io_info *global, ulong global_idx)
{
connection_info *conn;
conn = (connection_info *)(calloc(1, sizeof(connection_info)));
if (conn == NULL) {
goto error;
}
conn->global = global;
conn->global_idx = global_idx;
conn->buffer_size = SWIFT_CHUNK_SIZE;
if ((conn->buffer = (char *)(calloc(conn->buffer_size, 1))) ==
NULL) {
goto error;
}
return conn;
error:
if (conn != NULL) {
conn_cleanup(conn);
}
fprintf(stderr, "error: out of memory\n");
exit(EXIT_FAILURE);
return NULL;
}
/*********************************************************************//**
Handle input buffer updates. Parse chunk header and set appropriate
buffer size. */
static
void
conn_buffer_updated(connection_info *conn)
{
bool ready_for_upload = false;
/* chunk header */
if (!conn->magic_verified &&
conn->filled_size >= CHUNK_HEADER_CONSTANT_LEN) {
if (strncmp(XB_STREAM_CHUNK_MAGIC, conn->buffer,
sizeof(XB_STREAM_CHUNK_MAGIC) - 1) != 0) {
fprintf(stderr, "Error: magic expected\n");
exit(EXIT_FAILURE);
}
conn->magic_verified = true;
conn->chunk_path_len = uint4korr(conn->buffer
+ PATH_LENGTH_OFFSET);
conn->chunk_type = (xb_chunk_type_t)
(conn->buffer[CHUNK_TYPE_OFFSET]);
conn->chunk_size = CHUNK_HEADER_CONSTANT_LEN +
conn->chunk_path_len;
if (conn->chunk_type != XB_CHUNK_TYPE_EOF) {
conn->chunk_size += 16;
}
}
/* ordinary chunk */
if (conn->magic_verified &&
conn->payload_size == 0 &&
conn->chunk_type != XB_CHUNK_TYPE_EOF &&
conn->filled_size >= CHUNK_HEADER_CONSTANT_LEN
+ conn->chunk_path_len + 16) {
conn->payload_size = uint8korr(conn->buffer +
CHUNK_HEADER_CONSTANT_LEN +
conn->chunk_path_len);
conn->chunk_size = conn->payload_size + 4 + 16 +
conn->chunk_path_len +
CHUNK_HEADER_CONSTANT_LEN;
if (conn->name == NULL) {
conn->name = (char*)(malloc(conn->chunk_path_len + 1));
} else if (conn->name_len < conn->chunk_path_len + 1) {
conn->name = (char*)(realloc(conn->name,
conn->chunk_path_len + 1));
}
conn->name_len = conn->chunk_path_len + 1;
memcpy(conn->name, conn->buffer + CHUNK_HEADER_CONSTANT_LEN,
conn->chunk_path_len);
conn->name[conn->chunk_path_len] = 0;
if (conn->buffer_size < conn->chunk_size) {
conn->buffer =
(char *)(realloc(conn->buffer, conn->chunk_size));
conn->buffer_size = conn->chunk_size;
}
}
/* EOF chunk has no payload */
if (conn->magic_verified &&
conn->chunk_type == XB_CHUNK_TYPE_EOF &&
conn->filled_size >= CHUNK_HEADER_CONSTANT_LEN
+ conn->chunk_path_len) {
if (conn->name == NULL) {
conn->name = (char*)(malloc(conn->chunk_path_len + 1));
} else if (conn->name_len < conn->chunk_path_len + 1) {
conn->name = (char*)(realloc(conn->name,
conn->chunk_path_len + 1));
}
conn->name_len = conn->chunk_path_len + 1;
memcpy(conn->name, conn->buffer + CHUNK_HEADER_CONSTANT_LEN,
conn->chunk_path_len);
conn->name[conn->chunk_path_len] = 0;
}
if (conn->filled_size > 0 && conn->filled_size == conn->chunk_size) {
ready_for_upload = true;
}
/* start upload once recieved the size of the chunk */
if (!conn->upload_started && ready_for_upload) {
conn->chunk_no = file_chunk_count[conn->name]++;
conn_upload_prepare(conn);
conn_upload_start(conn);
}
}
static int init_input(global_io_info *io_global)
{
ev_io_init(&io_global->input_event, input_cb, STDIN_FILENO, EV_READ);
io_global->input_event.data = io_global;
ev_io_start(io_global->loop, &io_global->input_event);
return 0;
}
/* Update the event timer after curl_multi library calls */
static int multi_timer_cb(CURLM *multi, long timeout_ms, global_io_info *global)
{
ev_timer_stop(global->loop, &global->timer_event);
if (timeout_ms > 0) {
double t = timeout_ms / 1000.0;
ev_timer_init(&global->timer_event, timer_cb, t, 0.);
ev_timer_start(global->loop, &global->timer_event);
} else {
timer_cb(global->loop, &global->timer_event, 0);
}
return 0;
}
static
int swift_upload_parts(swift_auth_info *auth, const char *container,
const char *name)
{
global_io_info io_global;
ulong i;
#if (OLD_CURL_MULTI)
long timeout;
#endif
CURLMcode rc;
int n_dirty_buffers;
memset(&io_global, 0, sizeof(io_global));
io_global.loop = ev_default_loop(0);
init_input(&io_global);
io_global.multi = curl_multi_init();
ev_timer_init(&io_global.timer_event, timer_cb, 0., 0.);
io_global.timer_event.data = &io_global;
io_global.connections = (connection_info **)
(calloc(opt_parallel, sizeof(connection_info)));
io_global.url = auth->url;
io_global.container = container;
io_global.backup_name = name;
io_global.token = auth->token;
for (i = 0; i < opt_parallel; i++) {
io_global.connections[i] = conn_new(&io_global, i);
}
/* setup the generic multi interface options we want */
curl_multi_setopt(io_global.multi, CURLMOPT_SOCKETFUNCTION, sock_cb);
curl_multi_setopt(io_global.multi, CURLMOPT_SOCKETDATA, &io_global);
#if !(OLD_CURL_MULTI)
curl_multi_setopt(io_global.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb);
curl_multi_setopt(io_global.multi, CURLMOPT_TIMERDATA, &io_global);
do {
rc = curl_multi_socket_action(io_global.multi,
CURL_SOCKET_TIMEOUT, 0,
&io_global.still_running);
} while (rc == CURLM_CALL_MULTI_PERFORM);
#else
curl_multi_timeout(io_global.multi, &timeout);
if (timeout >= 0) {
multi_timer_cb(io_global.multi, timeout, &io_global);
}
do {
rc = curl_multi_socket_all(io_global.multi, &io_global.still_running);
} while(rc == CURLM_CALL_MULTI_PERFORM);
#endif
ev_loop(io_global.loop, 0);
check_multi_info(&io_global);
curl_multi_cleanup(io_global.multi);
n_dirty_buffers = 0;
for (i = 0; i < opt_parallel; i++) {
connection_info *conn = io_global.connections[i];
if (conn && conn->upload_size != conn->filled_size) {
fprintf(stderr, "error: upload failed: %lu bytes left "
"in the buffer %s (uploaded = %d)\n",
(ulong)(conn->filled_size - conn->upload_size),
conn->name, conn->chunk_uploaded);
++n_dirty_buffers;
}
}
for (i = 0; i < opt_parallel; i++) {
if (io_global.connections[i] != NULL) {
conn_cleanup(io_global.connections[i]);
}
}
free(io_global.connections);
if (n_dirty_buffers > 0) {
return(EXIT_FAILURE);
}
return 0;
}
struct download_buffer_info {
off_t offset;
size_t size;
size_t result_len;
char *buf;
curl_read_callback custom_header_callback;
void *custom_header_callback_data;
};
/*********************************************************************//**
Callback to parse header of GET request on swift contaier. */
static
size_t fetch_buffer_header_cb(char *ptr, size_t size, size_t nmemb,
void *data)
{
download_buffer_info *buffer_info = (download_buffer_info*)(data);
size_t buf_size;
char content_length_str[100];
char *endptr;
if (get_http_header("Content-Length: ", ptr,
content_length_str, sizeof(content_length_str))) {
buf_size = strtoull(content_length_str, &endptr, 10);
if (buffer_info->buf == NULL) {
buffer_info->buf = (char*)(malloc(buf_size));
buffer_info->size = buf_size;
}
if (buf_size > buffer_info->size) {
buffer_info->buf = (char*)
(realloc(buffer_info->buf, buf_size));
buffer_info->size = buf_size;
}
buffer_info->result_len = buf_size;
}
if (buffer_info->custom_header_callback) {
buffer_info->custom_header_callback(ptr, size, nmemb,
buffer_info->custom_header_callback_data);
}
return nmemb * size;
}
/*********************************************************************//**
Write contents into string buffer */
static
size_t
fetch_buffer_cb(char *buffer, size_t size, size_t nmemb, void *out_buffer)
{
download_buffer_info *buffer_info = (download_buffer_info*)(out_buffer);
assert(buffer_info->size >= buffer_info->offset + size * nmemb);
memcpy(buffer_info->buf + buffer_info->offset, buffer, size * nmemb);
buffer_info->offset += size * nmemb;
return size * nmemb;
}
/*********************************************************************//**
Downloads contents of URL into buffer. Caller is responsible for
deallocating the buffer.
@return pointer to a buffer or NULL */
static
char *
swift_fetch_into_buffer(swift_auth_info *auth, const char *url,
char **buf, size_t *buf_size, size_t *result_len,
curl_read_callback header_callback,
void *header_callback_data)
{
char auth_token[SWIFT_MAX_HDR_SIZE];
download_buffer_info buffer_info;
struct curl_slist *slist = NULL;
long http_code;
CURL *curl;
CURLcode res;
memset(&buffer_info, 0, sizeof(buffer_info));
buffer_info.buf = *buf;
buffer_info.size = *buf_size;
buffer_info.custom_header_callback = header_callback;
buffer_info.custom_header_callback_data = header_callback_data;
snprintf(auth_token, array_elements(auth_token), "X-Auth-Token: %s",
auth->token);
curl = curl_easy_init();
if (curl != NULL) {
slist = curl_slist_append(slist, auth_token);
curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose);
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fetch_buffer_cb);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer_info);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION,
fetch_buffer_header_cb);
curl_easy_setopt(curl, CURLOPT_HEADERDATA,
&buffer_info);
if (opt_cacert != NULL)
curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert);
if (opt_insecure)
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE);
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr,
"error: curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
goto cleanup;
}
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
if (http_code < 200 || http_code >= 300) {
fprintf(stderr, "error: request failed "
"with response code: %ld\n", http_code);
res = CURLE_LOGIN_DENIED;
goto cleanup;
}
} else {
res = CURLE_FAILED_INIT;
fprintf(stderr, "error: curl_easy_init() failed\n");
goto cleanup;
}
cleanup:
if (slist) {
curl_slist_free_all(slist);
}
if (curl) {
curl_easy_cleanup(curl);
}
if (res == CURLE_OK) {
*buf = buffer_info.buf;
*buf_size = buffer_info.size;
*result_len = buffer_info.result_len;
return(buffer_info.buf);
}
free(buffer_info.buf);
*buf = NULL;
*buf_size = 0;
*result_len = 0;
return(NULL);
}
static
container_list *
container_list_new()
{
container_list *list =
(container_list *)(calloc(1, sizeof(container_list)));
list->object_count = 1000;
list->objects = (object_info*)
(calloc(list->object_count, sizeof(object_info)));
if (list->objects == NULL) {
fprintf(stderr, "error: out of memory\n");
free(list);
return(NULL);
}
return(list);
}
static
void
container_list_free(container_list *list)
{
free(list->content_json);
free(list->objects);
free(list);
}
static
void
container_list_add_object(container_list *list, const char *name,
const char *hash, size_t bytes)
{
const size_t object_count_step = 1000;
if (list->idx >= list->object_count) {
list->objects = (object_info*)
realloc(list->objects,
(list->object_count + object_count_step) *
sizeof(object_info));
memset(list->objects + list->object_count, 0,
object_count_step * sizeof(object_info));
list->object_count += object_count_step;
}
assert(list->idx <= list->object_count);
safe_strcpy(list->objects[list->idx].name,
sizeof(list->objects[list->idx].name), name);
safe_strcpy(list->objects[list->idx].hash,
sizeof(list->objects[list->idx].hash), hash);
list->objects[list->idx].bytes = bytes;
++list->idx;
}
/*********************************************************************//**
Tokenize json string. Return array of tokens. Caller is responsoble for
deallocating the array. */
jsmntok_t *
json_tokenise(char *json, size_t len, int initial_tokens)
{
jsmn_parser parser;
jsmn_init(&parser);
unsigned int n = initial_tokens;
jsmntok_t *tokens = (jsmntok_t *)(malloc(sizeof(jsmntok_t) * n));
int ret = jsmn_parse(&parser, json, len, tokens, n);
while (ret == JSMN_ERROR_NOMEM)
{
n = n * 2 + 1;
tokens = (jsmntok_t*)(realloc(tokens, sizeof(jsmntok_t) * n));
ret = jsmn_parse(&parser, json, len, tokens, n);
}
if (ret == JSMN_ERROR_INVAL) {
fprintf(stderr, "error: invalid JSON string\n");
}
if (ret == JSMN_ERROR_PART) {
fprintf(stderr, "error: truncated JSON string\n");
}
return tokens;
}
/*********************************************************************//**
Return true if token representation equal to given string. */
static
bool
json_token_eq(const char *buf, jsmntok_t *t, const char *s)
{
size_t len = strlen(s);
assert(t->end > t->start);
return((size_t)(t->end - t->start) == len &&
(strncmp(buf + t->start, s, len) == 0));
}
/*********************************************************************//**
Copy given token as string. */
static
bool
json_token_str(const char *buf, jsmntok_t *t, char *out, int out_size)
{
size_t len = min(t->end - t->start, out_size - 1);
memcpy(out, buf + t->start, len);
out[len] = 0;
return(true);
}
/*********************************************************************//**
Parse SWIFT container list response and fill output array with values
sorted by object name. */
static
bool
swift_parse_container_list(container_list *list)
{
enum {MAX_DEPTH=20};
enum label_t {NONE, OBJECT};
char name[SWIFT_MAX_URL_SIZE];
char hash[33];
char bytes[30];
char *response = list->content_json;
struct stack_t {
jsmntok_t *t;
int n_items;
label_t label;
};
stack_t stack[MAX_DEPTH];
jsmntok_t *tokens;
int level;
size_t count = 0;
tokens = json_tokenise(list->content_json, list->content_length, 200);
stack[0].t = &tokens[0];
stack[0].label = NONE;
stack[0].n_items = 1;
level = 0;
for (size_t i = 0, j = 1; j > 0; i++, j--) {
jsmntok_t *t = &tokens[i];
assert(t->start != -1 && t->end != -1);
assert(level >= 0);
--stack[level].n_items;
switch (t->type) {
case JSMN_ARRAY:
case JSMN_OBJECT:
if (level < MAX_DEPTH - 1) {
level++;
}
stack[level].t = t;
stack[level].label = NONE;
if (t->type == JSMN_ARRAY) {
stack[level].n_items = t->size;
j += t->size;
} else {
stack[level].n_items = t->size * 2;
j += t->size * 2;
}
break;
case JSMN_PRIMITIVE:
case JSMN_STRING:
if (stack[level].t->type == JSMN_OBJECT &&
stack[level].n_items % 2 == 1) {
/* key */
if (json_token_eq(response, t, "name")) {
json_token_str(response, &tokens[i + 1],
name, sizeof(name));
}
if (json_token_eq(response, t, "hash")) {
json_token_str(response, &tokens[i + 1],
hash, sizeof(hash));
}
if (json_token_eq(response, t, "bytes")) {
json_token_str(response, &tokens[i + 1],
bytes, sizeof(bytes));
}
}
break;
}
while (stack[level].n_items == 0 && level > 0) {
if (stack[level].t->type == JSMN_OBJECT
&& level == 2) {
char *endptr;
container_list_add_object(list, name, hash,
strtoull(bytes, &endptr, 10));
++count;
}
--level;
}
}
if (count == 0) {
list->final = true;
}
free(tokens);
return(true);
}
/*********************************************************************//**
List swift container with given name. Return list of objects sorted by
object name. */
static
container_list *
swift_list(swift_auth_info *auth, const char *container, const char *path)
{
container_list *list;
char url[SWIFT_MAX_URL_SIZE];
list = container_list_new();
while (!list->final) {
/* download the list in json format */
snprintf(url, array_elements(url),
"%s/%s?format=json&limit=1000%s%s%s%s",
auth->url, container, path ? "&prefix=" : "",
path ? path : "", list->idx > 0 ? "&marker=" : "",
list->idx > 0 ?
list->objects[list->idx - 1].name : "");
list->content_json = swift_fetch_into_buffer(auth, url,
&list->content_json, &list->content_bufsize,
&list->content_length, NULL, NULL);
if (list->content_json == NULL) {
container_list_free(list);
return(NULL);
}
/* parse downloaded list */
if (!swift_parse_container_list(list)) {
fprintf(stderr, "error: unable to parse "
"container list\n");
container_list_free(list);
return(NULL);
}
}
return(list);
}
/*********************************************************************//**
Return true if chunk is a part of backup with given name. */
static
bool
chunk_belongs_to(const char *chunk_name, const char *backup_name)
{
size_t backup_name_len = strlen(backup_name);
return((strlen(chunk_name) > backup_name_len)
&& (chunk_name[backup_name_len] == '/')
&& strncmp(chunk_name, backup_name, backup_name_len) == 0);
}
/*********************************************************************//**
Return true if chunk is in given list. */
static
bool
chunk_in_list(const char *chunk_name, char **list, int list_size)
{
size_t chunk_name_len;
if (list_size == 0) {
return(true);
}
chunk_name_len = strlen(chunk_name);
if (chunk_name_len < 20) {
return(false);
}
for (int i = 0; i < list_size; i++) {
size_t item_len = strlen(list[i]);
if ((strncmp(chunk_name - item_len + chunk_name_len - 21,
list[i], item_len) == 0)
&& (chunk_name[chunk_name_len - 21] == '.')
&& (chunk_name[chunk_name_len - item_len - 22] == '/')) {
return(true);
}
}
return(false);
}
static
int swift_download(swift_auth_info *auth, const char *container,
const char *name)
{
container_list *list;
char *buf = NULL;
size_t buf_size = 0;
size_t result_len = 0;
if ((list = swift_list(auth, container, name)) == NULL) {
return(CURLE_FAILED_INIT);
}
for (size_t i = 0; i < list->idx; i++) {
const char *chunk_name = list->objects[i].name;
if (chunk_belongs_to(chunk_name, name)
&& chunk_in_list(chunk_name, file_list, file_list_size)) {
char url[SWIFT_MAX_URL_SIZE];
snprintf(url, sizeof(url), "%s/%s/%s",
auth->url, container, chunk_name);
if ((buf = swift_fetch_into_buffer(
auth, url, &buf, &buf_size, &result_len,
NULL, NULL)) == NULL) {
fprintf(stderr, "error: failed to download "
"chunk %s\n", chunk_name);
container_list_free(list);
return(CURLE_FAILED_INIT);
}
fwrite(buf, 1, result_len, stdout);
}
}
free(buf);
container_list_free(list);
return(CURLE_OK);
}
/*********************************************************************//**
Delete backup with given name from given container.
@return true if backup deleted successfully */
static
bool swift_delete(swift_auth_info *auth, const char *container,
const char *name)
{
container_list *list;
if ((list = swift_list(auth, container, name)) == NULL) {
return(CURLE_FAILED_INIT);
}
for (size_t i = 0; i < list->object_count; i++) {
const char *chunk_name = list->objects[i].name;
if (chunk_belongs_to(chunk_name, name)) {
char url[SWIFT_MAX_URL_SIZE];
snprintf(url, sizeof(url), "%s/%s/%s",
auth->url, container, chunk_name);
fprintf(stderr, "delete %s\n", chunk_name);
if (!swift_delete_object(auth, url)) {
fprintf(stderr, "error: failed to delete "
"chunk %s\n", chunk_name);
container_list_free(list);
return(CURLE_FAILED_INIT);
}
}
}
container_list_free(list);
return(CURLE_OK);
}
/*********************************************************************//**
Check if backup with given name exists.
@return true if backup exists */
static
bool swift_backup_exists(swift_auth_info *auth, const char *container,
const char *backup_name)
{
container_list *list;
if ((list = swift_list(auth, container, backup_name)) == NULL) {
fprintf(stderr, "error: unable to list container %s\n",
container);
exit(EXIT_FAILURE);
}
for (size_t i = 0; i < list->object_count; i++) {
if (chunk_belongs_to(list->objects[i].name, backup_name)) {
container_list_free(list);
return(true);
}
}
container_list_free(list);
return(false);
}
/*********************************************************************//**
Fills auth_info with response from keystone response.
@return true is response parsed successfully */
static
bool
swift_parse_keystone_response_v2(char *response, size_t response_length,
swift_auth_info *auth_info)
{
enum {MAX_DEPTH=20};
enum label_t {NONE, ACCESS, CATALOG, ENDPOINTS, TOKEN};
char filtered_url[SWIFT_MAX_URL_SIZE];
char public_url[SWIFT_MAX_URL_SIZE];
char region[SWIFT_MAX_URL_SIZE];
char id[SWIFT_MAX_URL_SIZE];
char token_id[SWIFT_MAX_URL_SIZE];
char type[SWIFT_MAX_URL_SIZE];
struct stack_t {
jsmntok_t *t;
int n_items;
label_t label;
};
stack_t stack[MAX_DEPTH];
jsmntok_t *tokens;
int level;
tokens = json_tokenise(response, response_length, 200);
stack[0].t = &tokens[0];
stack[0].label = NONE;
stack[0].n_items = 1;
level = 0;
for (size_t i = 0, j = 1; j > 0; i++, j--) {
jsmntok_t *t = &tokens[i];
assert(t->start != -1 && t->end != -1);
assert(level >= 0);
--stack[level].n_items;
switch (t->type) {
case JSMN_ARRAY:
case JSMN_OBJECT:
if (level < MAX_DEPTH - 1) {
level++;
}
stack[level].t = t;
stack[level].label = NONE;
if (t->type == JSMN_ARRAY) {
stack[level].n_items = t->size;
j += t->size;
} else {
stack[level].n_items = t->size * 2;
j += t->size * 2;
}
break;
case JSMN_PRIMITIVE:
case JSMN_STRING:
if (stack[level].t->type == JSMN_OBJECT &&
stack[level].n_items % 2 == 1) {
/* key */
if (json_token_eq(response, t, "access")) {
stack[level].label = ACCESS;
}
if (json_token_eq(response, t,
"serviceCatalog")) {
stack[level].label = CATALOG;
}
if (json_token_eq(response, t, "endpoints")) {
stack[level].label = ENDPOINTS;
}
if (json_token_eq(response, t, "token")) {
stack[level].label = TOKEN;
}
if (json_token_eq(response, t, "id")) {
json_token_str(response, &tokens[i + 1],
id, sizeof(id));
}
if (json_token_eq(response, t, "id")
&& stack[level - 1].label == TOKEN) {
json_token_str(response, &tokens[i + 1],
token_id, sizeof(token_id));
}
if (json_token_eq(response, t, "region")) {
json_token_str(response, &tokens[i + 1],
region, sizeof(region));
}
if (json_token_eq(response, t, "publicURL")) {
json_token_str(response, &tokens[i + 1],
public_url, sizeof(public_url));
}
if (json_token_eq(response, t, "type")) {
json_token_str(response, &tokens[i + 1],
type, sizeof(type));
}
}
break;
}
while (stack[level].n_items == 0 && level > 0) {
if (stack[level].t->type == JSMN_OBJECT
&& level == 6
&& stack[level - 1].t->type == JSMN_ARRAY
&& stack[level - 2].label == ENDPOINTS) {
if (opt_swift_region == NULL
|| strcmp(opt_swift_region, region) == 0) {
strncpy(filtered_url, public_url,
sizeof(filtered_url));
}
}
if (stack[level].t->type == JSMN_OBJECT &&
level == 4 &&
stack[level - 1].t->type == JSMN_ARRAY &&
stack[level - 2].label == CATALOG) {
if (strcmp(type, "object-store") == 0) {
strncpy(auth_info->url, filtered_url,
sizeof(auth_info->url));
}
}
--level;
}
}
free(tokens);
strncpy(auth_info->token, token_id, sizeof(auth_info->token));
assert(level == 0);
if (*auth_info->token == 0) {
fprintf(stderr, "error: can not receive token from response\n");
return(false);
}
if (*auth_info->url == 0) {
fprintf(stderr, "error: can not get URL from response\n");
return(false);
}
return(true);
}
/*********************************************************************//**
Authenticate against Swift TempAuth. Fills swift_auth_info struct.
Uses creadentials privided as global variables.
@returns true if access is granted and token received. */
static
bool
swift_keystone_auth_v2(const char *auth_url, swift_auth_info *info)
{
char tenant_arg[SWIFT_MAX_URL_SIZE];
char payload[SWIFT_MAX_URL_SIZE];
struct curl_slist *slist = NULL;
download_buffer_info buf_info;
long http_code;
CURLcode res;
CURL *curl;
bool auth_res = false;
memset(&buf_info, 0, sizeof(buf_info));
if (opt_swift_user == NULL) {
fprintf(stderr, "error: both --swift-user is required "
"for keystone authentication.\n");
return(false);
}
if (opt_swift_password == NULL) {
fprintf(stderr, "error: both --swift-password is required "
"for keystone authentication.\n");
return(false);
}
if (opt_swift_tenant != NULL && opt_swift_tenant_id != NULL) {
fprintf(stderr, "error: both --swift-tenant and "
"--swift-tenant-id specified for keystone "
"authentication.\n");
return(false);
}
if (opt_swift_tenant != NULL) {
snprintf(tenant_arg, sizeof(tenant_arg), ",\"%s\":\"%s\"",
"tenantName", opt_swift_tenant);
} else if (opt_swift_tenant_id != NULL) {
snprintf(tenant_arg, sizeof(tenant_arg), ",\"%s\":\"%s\"",
"tenantId", opt_swift_tenant_id);
} else {
*tenant_arg = 0;
}
snprintf(payload, sizeof(payload), "{\"auth\": "
"{\"passwordCredentials\": {\"username\":\"%s\","
"\"password\":\"%s\"}%s}}",
opt_swift_user, opt_swift_password, tenant_arg);
curl = curl_easy_init();
if (curl != NULL) {
slist = curl_slist_append(slist,
"Content-Type: application/json");
slist = curl_slist_append(slist,
"Accept: application/json");
curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose);
curl_easy_setopt(curl, CURLOPT_POST, 1L);
curl_easy_setopt(curl, CURLOPT_URL, auth_url);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fetch_buffer_cb);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buf_info);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION,
fetch_buffer_header_cb);
curl_easy_setopt(curl, CURLOPT_HEADERDATA,
&buf_info);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
if (opt_cacert != NULL)
curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert);
if (opt_insecure)
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE);
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr,
"error: curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
goto cleanup;
}
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
if (http_code < 200 || http_code >= 300) {
fprintf(stderr, "error: request failed "
"with response code: %ld\n", http_code);
res = CURLE_LOGIN_DENIED;
goto cleanup;
}
} else {
res = CURLE_FAILED_INIT;
fprintf(stderr, "error: curl_easy_init() failed\n");
goto cleanup;
}
if (!swift_parse_keystone_response_v2(buf_info.buf,
buf_info.size, info)) {
goto cleanup;
}
auth_res = true;
cleanup:
if (slist) {
curl_slist_free_all(slist);
}
if (curl) {
curl_easy_cleanup(curl);
}
free(buf_info.buf);
return(auth_res);
}
/*********************************************************************//**
Fills auth_info with response from keystone response.
@return true is response parsed successfully */
static
bool
swift_parse_keystone_response_v3(char *response, size_t response_length,
swift_auth_info *auth_info)
{
enum {MAX_DEPTH=20};
enum label_t {NONE, TOKEN, CATALOG, ENDPOINTS};
char url[SWIFT_MAX_URL_SIZE];
char filtered_url[SWIFT_MAX_URL_SIZE];
char region[SWIFT_MAX_URL_SIZE];
char interface[SWIFT_MAX_URL_SIZE];
char type[SWIFT_MAX_URL_SIZE];
struct stack_t {
jsmntok_t *t;
int n_items;
label_t label;
};
stack_t stack[MAX_DEPTH];
jsmntok_t *tokens;
int level;
tokens = json_tokenise(response, response_length, 200);
stack[0].t = &tokens[0];
stack[0].label = NONE;
stack[0].n_items = 1;
level = 0;
for (size_t i = 0, j = 1; j > 0; i++, j--) {
jsmntok_t *t = &tokens[i];
assert(t->start != -1 && t->end != -1);
assert(level >= 0);
--stack[level].n_items;
switch (t->type) {
case JSMN_ARRAY:
case JSMN_OBJECT:
if (level < MAX_DEPTH - 1) {
level++;
}
stack[level].t = t;
stack[level].label = NONE;
if (t->type == JSMN_ARRAY) {
stack[level].n_items = t->size;
j += t->size;
} else {
stack[level].n_items = t->size * 2;
j += t->size * 2;
}
break;
case JSMN_PRIMITIVE:
case JSMN_STRING:
if (stack[level].t->type == JSMN_OBJECT &&
stack[level].n_items % 2 == 1) {
/* key */
if (json_token_eq(response, t, "token")) {
stack[level].label = TOKEN;
fprintf(stderr, "token\n");
}
if (json_token_eq(response, t,
"catalog")) {
stack[level].label = CATALOG;
fprintf(stderr, "catalog\n");
}
if (json_token_eq(response, t, "endpoints")) {
stack[level].label = ENDPOINTS;
}
if (json_token_eq(response, t, "region")) {
json_token_str(response, &tokens[i + 1],
region, sizeof(region));
}
if (json_token_eq(response, t, "url")) {
json_token_str(response, &tokens[i + 1],
url, sizeof(url));
}
if (json_token_eq(response, t, "interface")) {
json_token_str(response, &tokens[i + 1],
interface, sizeof(interface));
}
if (json_token_eq(response, t, "type")) {
json_token_str(response, &tokens[i + 1],
type, sizeof(type));
}
}
break;
}
while (stack[level].n_items == 0 && level > 0) {
if (stack[level].t->type == JSMN_OBJECT
&& level == 6
&& stack[level - 1].t->type == JSMN_ARRAY
&& stack[level - 2].label == ENDPOINTS) {
if ((opt_swift_region == NULL
|| strcmp(opt_swift_region, region) == 0)
&& strcmp(interface, "public") == 0) {
strncpy(filtered_url, url,
sizeof(filtered_url));
}
}
if (stack[level].t->type == JSMN_OBJECT &&
level == 4 &&
stack[level - 1].t->type == JSMN_ARRAY &&
stack[level - 2].label == CATALOG) {
if (strcmp(type, "object-store") == 0) {
strncpy(auth_info->url, filtered_url,
sizeof(auth_info->url));
}
}
--level;
}
}
free(tokens);
assert(level == 0);
if (*auth_info->url == 0) {
fprintf(stderr, "error: can not get URL from response\n");
return(false);
}
return(true);
}
/*********************************************************************//**
Captures X-Subject-Token header. */
static
size_t keystone_v3_header_cb(char *ptr, size_t size, size_t nmemb, void *data)
{
swift_auth_info *info = (swift_auth_info*)(data);
get_http_header("X-Subject-Token: ", ptr,
info->token, array_elements(info->token));
return nmemb * size;
}
/*********************************************************************//**
Authenticate against Swift TempAuth. Fills swift_auth_info struct.
Uses creadentials privided as global variables.
@returns true if access is granted and token received. */
static
bool
swift_keystone_auth_v3(const char *auth_url, swift_auth_info *info)
{
char scope[SWIFT_MAX_URL_SIZE];
char domain[SWIFT_MAX_URL_SIZE];
char payload[SWIFT_MAX_URL_SIZE];
struct curl_slist *slist = NULL;
download_buffer_info buf_info;
long http_code;
CURLcode res;
CURL *curl;
bool auth_res = false;
memset(&buf_info, 0, sizeof(buf_info));
buf_info.custom_header_callback = keystone_v3_header_cb;
buf_info.custom_header_callback_data = info;
if (opt_swift_user == NULL) {
fprintf(stderr, "error: both --swift-user is required "
"for keystone authentication.\n");
return(false);
}
if (opt_swift_password == NULL) {
fprintf(stderr, "error: both --swift-password is required "
"for keystone authentication.\n");
return(false);
}
if (opt_swift_project_id != NULL && opt_swift_project != NULL) {
fprintf(stderr, "error: both --swift-project and "
"--swift-project-id specified for keystone "
"authentication.\n");
return(false);
}
if (opt_swift_domain_id != NULL && opt_swift_domain != NULL) {
fprintf(stderr, "error: both --swift-domain and "
"--swift-domain-id specified for keystone "
"authentication.\n");
return(false);
}
if (opt_swift_project_id != NULL && opt_swift_domain != NULL) {
fprintf(stderr, "error: both --swift-project-id and "
"--swift-domain specified for keystone "
"authentication.\n");
return(false);
}
if (opt_swift_project_id != NULL && opt_swift_domain_id != NULL) {
fprintf(stderr, "error: both --swift-project-id and "
"--swift-domain-id specified for keystone "
"authentication.\n");
return(false);
}
scope[0] = 0; domain[0] = 0;
if (opt_swift_domain != NULL) {
snprintf(domain, sizeof(domain),
",{\"domain\":{\"name\":\"%s\"}}",
opt_swift_domain);
} else if (opt_swift_domain_id != NULL) {
snprintf(domain, sizeof(domain),
",{\"domain\":{\"id\":\"%s\"}}",
opt_swift_domain_id);
}
if (opt_swift_project_id != NULL) {
snprintf(scope, sizeof(scope),
",\"scope\":{\"project\":{\"id\":\"%s\"}}",
opt_swift_project_id);
} else if (opt_swift_project != NULL) {
snprintf(scope, sizeof(scope),
",\"scope\":{\"project\":{\"name\":\"%s\"%s}}",
opt_swift_project, domain);
}
snprintf(payload, sizeof(payload), "{\"auth\":{\"identity\":"
"{\"methods\":[\"password\"],\"password\":{\"user\":"
"{\"name\":\"%s\",\"password\":\"%s\"%s}}}%s}}",
opt_swift_user, opt_swift_password,
*scope ? "" : ",\"domain\":{\"id\":\"default\"}",
scope);
curl = curl_easy_init();
if (curl != NULL) {
slist = curl_slist_append(slist,
"Content-Type: application/json");
slist = curl_slist_append(slist,
"Accept: application/json");
curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose);
curl_easy_setopt(curl, CURLOPT_POST, 1L);
curl_easy_setopt(curl, CURLOPT_URL, auth_url);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fetch_buffer_cb);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buf_info);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION,
fetch_buffer_header_cb);
curl_easy_setopt(curl, CURLOPT_HEADERDATA,
&buf_info);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
if (opt_cacert != NULL)
curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert);
if (opt_insecure)
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE);
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr,
"error: curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
goto cleanup;
}
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
if (http_code < 200 || http_code >= 300) {
fprintf(stderr, "error: request failed "
"with response code: %ld\n", http_code);
res = CURLE_LOGIN_DENIED;
goto cleanup;
}
} else {
res = CURLE_FAILED_INIT;
fprintf(stderr, "error: curl_easy_init() failed\n");
goto cleanup;
}
if (!swift_parse_keystone_response_v3(buf_info.buf,
buf_info.size, info)) {
goto cleanup;
}
auth_res = true;
cleanup:
if (slist) {
curl_slist_free_all(slist);
}
if (curl) {
curl_easy_cleanup(curl);
}
free(buf_info.buf);
return(auth_res);
}
int main(int argc, char **argv)
{
swift_auth_info info;
char auth_url[SWIFT_MAX_URL_SIZE];
MY_INIT(argv[0]);
/* handle_options in parse_args is destructive so
* we make a copy of our argument pointers so we can
* mask the sensitive values afterwards */
char **mask_argv = (char **)malloc(sizeof(char *) * (argc - 1));
memcpy(mask_argv, argv + 1, sizeof(char *) * (argc - 1));
if (parse_args(argc, argv)) {
return(EXIT_FAILURE);
}
mask_args(argc, mask_argv); /* mask args on cmdline */
curl_global_init(CURL_GLOBAL_ALL);
if (opt_swift_auth_version == NULL || *opt_swift_auth_version == '1') {
/* TempAuth */
snprintf(auth_url, SWIFT_MAX_URL_SIZE, "%sauth/v%s/",
opt_swift_auth_url, opt_swift_auth_version ?
opt_swift_auth_version : "1.0");
if (!swift_temp_auth(auth_url, &info)) {
fprintf(stderr, "error: failed to authenticate\n");
return(EXIT_FAILURE);
}
} else if (*opt_swift_auth_version == '2') {
/* Keystone v2 */
snprintf(auth_url, SWIFT_MAX_URL_SIZE, "%sv%s/tokens",
opt_swift_auth_url, opt_swift_auth_version);
if (!swift_keystone_auth_v2(auth_url, &info)) {
fprintf(stderr, "error: failed to authenticate\n");
return(EXIT_FAILURE);
}
} else if (*opt_swift_auth_version == '3') {
/* Keystone v3 */
snprintf(auth_url, SWIFT_MAX_URL_SIZE, "%sv%s/auth/tokens",
opt_swift_auth_url, opt_swift_auth_version);
if (!swift_keystone_auth_v3(auth_url, &info)) {
fprintf(stderr, "error: failed to authenticate\n");
exit(EXIT_FAILURE);
}
}
if (opt_swift_storage_url != NULL) {
snprintf(info.url, sizeof(info.url), "%s",
opt_swift_storage_url);
}
fprintf(stderr, "Object store URL: %s\n", info.url);
if (opt_mode == MODE_PUT) {
if (swift_create_container(&info, opt_swift_container) != 0) {
fprintf(stderr, "error: failed to create "
"container %s\n",
opt_swift_container);
return(EXIT_FAILURE);
}
if (swift_backup_exists(&info, opt_swift_container, opt_name)) {
fprintf(stderr, "error: backup named '%s' "
"already exists!\n",
opt_name);
return(EXIT_FAILURE);
}
if (swift_upload_parts(&info, opt_swift_container,
opt_name) != 0) {
fprintf(stderr, "error: upload failed\n");
return(EXIT_FAILURE);
}
} else if (opt_mode == MODE_GET) {
if (swift_download(&info, opt_swift_container, opt_name)
!= CURLE_OK) {
fprintf(stderr, "error: download failed\n");
return(EXIT_FAILURE);
}
} else if (opt_mode == MODE_DELETE) {
if (swift_delete(&info, opt_swift_container, opt_name)
!= CURLE_OK) {
fprintf(stderr, "error: delete failed\n");
return(EXIT_FAILURE);
}
} else {
fprintf(stderr, "Unknown command supplied.\n");
exit(EXIT_FAILURE);
}
curl_global_cleanup();
return(EXIT_SUCCESS);
}
Reference in a new issue View git blame Copy permalink