From 293884b31b03dd27111612e58dbe99d67ca2c81b Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 17 Jun 2017 18:39:57 +0200 Subject: [PATCH] advio: support resumable up and downloads --- lib/advio.c | 152 ++++++++++++++++++++++++++++++++++++--------- tests/unit/advio.c | 49 +++++++++++++++ 2 files changed, 172 insertions(+), 29 deletions(-) diff --git a/lib/advio.c b/lib/advio.c index db715e6b4..503180e08 100644 --- a/lib/advio.c +++ b/lib/advio.c @@ -195,12 +195,20 @@ AFILE * afopen(const char *uri, const char *mode) curl_easy_setopt(af->curl, CURLOPT_USERAGENT, USER_AGENT); curl_easy_setopt(af->curl, CURLOPT_URL, af->uri); curl_easy_setopt(af->curl, CURLOPT_WRITEDATA, af->file); + curl_easy_setopt(af->curl, CURLOPT_READDATA, af->file); + + curl_easy_setopt(af->curl, CURLOPT_DEBUGFUNCTION, advio_trace); + curl_easy_setopt(af->curl, CURLOPT_VERBOSE, 1); + curl_easy_setopt(af->curl, CURLOPT_XFERINFOFUNCTION, advio_xferinfo); curl_easy_setopt(af->curl, CURLOPT_XFERINFODATA, af); - ret = adownload(af); + ret = adownload(af, 0); if (ret) goto out0; + + af->uploaded = 0; + af->downloaded = 0; return af; @@ -227,6 +235,42 @@ int afclose(AFILE *af) return ret; } +int afseek(AFILE *af, long offset, int origin) +{ + long new, cur = aftell(af); + + switch (origin) { + case SEEK_SET: + new = offset; + break; + + case SEEK_END: + fseek(af->file, 0, SEEK_END); + new = aftell(af); + fseek(af->file, cur, SEEK_SET); + break; + + case SEEK_CUR: + new = cur + offset; + break; + + default: + return -1; + } + + if (new < af->uploaded) + af->uploaded = new; + + return fseek(af->file, offset, origin); +} + +void arewind(AFILE *af) +{ + af->uploaded = 0; + + return rewind(af->file); +} + int afflush(AFILE *af) { bool dirty; @@ -237,60 +281,108 @@ int afflush(AFILE *af) dirty = memcmp(hash, af->hash, sizeof(hash)); if (dirty) - return aupload(af); + return aupload(af, 1); return 0; } - -int aupload(AFILE *af) + +int aupload(AFILE *af, int resume) { CURLcode res; - long pos; - int ret; - - ret = fflush(af->file); - if (ret) - return ret; + + long pos, end; + + pos = aftell(af); + fseek(af->file, 0, SEEK_END); + end = aftell(af); + fseek(af->file, 0, SEEK_SET); + + if (resume) { + if (end == af->uploaded) + return 0; + + info("Resume upload of %s from offset %lu", af->uri, af->uploaded); + curl_easy_setopt(af->curl, CURLOPT_RESUME_FROM, af->uploaded); + } + else { + info("Start upload of %s", af->uri); + curl_easy_setopt(af->curl, CURLOPT_RESUME_FROM, 0); + } curl_easy_setopt(af->curl, CURLOPT_UPLOAD, 1L); - curl_easy_setopt(af->curl, CURLOPT_READDATA, af->file); - - pos = ftell(af->file); /* Remember old stream pointer */ - fseek(af->file, 0, SEEK_SET); + curl_easy_setopt(af->curl, CURLOPT_INFILESIZE, end - af->uploaded); curl_easy_setopt(af->curl, CURLOPT_NOPROGRESS, !isatty(fileno(stderr))); res = curl_easy_perform(af->curl); - fprintf(stderr, "\e[2K"); - fflush(stderr); /* do not continue in the same line as the progress bar */ - fseek(af->file, pos, SEEK_SET); /* Restore old stream pointer */ if (res != CURLE_OK) return -1; sha1sum(af->file, af->hash); + + double total_bytes = 0, total_time = 0; + char buf[2][32]; + + curl_easy_getinfo(af->curl, CURLINFO_SIZE_UPLOAD, &total_bytes); + curl_easy_getinfo(af->curl, CURLINFO_TOTAL_TIME, &total_time); + + char *total_bytes_human = advio_human_size(total_bytes, buf[0], sizeof(buf[0])); + char *total_time_human = advio_human_time(total_time, buf[1], sizeof(buf[1])); + + info("Finished uploaded of %s in %s", total_bytes_human, total_time_human); + + af->uploaded += total_bytes; return 0; } -int adownload(AFILE *af) +int adownload(AFILE *af, int resume) { CURLcode res; - long code; + long code, pos; int ret; + + double total_bytes = 0, total_time = 0; + char buf[2][32]; + + pos = aftell(af); - fseek(af->file, 0, SEEK_SET); + if (resume) { + info("Resume download of %s from offset %lu", af->uri, af->downloaded); + + curl_easy_setopt(af->curl, CURLOPT_RESUME_FROM, af->downloaded); + } + else { + info("Start download of %s", af->uri); + + rewind(af->file); + curl_easy_setopt(af->curl, CURLOPT_RESUME_FROM, 0); + } + + curl_easy_setopt(af->curl, CURLOPT_UPLOAD, 0L); curl_easy_setopt(af->curl, CURLOPT_NOPROGRESS, !isatty(fileno(stderr))); res = curl_easy_perform(af->curl); - fprintf(stderr, "\e[2K"); - fflush(stderr); /* do not continue in the same line as the progress bar */ - switch (res) { case CURLE_OK: - curl_easy_getinfo(af->curl, CURLINFO_RESPONSE_CODE, &code); + curl_easy_getinfo(af->curl, CURLINFO_SIZE_DOWNLOAD, &total_bytes); + curl_easy_getinfo(af->curl, CURLINFO_TOTAL_TIME, &total_time); + + char *total_bytes_human = advio_human_size(total_bytes, buf[0], sizeof(buf[0])); + char *total_time_human = advio_human_time(total_time, buf[1], sizeof(buf[1])); + + info("Finished download of %s in %s", total_bytes_human, total_time_human); + + af->downloaded += total_bytes; + af->uploaded = af->downloaded; + + res = curl_easy_getinfo(af->curl, CURLINFO_RESPONSE_CODE, &code); + if (res) + return -1; + switch (code) { case 0: case 200: goto exist; @@ -312,7 +404,7 @@ int adownload(AFILE *af) return -1; default: - error("Failed to fetch file: %s: %s\n", af->uri, curl_easy_strerror(res)); + error("ADVIO: Failed to fetch file: %s: %s", af->uri, curl_easy_strerror(res)); return -1; } @@ -333,11 +425,13 @@ notexist: /* File does not exist */ return ret; exist: /* File exists */ - if (af->mode[0] == 'a') - fseek(af->file, 0, SEEK_END); + if (resume) + afseek(af, pos, SEEK_SET); + else if (af->mode[0] == 'a') + afseek(af, 0, SEEK_END); else if (af->mode[0] == 'r' || af->mode[0] == 'w') - fseek(af->file, 0, SEEK_SET); - + afseek(af, 0, SEEK_SET); + sha1sum(af->file, af->hash); return 0; diff --git a/tests/unit/advio.c b/tests/unit/advio.c index 451b30d61..b882a5f74 100644 --- a/tests/unit/advio.c +++ b/tests/unit/advio.c @@ -68,6 +68,55 @@ Test(advio, download) cr_assert_eq(ret, 0, "Failed to close file"); } +Test(advio, resume) +{ + int ret; + AFILE *af1, *af2; + char *fn, dir[] = "/tmp/temp.XXXXXX"; + char line1[32]; + char *line2 = NULL; + size_t linelen = 0; + + mkdtemp(dir); + ret = asprintf(&fn, "%s/file", dir); + cr_assert_gt(ret, 0); + + af1 = afopen(fn, "w+"); + cr_assert_not_null(af1); + + /* We flush once the empty file in order to upload an empty file. */ + aupload(af1, 0); + + af2 = afopen(fn, "r"); + cr_assert_not_null(af2); + + for (int i = 0; i < 100; i++) { + snprintf(line1, sizeof(line1), "This is line %d\n", i); + + afputs(line1, af1); + aupload(af1, 1); + + adownload(af2, 1); + agetline(&line2, &linelen, af2); + + cr_assert_str_eq(line1, line2); + } + + ret = afclose(af1); + cr_assert_eq(ret, 0); + + ret = afclose(af2); + cr_assert_eq(ret, 0); + + ret = unlink(fn); + cr_assert_eq(ret, 0); + + ret = rmdir(dir); + cr_assert_eq(ret, 0); + + free(line2); +} + Test(advio, upload) { AFILE *af;