From f505e6cb7f88720935b62f5153f9ddcf714608d5 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Fri, 23 Dec 2016 18:12:48 +0100 Subject: [PATCH] added curilo functions to read / write files on libcurl compatible protocols --- include/villas/curlio.h | 19 ++ lib/curlio.c | 444 ++++++++++++++++++++++++++++++++++++++++ tests/curlio.c | 99 +++++++++ 3 files changed, 562 insertions(+) create mode 100644 include/villas/curlio.h create mode 100644 lib/curlio.c create mode 100644 tests/curlio.c diff --git a/include/villas/curlio.h b/include/villas/curlio.h new file mode 100644 index 000000000..f8fe08666 --- /dev/null +++ b/include/villas/curlio.h @@ -0,0 +1,19 @@ + +#ifndef _CURLIO_H_ +#define _CURLIO_H_ + +typedef struct fcurl_data URL_FILE; + +URL_FILE *url_fopen(const char *url, const char *operation); + +int url_fclose(URL_FILE *file); + +int url_feof(URL_FILE *file); + +size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file); + +char * url_fgets(char *ptr, size_t size, URL_FILE *file); + +void url_rewind(URL_FILE *file); + +#endif /* _CURLIO_H_ */ \ No newline at end of file diff --git a/lib/curlio.c b/lib/curlio.c new file mode 100644 index 000000000..8e8544f26 --- /dev/null +++ b/lib/curlio.c @@ -0,0 +1,444 @@ +/** libcurl based remote IO + * + * @author Steffen Vogel + * @copyright 2014-2016, Institute for Automation of Complex Power Systems, EONERC + * This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential. + * Unauthorized copying of this file, via any medium is strictly prohibited. + *********************************************************************************/ + +#include "curlio.h" + +/***************************************************************************** + * + * This example source code introduces a c library buffered I/O interface to + * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(), + * rewind(). Supported functions have identical prototypes to their normal c + * lib namesakes and are preceaded by url_ . + * + * Using this code you can replace your program's fopen() with url_fopen() + * and fread() with url_fread() and it become possible to read remote streams + * instead of (only) local files. Local files (ie those that can be directly + * fopened) will drop back to using the underlying clib implementations + * + * See the main() function at the bottom that shows an app that retrives from a + * specified url using fgets() and fread() and saves as two output files. + * + * Copyright (c) 2003 Simtec Electronics + * + * Re-implemented by Vincent Sanders with extensive + * reference to original curl example code + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * This example requires libcurl 7.9.7 or later. + */ +/* + * implements an fopen() abstraction allowing reading from URLs + * + */ + +#include +#include +#ifndef WIN32 + #include +#endif +#include +#include + +#include + +enum fcurl_type_e { + CFTYPE_NONE = 0, + CFTYPE_FILE = 1, + CFTYPE_CURL = 2 +}; + +struct fcurl_data +{ + enum fcurl_type_e type; /**< Type of handle */ + union { + CURL *curl; + FILE *file; + } handle; /**< Handle */ + + char *buffer; /**< Buffer to store cached data*/ + size_t buffer_len; /**< Currently allocated buffers length */ + size_t buffer_pos; /**< End of data in buffer*/ + int still_running; /**< Is background url fetch still in progress */ +}; + +/* We use a global one for convenience */ +CURLM *multi_handle; + +/* libcurl calls this routine to get more data */ +static size_t write_callback(char *buffer, size_t size, size_t nitems, void *userp) +{ + char *newbuff; + size_t rembuff; + + URL_FILE *url = (URL_FILE *)userp; + size *= nitems; + + rembuff = url->buffer_len - url->buffer_pos; /* Remaining space in buffer */ + + if (size > rembuff) { + /* Not enough space in buffer */ + newbuff = realloc(url->buffer, url->buffer_len + (size - rembuff)); + if (newbuff == NULL) { + fprintf(stderr, "callback buffer grow failed\n"); + size = rembuff; + } + else { + /* Realloc succeeded increase buffer size*/ + url->buffer_len += size - rembuff; + url->buffer = newbuff; + } + } + + memcpy(&url->buffer[url->buffer_pos], buffer, size); + url->buffer_pos += size; + + return size; +} + +/* Use to attempt to fill the read buffer up to requested number of bytes */ +static int fill_buffer(URL_FILE *file, size_t want) +{ + fd_set fdread; + fd_set fdwrite; + fd_set fdexcep; + struct timeval timeout; + int rc; + CURLMcode mc; /* curl_multi_fdset() return code */ + + /* Only attempt to fill buffer if transactions still running and buffer + * doesn't exceed required size already + */ + if ((!file->still_running) || (file->buffer_pos > want)) + return 0; + + /* Attempt to fill buffer */ + do { + int maxfd = -1; + long curl_timeo = -1; + + FD_ZERO(&fdread); + FD_ZERO(&fdwrite); + FD_ZERO(&fdexcep); + + /* Set a suitable timeout to fail on */ + timeout.tv_sec = 60; /* 1 minute */ + timeout.tv_usec = 0; + + curl_multi_timeout(multi_handle, &curl_timeo); + if (curl_timeo >= 0) { + timeout.tv_sec = curl_timeo / 1000; + if (timeout.tv_sec > 1) + timeout.tv_sec = 1; + else + timeout.tv_usec = (curl_timeo % 1000) * 1000; + } + + /* Get file descriptors from the transfers */ + mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd); + + if (mc != CURLM_OK) { + fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc); + break; + } + + /* On success the value of maxfd is guaranteed to be >= -1. We call + * select(maxfd + 1, ...); specially in case of (maxfd == -1) there are + * no fds ready yet so we call select(0, ...) --or Sleep() on Windows-- + * to sleep 100ms, which is the minimum suggested value in the + * curl_multi_fdset() doc. */ + + if (maxfd == -1) { +#ifdef _WIN32 + Sleep(100); + rc = 0; +#else + /* Portable sleep for platforms other than Windows. */ + struct timeval wait = { 0, 100 * 1000 }; /* 100ms */ + rc = select(0, NULL, NULL, NULL, &wait); +#endif + } + else { + /* Note that on some platforms 'timeout' may be modified by select(). + If you need access to the original value save a copy beforehand. */ + rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout); + } + + switch (rc) { + case -1: + /* Select error */ + break; + + case 0: + default: + /* Timeout or readable/writable sockets */ + curl_multi_perform(multi_handle, &file->still_running); + break; + } + } while(file->still_running && (file->buffer_pos < want)); + return 1; +} + +/* Use to remove want bytes from the front of a files buffer */ +static int use_buffer(URL_FILE *file, size_t want) +{ + /* Sort out buffer */ + if ((file->buffer_pos - want) <= 0) { + /* Ditch buffer - write will recreate */ + free(file->buffer); + file->buffer = NULL; + file->buffer_pos = 0; + file->buffer_len = 0; + } + else { + /* Move rest down make it available for later */ + memmove(file->buffer, &file->buffer[want], (file->buffer_pos - want)); + + file->buffer_pos -= want; + } + return 0; +} + +URL_FILE *url_fopen(const char *url, const char *operation) +{ + /* This code could check for URLs or types in the 'url' and + basically use the real fopen() for standard files */ + + URL_FILE *file; + (void)operation; + + file = malloc(sizeof(URL_FILE)); + if (!file) + return NULL; + + memset(file, 0, sizeof(URL_FILE)); + + file->handle.file = fopen(url, operation); + if (file->handle.file) + file->type = CFTYPE_FILE; /* Marked as standard file */ + else { + file->type = CFTYPE_CURL; /* Marked as URL */ + file->handle.curl = curl_easy_init(); + + curl_easy_setopt(file->handle.curl, CURLOPT_URL, url); + curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file); + curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L); + curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback); + + if (!multi_handle) + multi_handle = curl_multi_init(); + + curl_multi_add_handle(multi_handle, file->handle.curl); + + /* Lets start the fetch */ + curl_multi_perform(multi_handle, &file->still_running); + + if ((file->buffer_pos == 0) && (!file->still_running)) { + /* If still_running is 0 now, we should return NULL */ + + /* Make sure the easy handle is not in the multi handle anymore */ + curl_multi_remove_handle(multi_handle, file->handle.curl); + + /* Cleanup */ + curl_easy_cleanup(file->handle.curl); + + free(file); + + file = NULL; + } + } + + return file; +} + +int url_fclose(URL_FILE *file) +{ + int ret = 0; /* Default is good return */ + + switch (file->type) { + case CFTYPE_FILE: + ret = fclose(file->handle.file); /* Passthrough */ + break; + + case CFTYPE_CURL: + /* Make sure the easy handle is not in the multi handle anymore */ + curl_multi_remove_handle(multi_handle, file->handle.curl); + + /* Cleanup */ + curl_easy_cleanup(file->handle.curl); + break; + + default: /* Unknown or supported type - oh dear */ + ret = EOF; + errno = EBADF; + break; + } + + free(file->buffer);/* Free any allocated buffer space */ + free(file); + + return ret; +} + +int url_feof(URL_FILE *file) +{ + int ret = 0; + + switch (file->type) { + case CFTYPE_FILE: + ret = feof(file->handle.file); + break; + + case CFTYPE_CURL: + if ((file->buffer_pos == 0) && (!file->still_running)) + ret = 1; + break; + + default: /* Unknown or supported type - oh dear */ + ret = -1; + errno = EBADF; + break; + } + + return ret; +} + +size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file) +{ + size_t want; + + switch (file->type) { + case CFTYPE_FILE: + want = fread(ptr, size, nmemb, file->handle.file); + break; + + case CFTYPE_CURL: + want = nmemb * size; + + fill_buffer(file, want); + + /* Check if theres data in the buffer - if not fill_buffer() + * either errored or EOF */ + if (!file->buffer_pos) + return 0; + + /* Ensure only available data is considered */ + if (file->buffer_pos < want) + want = file->buffer_pos; + + /* Xfer data to caller */ + memcpy(ptr, file->buffer, want); + + use_buffer(file, want); + + want = want / size; /* Number of items */ + break; + + default: /* Unknown or supported type - oh dear */ + want = 0; + errno = EBADF; + break; + } + + return want; +} + +char *url_fgets(char *ptr, size_t size, URL_FILE *file) +{ + size_t want = size - 1; /* Always need to leave room for zero termination */ + size_t loop; + + switch (file->type) { + case CFTYPE_FILE: + ptr = fgets(ptr, (int)size, file->handle.file); + break; + + case CFTYPE_CURL: + fill_buffer(file, want); + + /* Check if theres data in the buffer - if not fill either errored or + * EOF */ + if (!file->buffer_pos) + return NULL; + + /* Ensure only available data is considered */ + if (file->buffer_pos < want) + want = file->buffer_pos; + + /* Buffer contains data */ + /* Look for newline or eof */ + for (loop = 0; loop < want; loop++) { + if (file->buffer[loop] == '\n') { + want = loop+1; /* Include newline */ + break; + } + } + + /* Xfer data to caller */ + memcpy(ptr, file->buffer, want); + ptr[want] = 0; /* Allways null terminate */ + + use_buffer(file, want); + + break; + + default: /* Unknown or supported type - oh dear */ + ptr = NULL; + errno = EBADF; + break; + } + + return ptr;/*success */ +} + +void url_rewind(URL_FILE *file) +{ + switch (file->type) { + case CFTYPE_FILE: + rewind(file->handle.file); /* Passthrough */ + break; + + case CFTYPE_CURL: + /* Halt transaction */ + curl_multi_remove_handle(multi_handle, file->handle.curl); + + /* Restart */ + curl_multi_add_handle(multi_handle, file->handle.curl); + + /* Ditch buffer - write will recreate - resets stream pos */ + free(file->buffer); + file->buffer = NULL; + file->buffer_pos = 0; + file->buffer_len = 0; + + break; + + default: /* Unknown or supported type - oh dear */ + break; + } +} diff --git a/tests/curlio.c b/tests/curlio.c new file mode 100644 index 000000000..cd01b50c5 --- /dev/null +++ b/tests/curlio.c @@ -0,0 +1,99 @@ +#define FGETSFILE "fgets.test" +#define FREADFILE "fread.test" +#define REWINDFILE "rewind.test" + +/* Small main program to retrive from a url using fgets and fread saving the + * output to two test files (note the fgets method will corrupt binary files if + * they contain 0 chars */ +Test(curlio, main) +{ + URL_FILE *handle; + FILE *outf; + + size_t nread; + char buffer[256]; + const char *url; + + if(argc < 2) + url="http://192.168.7.3/testfile";/* default to testurl */ + else + url=argv[1];/* use passed url */ + + /* copy from url line by line with fgets */ + outf=fopen(FGETSFILE, "wb+"); + if(!outf) { + perror("couldn't open fgets output file\n"); + return 1; + } + + handle = url_fopen(url, "r"); + if(!handle) { + printf("couldn't url_fopen() %s\n", url); + fclose(outf); + return 2; + } + + while(!url_feof(handle)) { + url_fgets(buffer, sizeof(buffer), handle); + fwrite(buffer, 1, strlen(buffer), outf); + } + + url_fclose(handle); + + fclose(outf); + + + /* Copy from url with fread */ + outf=fopen(FREADFILE, "wb+"); + if(!outf) { + perror("couldn't open fread output file\n"); + return 1; + } + + handle = url_fopen("testfile", "r"); + if(!handle) { + printf("couldn't url_fopen() testfile\n"); + fclose(outf); + return 2; + } + + do { + nread = url_fread(buffer, 1, sizeof(buffer), handle); + fwrite(buffer, 1, nread, outf); + } while(nread); + + url_fclose(handle); + + fclose(outf); + + + /* Test rewind */ + outf=fopen(REWINDFILE, "wb+"); + if(!outf) { + perror("couldn't open fread output file\n"); + return 1; + } + + handle = url_fopen("testfile", "r"); + if(!handle) { + printf("couldn't url_fopen() testfile\n"); + fclose(outf); + return 2; + } + + nread = url_fread(buffer, 1, sizeof(buffer), handle); + fwrite(buffer, 1, nread, outf); + url_rewind(handle); + + buffer[0]='\n'; + fwrite(buffer, 1, 1, outf); + + nread = url_fread(buffer, 1, sizeof(buffer), handle); + fwrite(buffer, 1, nread, outf); + + url_fclose(handle); + + fclose(outf); + + return 0;/* all done */ +} \ No newline at end of file