mirror of
https://git.rwth-aachen.de/acs/public/villas/node/
synced 2025-03-09 00:00:00 +01:00
added curilo functions to read / write files on libcurl compatible protocols
This commit is contained in:
parent
04b999b2e9
commit
f505e6cb7f
3 changed files with 562 additions and 0 deletions
19
include/villas/curlio.h
Normal file
19
include/villas/curlio.h
Normal file
|
@ -0,0 +1,19 @@
|
|||
|
||||
#ifndef _CURLIO_H_
|
||||
#define _CURLIO_H_
|
||||
|
||||
typedef struct fcurl_data URL_FILE;
|
||||
|
||||
URL_FILE *url_fopen(const char *url, const char *operation);
|
||||
|
||||
int url_fclose(URL_FILE *file);
|
||||
|
||||
int url_feof(URL_FILE *file);
|
||||
|
||||
size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
|
||||
|
||||
char * url_fgets(char *ptr, size_t size, URL_FILE *file);
|
||||
|
||||
void url_rewind(URL_FILE *file);
|
||||
|
||||
#endif /* _CURLIO_H_ */
|
444
lib/curlio.c
Normal file
444
lib/curlio.c
Normal file
|
@ -0,0 +1,444 @@
|
|||
/** libcurl based remote IO
|
||||
*
|
||||
* @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
|
||||
* @copyright 2014-2016, Institute for Automation of Complex Power Systems, EONERC
|
||||
* This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential.
|
||||
* Unauthorized copying of this file, via any medium is strictly prohibited.
|
||||
*********************************************************************************/
|
||||
|
||||
#include "curlio.h"
|
||||
|
||||
/*****************************************************************************
|
||||
*
|
||||
* This example source code introduces a c library buffered I/O interface to
|
||||
* URL reads it supports fopen(), fread(), fgets(), feof(), fclose(),
|
||||
* rewind(). Supported functions have identical prototypes to their normal c
|
||||
* lib namesakes and are preceaded by url_ .
|
||||
*
|
||||
* Using this code you can replace your program's fopen() with url_fopen()
|
||||
* and fread() with url_fread() and it become possible to read remote streams
|
||||
* instead of (only) local files. Local files (ie those that can be directly
|
||||
* fopened) will drop back to using the underlying clib implementations
|
||||
*
|
||||
* See the main() function at the bottom that shows an app that retrives from a
|
||||
* specified url using fgets() and fread() and saves as two output files.
|
||||
*
|
||||
* Copyright (c) 2003 Simtec Electronics
|
||||
*
|
||||
* Re-implemented by Vincent Sanders <vince@kyllikki.org> with extensive
|
||||
* reference to original curl example code
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* 3. The name of the author may not be used to endorse or promote products
|
||||
* derived from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
||||
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
* This example requires libcurl 7.9.7 or later.
|
||||
*/
|
||||
/* <DESC>
|
||||
* implements an fopen() abstraction allowing reading from URLs
|
||||
* </DESC>
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#ifndef WIN32
|
||||
#include <sys/time.h>
|
||||
#endif
|
||||
#include <stdlib.h>
|
||||
#include <errno.h>
|
||||
|
||||
#include <curl/curl.h>
|
||||
|
||||
enum fcurl_type_e {
|
||||
CFTYPE_NONE = 0,
|
||||
CFTYPE_FILE = 1,
|
||||
CFTYPE_CURL = 2
|
||||
};
|
||||
|
||||
struct fcurl_data
|
||||
{
|
||||
enum fcurl_type_e type; /**< Type of handle */
|
||||
union {
|
||||
CURL *curl;
|
||||
FILE *file;
|
||||
} handle; /**< Handle */
|
||||
|
||||
char *buffer; /**< Buffer to store cached data*/
|
||||
size_t buffer_len; /**< Currently allocated buffers length */
|
||||
size_t buffer_pos; /**< End of data in buffer*/
|
||||
int still_running; /**< Is background url fetch still in progress */
|
||||
};
|
||||
|
||||
/* We use a global one for convenience */
|
||||
CURLM *multi_handle;
|
||||
|
||||
/* libcurl calls this routine to get more data */
|
||||
static size_t write_callback(char *buffer, size_t size, size_t nitems, void *userp)
|
||||
{
|
||||
char *newbuff;
|
||||
size_t rembuff;
|
||||
|
||||
URL_FILE *url = (URL_FILE *)userp;
|
||||
size *= nitems;
|
||||
|
||||
rembuff = url->buffer_len - url->buffer_pos; /* Remaining space in buffer */
|
||||
|
||||
if (size > rembuff) {
|
||||
/* Not enough space in buffer */
|
||||
newbuff = realloc(url->buffer, url->buffer_len + (size - rembuff));
|
||||
if (newbuff == NULL) {
|
||||
fprintf(stderr, "callback buffer grow failed\n");
|
||||
size = rembuff;
|
||||
}
|
||||
else {
|
||||
/* Realloc succeeded increase buffer size*/
|
||||
url->buffer_len += size - rembuff;
|
||||
url->buffer = newbuff;
|
||||
}
|
||||
}
|
||||
|
||||
memcpy(&url->buffer[url->buffer_pos], buffer, size);
|
||||
url->buffer_pos += size;
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
/* Use to attempt to fill the read buffer up to requested number of bytes */
|
||||
static int fill_buffer(URL_FILE *file, size_t want)
|
||||
{
|
||||
fd_set fdread;
|
||||
fd_set fdwrite;
|
||||
fd_set fdexcep;
|
||||
struct timeval timeout;
|
||||
int rc;
|
||||
CURLMcode mc; /* curl_multi_fdset() return code */
|
||||
|
||||
/* Only attempt to fill buffer if transactions still running and buffer
|
||||
* doesn't exceed required size already
|
||||
*/
|
||||
if ((!file->still_running) || (file->buffer_pos > want))
|
||||
return 0;
|
||||
|
||||
/* Attempt to fill buffer */
|
||||
do {
|
||||
int maxfd = -1;
|
||||
long curl_timeo = -1;
|
||||
|
||||
FD_ZERO(&fdread);
|
||||
FD_ZERO(&fdwrite);
|
||||
FD_ZERO(&fdexcep);
|
||||
|
||||
/* Set a suitable timeout to fail on */
|
||||
timeout.tv_sec = 60; /* 1 minute */
|
||||
timeout.tv_usec = 0;
|
||||
|
||||
curl_multi_timeout(multi_handle, &curl_timeo);
|
||||
if (curl_timeo >= 0) {
|
||||
timeout.tv_sec = curl_timeo / 1000;
|
||||
if (timeout.tv_sec > 1)
|
||||
timeout.tv_sec = 1;
|
||||
else
|
||||
timeout.tv_usec = (curl_timeo % 1000) * 1000;
|
||||
}
|
||||
|
||||
/* Get file descriptors from the transfers */
|
||||
mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
|
||||
|
||||
if (mc != CURLM_OK) {
|
||||
fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc);
|
||||
break;
|
||||
}
|
||||
|
||||
/* On success the value of maxfd is guaranteed to be >= -1. We call
|
||||
* select(maxfd + 1, ...); specially in case of (maxfd == -1) there are
|
||||
* no fds ready yet so we call select(0, ...) --or Sleep() on Windows--
|
||||
* to sleep 100ms, which is the minimum suggested value in the
|
||||
* curl_multi_fdset() doc. */
|
||||
|
||||
if (maxfd == -1) {
|
||||
#ifdef _WIN32
|
||||
Sleep(100);
|
||||
rc = 0;
|
||||
#else
|
||||
/* Portable sleep for platforms other than Windows. */
|
||||
struct timeval wait = { 0, 100 * 1000 }; /* 100ms */
|
||||
rc = select(0, NULL, NULL, NULL, &wait);
|
||||
#endif
|
||||
}
|
||||
else {
|
||||
/* Note that on some platforms 'timeout' may be modified by select().
|
||||
If you need access to the original value save a copy beforehand. */
|
||||
rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
|
||||
}
|
||||
|
||||
switch (rc) {
|
||||
case -1:
|
||||
/* Select error */
|
||||
break;
|
||||
|
||||
case 0:
|
||||
default:
|
||||
/* Timeout or readable/writable sockets */
|
||||
curl_multi_perform(multi_handle, &file->still_running);
|
||||
break;
|
||||
}
|
||||
} while(file->still_running && (file->buffer_pos < want));
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Use to remove want bytes from the front of a files buffer */
|
||||
static int use_buffer(URL_FILE *file, size_t want)
|
||||
{
|
||||
/* Sort out buffer */
|
||||
if ((file->buffer_pos - want) <= 0) {
|
||||
/* Ditch buffer - write will recreate */
|
||||
free(file->buffer);
|
||||
file->buffer = NULL;
|
||||
file->buffer_pos = 0;
|
||||
file->buffer_len = 0;
|
||||
}
|
||||
else {
|
||||
/* Move rest down make it available for later */
|
||||
memmove(file->buffer, &file->buffer[want], (file->buffer_pos - want));
|
||||
|
||||
file->buffer_pos -= want;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
URL_FILE *url_fopen(const char *url, const char *operation)
|
||||
{
|
||||
/* This code could check for URLs or types in the 'url' and
|
||||
basically use the real fopen() for standard files */
|
||||
|
||||
URL_FILE *file;
|
||||
(void)operation;
|
||||
|
||||
file = malloc(sizeof(URL_FILE));
|
||||
if (!file)
|
||||
return NULL;
|
||||
|
||||
memset(file, 0, sizeof(URL_FILE));
|
||||
|
||||
file->handle.file = fopen(url, operation);
|
||||
if (file->handle.file)
|
||||
file->type = CFTYPE_FILE; /* Marked as standard file */
|
||||
else {
|
||||
file->type = CFTYPE_CURL; /* Marked as URL */
|
||||
file->handle.curl = curl_easy_init();
|
||||
|
||||
curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
|
||||
curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file);
|
||||
curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L);
|
||||
curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
|
||||
|
||||
if (!multi_handle)
|
||||
multi_handle = curl_multi_init();
|
||||
|
||||
curl_multi_add_handle(multi_handle, file->handle.curl);
|
||||
|
||||
/* Lets start the fetch */
|
||||
curl_multi_perform(multi_handle, &file->still_running);
|
||||
|
||||
if ((file->buffer_pos == 0) && (!file->still_running)) {
|
||||
/* If still_running is 0 now, we should return NULL */
|
||||
|
||||
/* Make sure the easy handle is not in the multi handle anymore */
|
||||
curl_multi_remove_handle(multi_handle, file->handle.curl);
|
||||
|
||||
/* Cleanup */
|
||||
curl_easy_cleanup(file->handle.curl);
|
||||
|
||||
free(file);
|
||||
|
||||
file = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
int url_fclose(URL_FILE *file)
|
||||
{
|
||||
int ret = 0; /* Default is good return */
|
||||
|
||||
switch (file->type) {
|
||||
case CFTYPE_FILE:
|
||||
ret = fclose(file->handle.file); /* Passthrough */
|
||||
break;
|
||||
|
||||
case CFTYPE_CURL:
|
||||
/* Make sure the easy handle is not in the multi handle anymore */
|
||||
curl_multi_remove_handle(multi_handle, file->handle.curl);
|
||||
|
||||
/* Cleanup */
|
||||
curl_easy_cleanup(file->handle.curl);
|
||||
break;
|
||||
|
||||
default: /* Unknown or supported type - oh dear */
|
||||
ret = EOF;
|
||||
errno = EBADF;
|
||||
break;
|
||||
}
|
||||
|
||||
free(file->buffer);/* Free any allocated buffer space */
|
||||
free(file);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int url_feof(URL_FILE *file)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
switch (file->type) {
|
||||
case CFTYPE_FILE:
|
||||
ret = feof(file->handle.file);
|
||||
break;
|
||||
|
||||
case CFTYPE_CURL:
|
||||
if ((file->buffer_pos == 0) && (!file->still_running))
|
||||
ret = 1;
|
||||
break;
|
||||
|
||||
default: /* Unknown or supported type - oh dear */
|
||||
ret = -1;
|
||||
errno = EBADF;
|
||||
break;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
|
||||
{
|
||||
size_t want;
|
||||
|
||||
switch (file->type) {
|
||||
case CFTYPE_FILE:
|
||||
want = fread(ptr, size, nmemb, file->handle.file);
|
||||
break;
|
||||
|
||||
case CFTYPE_CURL:
|
||||
want = nmemb * size;
|
||||
|
||||
fill_buffer(file, want);
|
||||
|
||||
/* Check if theres data in the buffer - if not fill_buffer()
|
||||
* either errored or EOF */
|
||||
if (!file->buffer_pos)
|
||||
return 0;
|
||||
|
||||
/* Ensure only available data is considered */
|
||||
if (file->buffer_pos < want)
|
||||
want = file->buffer_pos;
|
||||
|
||||
/* Xfer data to caller */
|
||||
memcpy(ptr, file->buffer, want);
|
||||
|
||||
use_buffer(file, want);
|
||||
|
||||
want = want / size; /* Number of items */
|
||||
break;
|
||||
|
||||
default: /* Unknown or supported type - oh dear */
|
||||
want = 0;
|
||||
errno = EBADF;
|
||||
break;
|
||||
}
|
||||
|
||||
return want;
|
||||
}
|
||||
|
||||
char *url_fgets(char *ptr, size_t size, URL_FILE *file)
|
||||
{
|
||||
size_t want = size - 1; /* Always need to leave room for zero termination */
|
||||
size_t loop;
|
||||
|
||||
switch (file->type) {
|
||||
case CFTYPE_FILE:
|
||||
ptr = fgets(ptr, (int)size, file->handle.file);
|
||||
break;
|
||||
|
||||
case CFTYPE_CURL:
|
||||
fill_buffer(file, want);
|
||||
|
||||
/* Check if theres data in the buffer - if not fill either errored or
|
||||
* EOF */
|
||||
if (!file->buffer_pos)
|
||||
return NULL;
|
||||
|
||||
/* Ensure only available data is considered */
|
||||
if (file->buffer_pos < want)
|
||||
want = file->buffer_pos;
|
||||
|
||||
/* Buffer contains data */
|
||||
/* Look for newline or eof */
|
||||
for (loop = 0; loop < want; loop++) {
|
||||
if (file->buffer[loop] == '\n') {
|
||||
want = loop+1; /* Include newline */
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* Xfer data to caller */
|
||||
memcpy(ptr, file->buffer, want);
|
||||
ptr[want] = 0; /* Allways null terminate */
|
||||
|
||||
use_buffer(file, want);
|
||||
|
||||
break;
|
||||
|
||||
default: /* Unknown or supported type - oh dear */
|
||||
ptr = NULL;
|
||||
errno = EBADF;
|
||||
break;
|
||||
}
|
||||
|
||||
return ptr;/*success */
|
||||
}
|
||||
|
||||
void url_rewind(URL_FILE *file)
|
||||
{
|
||||
switch (file->type) {
|
||||
case CFTYPE_FILE:
|
||||
rewind(file->handle.file); /* Passthrough */
|
||||
break;
|
||||
|
||||
case CFTYPE_CURL:
|
||||
/* Halt transaction */
|
||||
curl_multi_remove_handle(multi_handle, file->handle.curl);
|
||||
|
||||
/* Restart */
|
||||
curl_multi_add_handle(multi_handle, file->handle.curl);
|
||||
|
||||
/* Ditch buffer - write will recreate - resets stream pos */
|
||||
free(file->buffer);
|
||||
file->buffer = NULL;
|
||||
file->buffer_pos = 0;
|
||||
file->buffer_len = 0;
|
||||
|
||||
break;
|
||||
|
||||
default: /* Unknown or supported type - oh dear */
|
||||
break;
|
||||
}
|
||||
}
|
99
tests/curlio.c
Normal file
99
tests/curlio.c
Normal file
|
@ -0,0 +1,99 @@
|
|||
#define FGETSFILE "fgets.test"
|
||||
#define FREADFILE "fread.test"
|
||||
#define REWINDFILE "rewind.test"
|
||||
|
||||
/* Small main program to retrive from a url using fgets and fread saving the
|
||||
* output to two test files (note the fgets method will corrupt binary files if
|
||||
* they contain 0 chars */
|
||||
Test(curlio, main)
|
||||
{
|
||||
URL_FILE *handle;
|
||||
FILE *outf;
|
||||
|
||||
size_t nread;
|
||||
char buffer[256];
|
||||
const char *url;
|
||||
|
||||
if(argc < 2)
|
||||
url="http://192.168.7.3/testfile";/* default to testurl */
|
||||
else
|
||||
url=argv[1];/* use passed url */
|
||||
|
||||
/* copy from url line by line with fgets */
|
||||
outf=fopen(FGETSFILE, "wb+");
|
||||
if(!outf) {
|
||||
perror("couldn't open fgets output file\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
handle = url_fopen(url, "r");
|
||||
if(!handle) {
|
||||
printf("couldn't url_fopen() %s\n", url);
|
||||
fclose(outf);
|
||||
return 2;
|
||||
}
|
||||
|
||||
while(!url_feof(handle)) {
|
||||
url_fgets(buffer, sizeof(buffer), handle);
|
||||
fwrite(buffer, 1, strlen(buffer), outf);
|
||||
}
|
||||
|
||||
url_fclose(handle);
|
||||
|
||||
fclose(outf);
|
||||
|
||||
|
||||
/* Copy from url with fread */
|
||||
outf=fopen(FREADFILE, "wb+");
|
||||
if(!outf) {
|
||||
perror("couldn't open fread output file\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
handle = url_fopen("testfile", "r");
|
||||
if(!handle) {
|
||||
printf("couldn't url_fopen() testfile\n");
|
||||
fclose(outf);
|
||||
return 2;
|
||||
}
|
||||
|
||||
do {
|
||||
nread = url_fread(buffer, 1, sizeof(buffer), handle);
|
||||
fwrite(buffer, 1, nread, outf);
|
||||
} while(nread);
|
||||
|
||||
url_fclose(handle);
|
||||
|
||||
fclose(outf);
|
||||
|
||||
|
||||
/* Test rewind */
|
||||
outf=fopen(REWINDFILE, "wb+");
|
||||
if(!outf) {
|
||||
perror("couldn't open fread output file\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
handle = url_fopen("testfile", "r");
|
||||
if(!handle) {
|
||||
printf("couldn't url_fopen() testfile\n");
|
||||
fclose(outf);
|
||||
return 2;
|
||||
}
|
||||
|
||||
nread = url_fread(buffer, 1, sizeof(buffer), handle);
|
||||
fwrite(buffer, 1, nread, outf);
|
||||
url_rewind(handle);
|
||||
|
||||
buffer[0]='\n';
|
||||
fwrite(buffer, 1, 1, outf);
|
||||
|
||||
nread = url_fread(buffer, 1, sizeof(buffer), handle);
|
||||
fwrite(buffer, 1, nread, outf);
|
||||
|
||||
url_fclose(handle);
|
||||
|
||||
fclose(outf);
|
||||
|
||||
return 0;/* all done */
|
||||
}
|
Loading…
Add table
Reference in a new issue