added curilo functions to read / write files on libcurl compatible protocols

include/villas/curlio.h

@@ -0,0 +1,19 @@
+
+#ifndef _CURLIO_H_
+#define _CURLIO_H_
+
+typedef struct fcurl_data URL_FILE;
+
+URL_FILE *url_fopen(const char *url, const char *operation);
+
+int url_fclose(URL_FILE *file);
+
+int url_feof(URL_FILE *file);
+
+size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
+
+char * url_fgets(char *ptr, size_t size, URL_FILE *file);
+
+void url_rewind(URL_FILE *file);
+
+#endif /* _CURLIO_H_ */

lib/curlio.c

@@ -0,0 +1,444 @@
+/** libcurl based remote IO
+ *
+ * @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
+ * @copyright 2014-2016, Institute for Automation of Complex Power Systems, EONERC
+ *	 This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential.
+ *	 Unauthorized copying of this file, via any medium is strictly prohibited.
+ *********************************************************************************/
+
+#include "curlio.h"
+
+/*****************************************************************************
+ *
+ * This example source code introduces a c library buffered I/O interface to
+ * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(),
+ * rewind(). Supported functions have identical prototypes to their normal c
+ * lib namesakes and are preceaded by url_ .
+ *
+ * Using this code you can replace your program's fopen() with url_fopen()
+ * and fread() with url_fread() and it become possible to read remote streams
+ * instead of (only) local files. Local files (ie those that can be directly
+ * fopened) will drop back to using the underlying clib implementations
+ *
+ * See the main() function at the bottom that shows an app that retrives from a
+ * specified url using fgets() and fread() and saves as two output files.
+ *
+ * Copyright (c) 2003 Simtec Electronics
+ *
+ * Re-implemented by Vincent Sanders <vince@kyllikki.org> with extensive
+ * reference to original curl example code
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *		notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *		notice, this list of conditions and the following disclaimer in the
+ *		documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *		derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This example requires libcurl 7.9.7 or later.
+ */
+/* <DESC>
+ * implements an fopen() abstraction allowing reading from URLs
+ * </DESC>
+ */
+
+#include <stdio.h>
+#include <string.h>
+#ifndef WIN32
+  #include <sys/time.h>
+#endif
+#include <stdlib.h>
+#include <errno.h>
+
+#include <curl/curl.h>
+
+enum fcurl_type_e {
+	CFTYPE_NONE = 0,
+	CFTYPE_FILE = 1,
+	CFTYPE_CURL = 2
+};
+
+struct fcurl_data
+{
+	enum fcurl_type_e type;	 /**< Type of handle */
+	union {
+		CURL *curl;
+		FILE *file;
+	} handle;		/**< Handle */
+
+	char *buffer;		/**< Buffer to store cached data*/
+	size_t buffer_len;	/**< Currently allocated buffers length */
+	size_t buffer_pos;	/**< End of data in buffer*/
+	int still_running;	/**< Is background url fetch still in progress */
+};
+
+/* We use a global one for convenience */
+CURLM *multi_handle;
+
+/* libcurl calls this routine to get more data */
+static size_t write_callback(char *buffer, size_t size, size_t nitems, void *userp)
+{
+	char *newbuff;
+	size_t rembuff;
+
+	URL_FILE *url = (URL_FILE *)userp;
+	size *= nitems;
+
+	rembuff = url->buffer_len - url->buffer_pos; /* Remaining space in buffer */
+
+	if (size > rembuff) {
+		/* Not enough space in buffer */
+		newbuff = realloc(url->buffer, url->buffer_len + (size - rembuff));
+		if (newbuff == NULL) {
+			fprintf(stderr, "callback buffer grow failed\n");
+			size = rembuff;
+		}
+		else {
+			/* Realloc succeeded increase buffer size*/
+			url->buffer_len += size - rembuff;
+			url->buffer = newbuff;
+		}
+	}
+
+	memcpy(&url->buffer[url->buffer_pos], buffer, size);
+	url->buffer_pos += size;
+
+	return size;
+}
+
+/* Use to attempt to fill the read buffer up to requested number of bytes */
+static int fill_buffer(URL_FILE *file, size_t want)
+{
+	fd_set fdread;
+	fd_set fdwrite;
+	fd_set fdexcep;
+	struct timeval timeout;
+	int rc;
+	CURLMcode mc; /* curl_multi_fdset() return code */
+
+	/* Only attempt to fill buffer if transactions still running and buffer
+	 * doesn't exceed required size already
+	 */
+	if ((!file->still_running) || (file->buffer_pos > want))
+		return 0;
+
+	/* Attempt to fill buffer */
+	do {
+		int maxfd = -1;
+		long curl_timeo = -1;
+
+		FD_ZERO(&fdread);
+		FD_ZERO(&fdwrite);
+		FD_ZERO(&fdexcep);
+
+		/* Set a suitable timeout to fail on */
+		timeout.tv_sec = 60; /* 1 minute */
+		timeout.tv_usec = 0;
+
+		curl_multi_timeout(multi_handle, &curl_timeo);
+		if (curl_timeo >= 0) {
+			timeout.tv_sec = curl_timeo / 1000;
+			if (timeout.tv_sec > 1)
+				timeout.tv_sec = 1;
+			else
+				timeout.tv_usec = (curl_timeo % 1000) * 1000;
+		}
+
+		/* Get file descriptors from the transfers */
+		mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
+
+		if (mc != CURLM_OK) {
+			fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc);
+			break;
+		}
+
+		/* On success the value of maxfd is guaranteed to be >= -1. We call
+		 * select(maxfd + 1, ...); specially in case of (maxfd == -1) there are
+		 * no fds ready yet so we call select(0, ...) --or Sleep() on Windows--
+		 * to sleep 100ms, which is the minimum suggested value in the
+		 * curl_multi_fdset() doc. */
+
+		if (maxfd == -1) {
+#ifdef _WIN32
+			Sleep(100);
+			rc = 0;
+#else
+			/* Portable sleep for platforms other than Windows. */
+			struct timeval wait = { 0, 100 * 1000 }; /* 100ms */
+			rc = select(0, NULL, NULL, NULL, &wait);
+#endif
+		}
+		else {
+			/* Note that on some platforms 'timeout' may be modified by select().
+				 If you need access to the original value save a copy beforehand. */
+			rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
+		}
+
+		switch (rc) {
+			case -1:
+				/* Select error */
+				break;
+
+			case 0:
+			default:
+				/* Timeout or readable/writable sockets */
+				curl_multi_perform(multi_handle, &file->still_running);
+				break;
+		}
+	} while(file->still_running && (file->buffer_pos < want));
+	return 1;
+}
+
+/* Use to remove want bytes from the front of a files buffer */
+static int use_buffer(URL_FILE *file, size_t want)
+{
+	/* Sort out buffer */
+	if ((file->buffer_pos - want) <= 0) {
+		/* Ditch buffer - write will recreate */
+		free(file->buffer);
+		file->buffer = NULL;
+		file->buffer_pos = 0;
+		file->buffer_len = 0;
+	}
+	else {
+		/* Move rest down make it available for later */
+		memmove(file->buffer, &file->buffer[want], (file->buffer_pos - want));
+
+		file->buffer_pos -= want;
+	}
+	return 0;
+}
+
+URL_FILE *url_fopen(const char *url, const char *operation)
+{
+	/* This code could check for URLs or types in the 'url' and
+	   basically use the real fopen() for standard files */
+
+	URL_FILE *file;
+	(void)operation;
+
+	file = malloc(sizeof(URL_FILE));
+	if (!file)
+		return NULL;
+
+	memset(file, 0, sizeof(URL_FILE));
+
+	file->handle.file = fopen(url, operation);
+	if (file->handle.file)
+		file->type = CFTYPE_FILE; /* Marked as standard file */
+	else {
+		file->type = CFTYPE_CURL; /* Marked as URL */
+		file->handle.curl = curl_easy_init();
+
+		curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
+		curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file);
+		curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L);
+		curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
+
+		if (!multi_handle)
+			multi_handle = curl_multi_init();
+
+		curl_multi_add_handle(multi_handle, file->handle.curl);
+
+		/* Lets start the fetch */
+		curl_multi_perform(multi_handle, &file->still_running);
+
+		if ((file->buffer_pos == 0) && (!file->still_running)) {
+			/* If still_running is 0 now, we should return NULL */
+
+			/* Make sure the easy handle is not in the multi handle anymore */
+			curl_multi_remove_handle(multi_handle, file->handle.curl);
+
+			/* Cleanup */
+			curl_easy_cleanup(file->handle.curl);
+
+			free(file);
+
+			file = NULL;
+		}
+	}
+
+	return file;
+}
+
+int url_fclose(URL_FILE *file)
+{
+	int ret = 0; /* Default is good return */
+
+	switch (file->type) {
+		case CFTYPE_FILE:
+			ret = fclose(file->handle.file); /* Passthrough */
+			break;
+
+		case CFTYPE_CURL:
+			/* Make sure the easy handle is not in the multi handle anymore */
+			curl_multi_remove_handle(multi_handle, file->handle.curl);
+
+			/* Cleanup */
+			curl_easy_cleanup(file->handle.curl);
+			break;
+
+		default: /* Unknown or supported type - oh dear */
+			ret = EOF;
+			errno = EBADF;
+			break;
+	}
+
+	free(file->buffer);/* Free any allocated buffer space */
+	free(file);
+
+	return ret;
+}
+
+int url_feof(URL_FILE *file)
+{
+	int ret = 0;
+
+	switch (file->type) {
+		case CFTYPE_FILE:
+			ret = feof(file->handle.file);
+			break;
+
+		case CFTYPE_CURL:
+			if ((file->buffer_pos == 0) && (!file->still_running))
+				ret = 1;
+			break;
+
+		default: /* Unknown or supported type - oh dear */
+			ret = -1;
+			errno = EBADF;
+			break;
+	}
+
+	return ret;
+}
+
+size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
+{
+	size_t want;
+
+	switch (file->type) {
+		case CFTYPE_FILE:
+			want = fread(ptr, size, nmemb, file->handle.file);
+			break;
+
+		case CFTYPE_CURL:
+			want = nmemb * size;
+
+			fill_buffer(file, want);
+
+			/* Check if theres data in the buffer - if not fill_buffer()
+			 * either errored or EOF */
+			if (!file->buffer_pos)
+				return 0;
+
+			/* Ensure only available data is considered */
+			if (file->buffer_pos < want)
+				want = file->buffer_pos;
+
+			/* Xfer data to caller */
+			memcpy(ptr, file->buffer, want);
+
+			use_buffer(file, want);
+
+			want = want / size; /* Number of items */
+			break;
+
+		default: /* Unknown or supported type - oh dear */
+			want = 0;
+			errno = EBADF;
+			break;
+	}
+
+	return want;
+}
+
+char *url_fgets(char *ptr, size_t size, URL_FILE *file)
+{
+	size_t want = size - 1; /* Always need to leave room for zero termination */
+	size_t loop;
+
+	switch (file->type) {
+		case CFTYPE_FILE:
+			ptr = fgets(ptr, (int)size, file->handle.file);
+			break;
+
+		case CFTYPE_CURL:
+			fill_buffer(file, want);
+
+			/* Check if theres data in the buffer - if not fill either errored or
+			 * EOF */
+			if (!file->buffer_pos)
+				return NULL;
+
+			/* Ensure only available data is considered */
+			if (file->buffer_pos < want)
+				want = file->buffer_pos;
+
+			/* Buffer contains data */
+			/* Look for newline or eof */
+			for (loop = 0; loop < want; loop++) {
+				if (file->buffer[loop] == '\n') {
+					want = loop+1; /* Include newline */
+					break;
+				}
+			}
+
+			/* Xfer data to caller */
+			memcpy(ptr, file->buffer, want);
+			ptr[want] = 0; /* Allways null terminate */
+
+			use_buffer(file, want);
+
+			break;
+
+		default: /* Unknown or supported type - oh dear */
+			ptr = NULL;
+			errno = EBADF;
+			break;
+	}
+
+	return ptr;/*success */
+}
+
+void url_rewind(URL_FILE *file)
+{
+	switch (file->type) {
+		case CFTYPE_FILE:
+			rewind(file->handle.file); /* Passthrough */
+			break;
+
+		case CFTYPE_CURL:
+			/* Halt transaction */
+			curl_multi_remove_handle(multi_handle, file->handle.curl);
+
+			/* Restart */
+			curl_multi_add_handle(multi_handle, file->handle.curl);
+
+			/* Ditch buffer - write will recreate - resets stream pos */
+			free(file->buffer);
+			file->buffer = NULL;
+			file->buffer_pos = 0;
+			file->buffer_len = 0;
+
+			break;
+
+		default: /* Unknown or supported type - oh dear */
+			break;
+	}
+}

tests/curlio.c

@@ -0,0 +1,99 @@
+#define FGETSFILE "fgets.test"
+#define FREADFILE "fread.test"
+#define REWINDFILE "rewind.test"
+
+/* Small main program to retrive from a url using fgets and fread saving the
+ * output to two test files (note the fgets method will corrupt binary files if
+ * they contain 0 chars */
+Test(curlio, main)
+{
+  URL_FILE *handle;
+  FILE *outf;
+
+  size_t nread;
+  char buffer[256];
+  const char *url;
+
+  if(argc < 2)
+    url="http://192.168.7.3/testfile";/* default to testurl */
+  else
+    url=argv[1];/* use passed url */
+
+  /* copy from url line by line with fgets */
+  outf=fopen(FGETSFILE, "wb+");
+  if(!outf) {
+    perror("couldn't open fgets output file\n");
+    return 1;
+  }
+
+  handle = url_fopen(url, "r");
+  if(!handle) {
+    printf("couldn't url_fopen() %s\n", url);
+    fclose(outf);
+    return 2;
+  }
+
+  while(!url_feof(handle)) {
+    url_fgets(buffer, sizeof(buffer), handle);
+    fwrite(buffer, 1, strlen(buffer), outf);
+  }
+
+  url_fclose(handle);
+
+  fclose(outf);
+
+
+  /* Copy from url with fread */
+  outf=fopen(FREADFILE, "wb+");
+  if(!outf) {
+    perror("couldn't open fread output file\n");
+    return 1;
+  }
+
+  handle = url_fopen("testfile", "r");
+  if(!handle) {
+    printf("couldn't url_fopen() testfile\n");
+    fclose(outf);
+    return 2;
+  }
+
+  do {
+    nread = url_fread(buffer, 1, sizeof(buffer), handle);
+    fwrite(buffer, 1, nread, outf);
+  } while(nread);
+
+  url_fclose(handle);
+
+  fclose(outf);
+
+
+  /* Test rewind */
+  outf=fopen(REWINDFILE, "wb+");
+  if(!outf) {
+    perror("couldn't open fread output file\n");
+    return 1;
+  }
+
+  handle = url_fopen("testfile", "r");
+  if(!handle) {
+    printf("couldn't url_fopen() testfile\n");
+    fclose(outf);
+    return 2;
+  }
+
+  nread = url_fread(buffer, 1, sizeof(buffer), handle);
+  fwrite(buffer, 1, nread, outf);
+  url_rewind(handle);
+
+  buffer[0]='\n';
+  fwrite(buffer, 1, 1, outf);
+
+  nread = url_fread(buffer, 1, sizeof(buffer), handle);
+  fwrite(buffer, 1, nread, outf);
+
+  url_fclose(handle);
+
+  fclose(outf);
+
+  return 0;/* all done */
+}