From aa4143aebdb59f4f21094781fb3b6ab4649643a6 Mon Sep 17 00:00:00 2001 From: Andy Green Date: Mon, 12 Nov 2018 15:24:42 +0800 Subject: [PATCH] lws_diskcache: split generic parts from gitohashi into lws --- CMakeLists.txt | 54 +- cmake/lws_config_private.h.in | 2 + include/libwebsockets.h | 1 + include/libwebsockets/lws-diskcache.h | 185 +++++++ lib/core/libwebsockets.c | 6 +- lib/misc/diskcache.c | 476 ++++++++++++++++++ lib/misc/threadpool/threadpool.c | 14 +- lib/roles/h2/http2.c | 20 +- lib/roles/h2/ops-h2.c | 15 +- lib/roles/http/client/client.c | 2 + .../CMakeLists.txt | 1 + minimal-examples/selftests.sh | 30 +- 12 files changed, 772 insertions(+), 34 deletions(-) create mode 100644 include/libwebsockets/lws-diskcache.h create mode 100644 lib/misc/diskcache.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 9aba77220..a904d862a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ set(LWS_WITH_POLL 1) # Select features recommended for PC distro packaging # option(LWS_WITH_DISTRO_RECOMMENDED "Enable features recommended for distro packaging" OFF) +option(LWS_FOR_GITOHASHI "Enable features recommended for use with gitohashi" OFF) # # Major individual features @@ -24,13 +25,13 @@ option(LWS_WITH_DISTRO_RECOMMENDED "Enable features recommended for distro packa option(LWS_ROLE_H1 "Compile with support for http/1 (needed for ws)" ON) option(LWS_ROLE_WS "Compile with support for websockets" ON) option(LWS_ROLE_DBUS "Compile with support for DBUS" OFF) -option(LWS_WITH_HTTP2 "Compile with server support for HTTP/2" OFF) +option(LWS_WITH_HTTP2 "Compile with server support for HTTP/2" ON) option(LWS_WITH_LWSWS "Libwebsockets Webserver" OFF) option(LWS_WITH_CGI "Include CGI (spawn process with network-connected stdin/out/err) APIs" OFF) option(LWS_IPV6 "Compile with support for ipv6" OFF) -option(LWS_UNIX_SOCK "Compile with support for UNIX domain socket" ON) +option(LWS_UNIX_SOCK "Compile with support for UNIX domain socket" OFF) option(LWS_WITH_PLUGINS "Support plugins for protocols and extensions" OFF) -option(LWS_WITH_HTTP_PROXY "Support for HTTP proxying" ON) +option(LWS_WITH_HTTP_PROXY "Support for HTTP proxying" OFF) option(LWS_WITH_ZIP_FOPS "Support serving pre-zipped files" OFF) option(LWS_WITH_SOCKS5 "Allow use of SOCKS5 proxy on client connections" OFF) option(LWS_WITH_GENERIC_SESSIONS "With the Generic Sessions plugin" OFF) @@ -38,12 +39,12 @@ option(LWS_WITH_PEER_LIMITS "Track peers and restrict resources a single peer ca option(LWS_WITH_ACCESS_LOG "Support generating Apache-compatible access logs" OFF) option(LWS_WITH_RANGES "Support http ranges (RFC7233)" OFF) option(LWS_WITH_SERVER_STATUS "Support json + jscript server monitoring" OFF) -option(LWS_WITH_THREADPOOL "Managed worker thread pool support (relies on pthreads)" ON) +option(LWS_WITH_THREADPOOL "Managed worker thread pool support (relies on pthreads)" OFF) option(LWS_WITH_HTTP_STREAM_COMPRESSION "Support HTTP stream compression" OFF) option(LWS_WITH_HTTP_BROTLI "Also offer brotli http stream compression (requires LWS_WITH_HTTP_STREAM_COMPRESSION)" OFF) option(LWS_WITH_ACME "Enable support for ACME automatic cert acquisition + maintenance (letsencrypt etc)" OFF) option(LWS_WITH_HUBBUB "Enable libhubbub rewriting support" OFF) -option(LWS_WITH_FTS "Full Text Search support" ON) +option(LWS_WITH_FTS "Full Text Search support" OFF) # # TLS library options... all except mbedTLS are basically OpenSSL variants. # @@ -112,11 +113,22 @@ option(LWS_WITH_EXPORT_LWSTARGETS "Export libwebsockets CMake targets. Disable option(LWS_REPRODUCIBLE "Build libwebsockets reproducible. It removes the build user and hostname from the build" ON) option(LWS_WITH_MINIMAL_EXAMPLES "Also build the normally standalone minimal examples, for QA" OFF) option(LWS_WITH_LWSAC "lwsac Chunk Allocation api" ON) +option(LWS_WITH_DISKCACHE "Hashed cache directory with lazy LRU deletion to size limit" OFF) option(LWS_WITH_ASAN "Build with gcc runtime sanitizer options enabled (needs libasan)" OFF) # # End of user settings # +if (LWS_FOR_GITOHASHI) + set(LWS_WITH_THREADPOOL 1) + set(LWS_WITH_HTTP2 1) + set(LWS_UNIX_SOCK 1) + set(LWS_WITH_HTTP_PROXY 1) + set(LWS_WITH_FTS 1) + set(LWS_WITH_DISKCACHE 1) + set(LWS_WITH_LWSAC 1) + set(LWS_WITH_LEJP_CONF 1) +endif() if(LWS_WITH_DISTRO_RECOMMENDED) set(LWS_WITH_HTTP2 1) @@ -135,6 +147,12 @@ if(LWS_WITH_DISTRO_RECOMMENDED) set(LWS_WITHOUT_EXTENSIONS 0) set(LWS_ROLE_DBUS 1) set(LWS_WITH_FTS 1) + set(LWS_WITH_THREADPOOL 1) + set(LWS_UNIX_SOCK 1) + set(LWS_WITH_HTTP_PROXY 1) + set(LWS_WITH_DISKCACHE 1) + set(LWS_WITH_LWSAC 1) + set(LWS_WITH_LEJP_CONF 1) endif() # do you care about this? Then send me a patch where it disables it on travis @@ -227,6 +245,11 @@ if(GIT_EXECUTABLE) endif() # translate old functionality enables to set up ROLE enables so nothing changes +if (LWS_WITH_HTTP2 AND LWS_WITHOUT_SERVER) + set(LWS_WITH_HTTP2 0) + message("HTTP2 disabled due to LWS_WITHOUT_SERVER") +endif() + if (LWS_WITH_HTTP2) set(LWS_ROLE_H2 1) endif() @@ -238,10 +261,6 @@ if (NOT LWS_ROLE_WS) set(LWS_WITHOUT_EXTENSIONS 1) endif() -if (LWS_WITH_HTTP2 AND LWS_WITHOUT_SERVER) - message(FATAL_ERROR "HTTP2 can only be used with server at the moment") -endif() - include_directories(include plugins) @@ -767,10 +786,14 @@ CHECK_C_SOURCE_COMPILES("#include return 0; }" LWS_HAS_INTPTR_T) -# These don't work Cross... -#CHECK_TYPE_SIZE(pid_t PID_T_SIZE) -#CHECK_TYPE_SIZE(size_t SIZE_T_SIZE) -#CHECK_TYPE_SIZE("void *" LWS_SIZEOFPTR LANGUAGE C) +set(CMAKE_REQUIRED_FLAGS "-pthread") +CHECK_C_SOURCE_COMPILES("#define _GNU_SOURCE + #include + int main(void) { + pthread_t th = 0; + pthread_setname_np(th, NULL); + return 0; + }" LWS_HAS_PTHREAD_SETNAME_NP) if (NOT PID_T_SIZE) set(pid_t int) @@ -896,6 +919,11 @@ if (LWS_WITH_FTS) lib/misc/fts/trie-fd.c) endif() +if (LWS_WITH_DISKCACHE) + list(APPEND SOURCES + lib/misc/diskcache.c) +endif() + if (NOT LWS_WITHOUT_CLIENT) list(APPEND SOURCES lib/core/connect.c diff --git a/cmake/lws_config_private.h.in b/cmake/lws_config_private.h.in index a3235818d..10eea96e5 100644 --- a/cmake/lws_config_private.h.in +++ b/cmake/lws_config_private.h.in @@ -120,3 +120,5 @@ #cmakedefine inline ${inline} #cmakedefine LWS_WITH_ZLIB +#cmakedefine LWS_HAS_PTHREAD_SETNAME_NP + diff --git a/include/libwebsockets.h b/include/libwebsockets.h index 8b6d2aba8..7cc5c28b7 100644 --- a/include/libwebsockets.h +++ b/include/libwebsockets.h @@ -411,6 +411,7 @@ struct lws; #include #include #include +#include #if defined(LWS_WITH_TLS) diff --git a/include/libwebsockets/lws-diskcache.h b/include/libwebsockets/lws-diskcache.h new file mode 100644 index 000000000..eb63fdd85 --- /dev/null +++ b/include/libwebsockets/lws-diskcache.h @@ -0,0 +1,185 @@ +/* + * libwebsockets - disk cache helpers + * + * Copyright (C) 2010-2018 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * included from libwebsockets.h + */ + +/*! \defgroup diskcache LWS disk cache + * ## Disk cache API + * + * Lws provides helper apis useful if you need a disk cache containing hashed + * files and need to delete files from it on an LRU basis to keep it below some + * size limit. + * + * The API `lws_diskcache_prepare()` deals with creating the cache dir and + * 256 subdirs, which are used according to the first two chars of the hex + * hash of the cache file. + * + * `lws_diskcache_create()` and `lws_diskcache_destroy()` allocate and free + * an opaque struct that represents the disk cache. + * + * `lws_diskcache_trim()` should be called at eg, 1s intervals to perform the + * cache dir monitoring and LRU autodelete in the background lazily. It can + * be done in its own thread or on a timer... it monitors the directories in a + * stateful way that stats one or more file in the cache per call, and keeps + * a list of the oldest files as it goes. When it completes a scan, if the + * aggregate size is over the limit, it will delete oldest files first to try + * to keep it under the limit. + * + * The cache size monitoring is extremely efficient in time and memory even when + * the cache directory becomes huge. + * + * `lws_diskcache_query()` is used to determine if the file already exists in + * the cache, or if it must be created. If it must be created, then the file + * is opened using a temp name that must be converted to a findable name with + * `lws_diskcache_finalize_name()` when the generation of the file contents are + * complete. Aborted cached files that did not complete generation will be + * flushed by the LRU eventually. If the file already exists, it is 'touched' + * to make it new again and the fd returned. + * + */ +///@{ + +struct lws_diskcache_scan; + +/** + * lws_diskcache_create() - creates an opaque struct representing the disk cache + * + * \param cache_dir_base: The cache dir path, eg `/var/cache/mycache` + * \param cache_size_limit: maximum size on disk the cache is allowed to use + * + * This returns an opaque `struct lws_diskcache_scan *` which represents the + * disk cache, the trim scanning state and so on. You should use + * `lws_diskcache_destroy()` to free it to destroy it. + */ +LWS_VISIBLE LWS_EXTERN struct lws_diskcache_scan * +lws_diskcache_create(const char *cache_dir_base, uint64_t cache_size_limit); + +/** + * lws_diskcache_destroy() - destroys the pointer returned by ...create() + * + * \param lds: pointer to the pointer returned by lws_diskcache_create() + * + * Frees *lds and any allocations it did, and then sets *lds to NULL and + * returns. + */ +LWS_VISIBLE LWS_EXTERN void +lws_diskcache_destroy(struct lws_diskcache_scan **lds); + +/** + * lws_diskcache_prepare() - ensures the cache dir structure exists on disk + * + * \param cache_base_dir: The cache dir path, eg `/var/cache/mycache` + * \param mode: octal dir mode to enforce, like 0700 + * \param uid: uid the cache dir should belong to + * + * This should be called while your app is still privileged. It will create + * the cache directory structure on disk as necessary, enforce the given access + * mode on it and set the given uid as the owner. It won't make any trouble + * if the cache already exists. + * + * Typically the mode is 0700 and the owner is the user that your application + * will transition to use when it drops root privileges. + */ +LWS_VISIBLE LWS_EXTERN int +lws_diskcache_prepare(const char *cache_base_dir, int mode, int uid); + +#define LWS_DISKCACHE_QUERY_NO_CACHE 0 +#define LWS_DISKCACHE_QUERY_EXISTS 1 +#define LWS_DISKCACHE_QUERY_CREATING 2 +#define LWS_DISKCACHE_QUERY_ONGOING 3 /* something else is creating it */ + +/** + * lws_diskcache_query() - ensures the cache dir structure exists on disk + * + * \param lds: The opaque struct representing the disk cache + * \param is_bot: nonzero means the request is from a bot. Don't create new cache contents if so. + * \param hash_hex: hex string representation of the cache object hash + * \param _fd: pointer to the fd to be set + * \param cache: destination string to take the cache filepath + * \param cache_len: length of the buffer at `cache` + * \param extant_cache_len: pointer to a size_t to take any extant cached file size + * + * This function is called when you want to find if the hashed name already + * exists in the cache. The possibilities for the return value are + * + * - LWS_DISKCACHE_QUERY_NO_CACHE: It's not in the cache and you can't create + * it in the cache for whatever reason. + * - LWS_DISKCACHE_QUERY_EXISTS: It exists in the cache. It's open RDONLY and + * *_fd has been set to the file descriptor. *extant_cache_len has been set + * to the size of the cached file in bytes. cache has been set to the + * full filepath of the cached file. Closing _fd is your responsibility. + * - LWS_DISKCACHE_QUERY_CREATING: It didn't exist, but a temp file has been + * created in the cache and *_fd set to a file descriptor opened on it RDWR. + * You should create the contents, and call `lws_diskcache_finalize_name()` + * when it is done. Closing _fd is your responsibility. + * - LWS_DISKCACHE_QUERY_ONGOING: not returned by this api, but you may find it + * desirable to make a wrapper function which can handle another asynchronous + * process that is already creating the cached file. This can be used to + * indicate that situation externally... how to determine the same thing is + * already being generated is out of scope of this api. + */ +LWS_VISIBLE LWS_EXTERN int +lws_diskcache_query(struct lws_diskcache_scan *lds, int is_bot, + const char *hash_hex, int *_fd, char *cache, int cache_len, + size_t *extant_cache_len); + +/** + * lws_diskcache_query() - ensures the cache dir structure exists on disk + * + * \param cache: The cache file temp name returned with LWS_DISKCACHE_QUERY_CREATING + * + * This renames the cache file you are creating to its final name. It should + * be called on the temp name returned by `lws_diskcache_query()` if it gave a + * LWS_DISKCACHE_QUERY_CREATING return, after you have filled the cache file and + * closed it. + */ +LWS_VISIBLE LWS_EXTERN int +lws_diskcache_finalize_name(char *cache); + +/** + * lws_diskcache_trim() - performs one or more file checks in the cache for size management + * + * \param lds: The opaque object representing the cache + * + * This should be called periodically to statefully walk the cache on disk + * collecting the oldest files. When it has visited every file, if the cache + * is oversize it will delete the oldest files until it's back under size again. + * + * Each time it's called, it will look at one or more dir in the cache. If + * called when the cache is oversize, it increases the amount of work done each + * call until it is reduced again. Typically it will take 256 calls before it + * deletes anything, so if called once per second, it will delete files once + * every 4 minutes. Each call is very inexpensive both in memory and time. + */ +LWS_VISIBLE LWS_EXTERN int +lws_diskcache_trim(struct lws_diskcache_scan *lds); + + +/** + * lws_diskcache_secs_to_idle() - see how long to idle before calling trim + * + * \param lds: The opaque object representing the cache + * + * If the cache is undersize, there's no need to monitor it immediately. This + * suggests how long to "sleep" before calling `lws_diskcache_trim()` again. + */ +LWS_VISIBLE LWS_EXTERN int +lws_diskcache_secs_to_idle(struct lws_diskcache_scan *lds); diff --git a/lib/core/libwebsockets.c b/lib/core/libwebsockets.c index d47a29bb0..a93537ae2 100644 --- a/lib/core/libwebsockets.c +++ b/lib/core/libwebsockets.c @@ -1382,7 +1382,11 @@ lws_get_network_wsi(struct lws *wsi) return NULL; #if defined(LWS_WITH_HTTP2) - if (!wsi->http2_substream && !wsi->client_h2_substream) + if (!wsi->http2_substream +#if !defined(LWS_NO_CLIENT) + && !wsi->client_h2_substream +#endif + ) return wsi; while (wsi->h2.parent_wsi) diff --git a/lib/misc/diskcache.c b/lib/misc/diskcache.c new file mode 100644 index 000000000..0a31810c1 --- /dev/null +++ b/lib/misc/diskcache.c @@ -0,0 +1,476 @@ +/* + * libwebsockets - disk cache helpers + * + * Copyright (C) 2018 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#define _GNU_SOURCE +#include + +#include "core/private.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +struct file_entry { + lws_list_ptr sorted; + lws_list_ptr prev; + char name[64]; + time_t modified; + size_t size; +}; + +struct lws_diskcache_scan { + struct file_entry *batch; + const char *cache_dir_base; + lws_list_ptr head; + time_t last_scan_completed; + uint64_t agg_size; + uint64_t cache_size_limit; + uint64_t avg_size; + uint64_t cache_tries; + uint64_t cache_hits; + int cache_subdir; + int batch_in_use; + int agg_file_count; + int secs_waiting; +}; + +#define KIB (1024) +#define MIB (KIB * KIB) + +#define lp_to_fe(p, _n) lws_list_ptr_container(p, struct file_entry, _n) + +static const char *hex = "0123456789abcdef"; + +#define BATCH_COUNT 128 + +static int +fe_modified_sort(lws_list_ptr a, lws_list_ptr b) +{ + struct file_entry *p1 = lp_to_fe(a, sorted), *p2 = lp_to_fe(b, sorted); + + return p2->modified - p1->modified; +} + +struct lws_diskcache_scan * +lws_diskcache_create(const char *cache_dir_base, uint64_t cache_size_limit) +{ + struct lws_diskcache_scan *lds = lws_malloc(sizeof(*lds), "cachescan"); + + if (!lds) + return NULL; + + memset(lds, 0, sizeof(*lds)); + + lds->cache_dir_base = cache_dir_base; + lds->cache_size_limit = cache_size_limit; + + return lds; +} + +void +lws_diskcache_destroy(struct lws_diskcache_scan **lds) +{ + if ((*lds)->batch) + lws_free((*lds)->batch); + lws_free(*lds); + *lds = NULL; +} + +int +lws_diskcache_prepare(const char *cache_base_dir, int mode, int uid) +{ + char dir[256]; + int n, m; + + (void)mkdir(cache_base_dir, mode); + if (chown(cache_base_dir, uid, -1)) + lwsl_err("%s: %s: unable to chown %d\n", __func__, + cache_base_dir, uid); + + for (n = 0; n < 16; n++) { + lws_snprintf(dir, sizeof(dir), "%s/%c", cache_base_dir, hex[n]); + (void)mkdir(dir, mode); + if (chown(dir, uid, -1)) + lwsl_err("%s: %s: unable to chown %d\n", __func__, + dir, uid); + for (m = 0; m < 16; m++) { + lws_snprintf(dir, sizeof(dir), "%s/%c/%c", + cache_base_dir, hex[n], hex[m]); + (void)mkdir(dir, mode); + if (chown(dir, uid, -1)) + lwsl_err("%s: %s: unable to chown %d\n", + __func__, dir, uid); + } + } + + return 0; +} + +/* copies and then truncates the incoming name, and renames the file at the + * untruncated path to have the new truncated name */ + +int +lws_diskcache_finalize_name(char *cache) +{ + char ren[256], *p; + + strncpy(ren, cache, sizeof(ren) - 1); + ren[sizeof(ren) - 1] = '\0'; + p = strchr(cache, '~'); + if (p) { + *p = '\0'; + if (rename(ren, cache)) { + lwsl_err("%s: problem renaming %s to %s\n", __func__, + ren, cache); + return 1; + } + + return 0; + } + + return 1; +} + +int +lws_diskcache_query(struct lws_diskcache_scan *lds, int is_bot, + const char *hash_hex, int *_fd, char *cache, int cache_len, + size_t *extant_cache_len) +{ + struct stat s; + int n; + + /* caching is disabled? */ + if (!lds->cache_dir_base) + return LWS_DISKCACHE_QUERY_NO_CACHE; + + if (!is_bot) + lds->cache_tries++; + + n = lws_snprintf(cache, cache_len, "%s/%c/%c/%s", lds->cache_dir_base, + hash_hex[0], hash_hex[1], hash_hex); + + lwsl_info("%s: job cache %s\n", __func__, cache); + + *_fd = open(cache, O_RDONLY); + if (*_fd >= 0) { + int fd; + + if (!is_bot) + lds->cache_hits++; + + if (fstat(*_fd, &s)) { + close(*_fd); + + return LWS_DISKCACHE_QUERY_NO_CACHE; + } + + *extant_cache_len = (size_t)s.st_size; + + /* "touch" the hit cache file so it's last for LRU now */ + fd = open(cache, O_RDWR); + if (fd >= 0) + close(fd); + + return LWS_DISKCACHE_QUERY_EXISTS; + } + + /* bots are too random to pollute the cache with their antics */ + if (is_bot) + return LWS_DISKCACHE_QUERY_NO_CACHE; + + /* let's create it first with a unique temp name */ + + lws_snprintf(cache + n, cache_len - n, "~%d-%p", (int)getpid(), + extant_cache_len); + + *_fd = open(cache, O_RDWR | O_CREAT | O_TRUNC, 0600); + if (*_fd < 0) { + /* well... ok... we will proceed without cache then... */ + lwsl_notice("%s: Problem creating cache %s: errno %d\n", + __func__, cache, errno); + return LWS_DISKCACHE_QUERY_NO_CACHE; + } + + return LWS_DISKCACHE_QUERY_CREATING; +} + +int +lws_diskcache_secs_to_idle(struct lws_diskcache_scan *lds) +{ + return lds->secs_waiting; +} + +/* + * The goal is to collect the oldest BATCH_COUNT filepaths and filesizes from + * the dirs under the cache dir. Since we don't need or want a full list of + * files in there in memory at once, we restrict the linked-list size to + * BATCH_COUNT entries, and once it is full, simply ignore any further files + * that are newer than the newest one on that list. Files older than the + * newest guy already on the list evict the newest guy already on the list + * and are sorted into the correct order. In this way no matter the number + * of files to be processed the memory requirement is fixed at BATCH_COUNT + * struct file_entry-s. + * + * The oldest subset of BATCH_COUNT files are sorted into the cd->batch + * allocation in more recent -> least recent order. + * + * We want to track the total size of all files we saw as well, so we know if + * we need to actually do anything yet to restrict how much space it's taking + * up. + * + * And we want to do those things statefully and incrementally instead of one + * big atomic operation, since the user may want a huge cache, so we look in + * one cache dir at a time and track state in the repodir struct. + * + * When we have seen everything, we add the doubly-linked prev pointers and then + * if we are over the limit, start deleting up to BATCH_COUNT files working back + * from the end. + */ + +int +lws_diskcache_trim(struct lws_diskcache_scan *lds) +{ + size_t cache_size_limit = lds->cache_size_limit; + char dirpath[132], filepath[132 + 32]; + lws_list_ptr lp, op = NULL; + int files_trimmed = 0; + struct file_entry *p; + int fd, n, ret = -1; + size_t trimmed = 0; + struct dirent *de; + struct stat s; + DIR *dir; + + if (!lds->cache_subdir) { + + if (lds->last_scan_completed + lds->secs_waiting > time(NULL)) + return 0; + + lds->batch = lws_malloc(sizeof(struct file_entry) * + BATCH_COUNT, "cache_trim"); + if (!lds->batch) { + lwsl_err("%s: OOM\n", __func__); + + return 1; + } + lds->agg_size = 0; + lds->head = NULL; + lds->batch_in_use = 0; + lds->agg_file_count = 0; + } + + lws_snprintf(dirpath, sizeof(dirpath), "%s/%c/%c", + lds->cache_dir_base, hex[(lds->cache_subdir >> 4) & 15], + hex[lds->cache_subdir & 15]); + + dir = opendir(dirpath); + if (!dir) { + lwsl_err("Unable to walk repo dir '%s'\n", + lds->cache_dir_base); + return -1; + } + + do { + de = readdir(dir); + if (!de) + break; + + if (de->d_type != DT_REG) + continue; + + lds->agg_file_count++; + + lws_snprintf(filepath, sizeof(filepath), "%s/%s", dirpath, + de->d_name); + + fd = open(filepath, O_RDONLY); + if (fd < 0) { + lwsl_err("%s: cannot open %s\n", __func__, filepath); + + continue; + } + + n = fstat(fd, &s); + close(fd); + if (n) { + lwsl_notice("%s: cannot stat %s\n", __func__, filepath); + continue; + } + + lds->agg_size += s.st_size; + + if (lds->batch_in_use == BATCH_COUNT) { + /* + * once we filled up the batch with candidates, we don't + * need to consider any files newer than the newest guy + * on the list... + */ + if (lp_to_fe(lds->head, sorted)->modified < s.st_mtime) + continue; + + /* + * ... and if we find an older file later, we know it + * will be replacing the newest guy on the list, so use + * that directly... + */ + p = lds->head; + lds->head = p->sorted; + } else + /* we are still accepting anything to fill the batch */ + + p = &lds->batch[lds->batch_in_use++]; + + p->sorted = NULL; + strncpy(p->name, de->d_name, sizeof(p->name) - 1); + p->name[sizeof(p->name) - 1] = '\0'; + p->modified = s.st_mtime; + p->size = s.st_size; + + lws_list_ptr_insert(&lds->head, &p->sorted, fe_modified_sort); + } while (de); + + ret = 0; + + lds->cache_subdir++; + if (lds->cache_subdir != 0x100) + goto done; + + /* we completed the whole scan... */ + + /* if really no guidence, then 256MiB */ + if (!cache_size_limit) + cache_size_limit = 256 * 1024 * 1024; + + if (lds->agg_size > cache_size_limit) { + + /* apply prev pointers to make the list doubly-linked */ + + lp = lds->head; + while (lp) { + p = lp_to_fe(lp, sorted); + + p->prev = op; + op = &p->prev; + lp = p->sorted; + } + + /* + * reverse the list (start from tail, now traverse using + * .prev)... it's oldest-first now... + */ + + lp = op; + + while (lp && lds->agg_size > cache_size_limit) { + p = lp_to_fe(lp, prev); + + lws_snprintf(filepath, sizeof(filepath), "%s/%c/%c/%s", + lds->cache_dir_base, p->name[0], + p->name[1], p->name); + + if (!unlink(filepath)) { + lds->agg_size -= p->size; + trimmed += p->size; + files_trimmed++; + } else + lwsl_notice("%s: Failed to unlink %s\n", + __func__, filepath); + + lp = p->prev; + } + + if (files_trimmed) + lwsl_notice("%s: %s: trimmed %d files totalling " + "%lldKib, leaving %lldMiB\n", __func__, + lds->cache_dir_base, files_trimmed, + ((unsigned long long)trimmed) / KIB, + ((unsigned long long)lds->agg_size) / MIB); + } + + if (lds->agg_size && lds->agg_file_count) + lds->avg_size = lds->agg_size / lds->agg_file_count; + + /* + * estimate how long we can go before scanning again... default we need + * to start again immediately + */ + + lds->last_scan_completed = time(NULL); + lds->secs_waiting = 1; + + if (lds->agg_size < cache_size_limit) { + uint64_t avg = 4096, capacity, projected; + + /* let's use 80% of the real average for margin */ + if (lds->agg_size && lds->agg_file_count) + avg = ((lds->agg_size * 8) / lds->agg_file_count) / 10; + + /* + * if we collected BATCH_COUNT files of the average size, + * how much can we clean up in 256s? + */ + + capacity = avg * BATCH_COUNT; + + /* + * if the cache grew by 10%, would we hit the limit even then? + */ + projected = (lds->agg_size * 11) / 10; + if (projected < cache_size_limit) + /* no... */ + lds->secs_waiting = (256 / 2) * ((cache_size_limit - + projected) / capacity); + + /* + * large waits imply we may not have enough info yet, so + * check once an hour at least. + */ + + if (lds->secs_waiting > 3600) + lds->secs_waiting = 3600; + } else + lds->secs_waiting = 0; + + lwsl_info("%s: cache %s: %lldKiB / %lldKiB, next scan %ds\n", __func__, + lds->cache_dir_base, + (unsigned long long)lds->agg_size / KIB, + (unsigned long long)cache_size_limit / KIB, + lds->secs_waiting); + + lws_free(lds->batch); + lds->batch = NULL; + + lds->cache_subdir = 0; + +done: + closedir(dir); + + return ret; +} diff --git a/lib/misc/threadpool/threadpool.c b/lib/misc/threadpool/threadpool.c index 10a007da8..de9bab7df 100644 --- a/lib/misc/threadpool/threadpool.c +++ b/lib/misc/threadpool/threadpool.c @@ -19,9 +19,11 @@ * MA 02110-1301 USA */ +#define _GNU_SOURCE +#include + #include "core/private.h" -#include #include #include @@ -668,14 +670,22 @@ lws_threadpool_create(struct lws_context *context, pthread_cond_init(&tp->wake_idle, NULL); for (n = 0; n < args->threads; n++) { +#if defined(LWS_HAS_PTHREAD_SETNAME_NP) + char name[16]; +#endif tp->pool_list[n].tp = tp; tp->pool_list[n].worker_index = n; pthread_mutex_init(&tp->pool_list[n].lock, NULL); if (pthread_create(&tp->pool_list[n].thread, NULL, lws_threadpool_worker, &tp->pool_list[n])) { lwsl_err("thread creation failed\n"); - } else + } else { +#if defined(LWS_HAS_PTHREAD_SETNAME_NP) + lws_snprintf(name, sizeof(name), "%s-%d", tp->name, n); + pthread_setname_np(tp->pool_list[n].thread, name); +#endif tp->threads_in_pool++; + } } return tp; diff --git a/lib/roles/h2/http2.c b/lib/roles/h2/http2.c index 2c43fd0c4..17d74efc2 100644 --- a/lib/roles/h2/http2.c +++ b/lib/roles/h2/http2.c @@ -245,7 +245,9 @@ lws_wsi_h2_adopt(struct lws *parent_wsi, struct lws *wsi) /* sid is set just before issuing the headers, ensuring monoticity */ wsi->seen_nonpseudoheader = 0; +#if !defined(LWS_NO_CLIENT) wsi->client_h2_substream = 1; +#endif wsi->h2.initialized = 1; wsi->h2.parent_wsi = parent_wsi; @@ -848,7 +850,11 @@ lws_h2_parse_frame_header(struct lws *wsi) else { /* if it's data, either way no swsi means CLOSED state */ if (h2n->type == LWS_H2_FRAME_TYPE_DATA) { - if (h2n->sid <= h2n->highest_sid_opened && wsi->client_h2_alpn) { + if (h2n->sid <= h2n->highest_sid_opened +#if !defined(LWS_NO_CLIENT) + && wsi->client_h2_alpn +#endif + ) { lwsl_notice("ignoring straggling data\n"); h2n->type = LWS_H2_FRAME_TYPE_COUNT; /* ie, IGNORE */ } else { @@ -1338,6 +1344,7 @@ lws_h2_parse_end_of_frame(struct lws *wsi) lwsl_info("http req, wsi=%p, h2n->swsi=%p\n", wsi, h2n->swsi); h2n->swsi->hdr_parsing_completed = 1; +#if !defined(LWS_NO_CLIENT) if (h2n->swsi->client_h2_substream) { if (lws_client_interpret_server_handshake(h2n->swsi)) { lws_h2_rst_stream(h2n->swsi, H2_ERR_STREAM_CLOSED, @@ -1345,6 +1352,7 @@ lws_h2_parse_end_of_frame(struct lws *wsi) break; } } +#endif if (lws_hdr_extant(h2n->swsi, WSI_TOKEN_HTTP_CONTENT_LENGTH)) { h2n->swsi->http.rx_content_length = atoll( @@ -1401,10 +1409,12 @@ lws_h2_parse_end_of_frame(struct lws *wsi) break; } +#if !defined(LWS_NO_CLIENT) if (h2n->swsi->client_h2_substream) { lwsl_info("%s: headers: client path\n", __func__); break; } +#endif if (!lws_hdr_total_length(h2n->swsi, WSI_TOKEN_HTTP_COLON_PATH) || !lws_hdr_total_length(h2n->swsi, WSI_TOKEN_HTTP_COLON_METHOD) || @@ -1465,6 +1475,7 @@ lws_h2_parse_end_of_frame(struct lws *wsi) h2n->swsi->h2.h2_state == LWS_H2_STATE_HALF_CLOSED_LOCAL) lws_h2_state(h2n->swsi, LWS_H2_STATE_CLOSED); +#if !defined(LWS_NO_CLIENT) /* * client... remote END_STREAM implies we weren't going to * send anything else anyway. @@ -1494,6 +1505,7 @@ lws_h2_parse_end_of_frame(struct lws *wsi) lwsl_debug("tx completed returned close\n"); } } +#endif break; case LWS_H2_FRAME_TYPE_PING: @@ -1788,7 +1800,7 @@ lws_h2_parser(struct lws *wsi, unsigned char *in, lws_filepos_t inlen, n = h2n->length - h2n->count + 1; lwsl_debug("---- restricting len to %d vs %ld\n", n, (long)inlen + 1); } - +#if !defined(LWS_NO_CLIENT) if (h2n->swsi->client_h2_substream) { m = user_callback_handle_rxflow( @@ -1807,7 +1819,9 @@ lws_h2_parser(struct lws *wsi, unsigned char *in, lws_filepos_t inlen, } break; - } else { + } else +#endif + { if (lwsi_state(h2n->swsi) == LRS_DEFERRING_ACTION) { // lwsl_notice("appending because we are in LRS_DEFERRING_ACTION\n"); diff --git a/lib/roles/h2/ops-h2.c b/lib/roles/h2/ops-h2.c index 3639b86a4..76d5a62b7 100644 --- a/lib/roles/h2/ops-h2.c +++ b/lib/roles/h2/ops-h2.c @@ -605,7 +605,11 @@ rops_close_kill_connection_h2(struct lws *wsi, enum lws_close_status reason) } lws_end_foreach_llp(w, h2.sibling_list); } - if (wsi->upgraded_to_http2 || wsi->http2_substream || wsi->client_h2_substream) { + if (wsi->upgraded_to_http2 || wsi->http2_substream +#if !defined(LWS_NO_CLIENT) + || wsi->client_h2_substream +#endif + ) { lwsl_info("closing %p: parent %p\n", wsi, wsi->h2.parent_wsi); if (wsi->h2.child_list && lwsl_visible(LLL_INFO)) { @@ -645,7 +649,11 @@ rops_close_kill_connection_h2(struct lws *wsi, enum lws_close_status reason) wsi->h2.h2n->pps = NULL; } - if ((wsi->client_h2_substream || wsi->http2_substream) && + if (( +#if !defined(LWS_NO_CLIENT) + wsi->client_h2_substream || +#endif + wsi->http2_substream) && wsi->h2.parent_wsi) { lwsl_info(" %p: disentangling from siblings\n", wsi); lws_start_foreach_llp(struct lws **, w, @@ -732,8 +740,9 @@ rops_callback_on_writable_h2(struct lws *wsi) /* for network action, act only on the network wsi */ wsi = network_wsi; - if (already && !wsi->client_h2_alpn + if (already #if !defined(LWS_NO_CLIENT) + && !wsi->client_h2_alpn && !wsi->client_h2_substream #endif ) diff --git a/lib/roles/http/client/client.c b/lib/roles/http/client/client.c index 80c73a9dc..ecf44c8df 100644 --- a/lib/roles/http/client/client.c +++ b/lib/roles/http/client/client.c @@ -352,7 +352,9 @@ start_ws_handshake: * So this is it, we are an h2 master client connection * now, not an h1 client connection. */ +#if defined (LWS_WITH_TLS) lws_tls_server_conn_alpn(wsi); +#endif /* send the H2 preface to legitimize the connection */ if (lws_h2_issue_preface(wsi)) { diff --git a/minimal-examples/http-server/minimal-http-server-fulltext-search/CMakeLists.txt b/minimal-examples/http-server/minimal-http-server-fulltext-search/CMakeLists.txt index d152f8a4e..60328456e 100644 --- a/minimal-examples/http-server/minimal-http-server-fulltext-search/CMakeLists.txt +++ b/minimal-examples/http-server/minimal-http-server-fulltext-search/CMakeLists.txt @@ -67,6 +67,7 @@ ENDMACRO() set(requirements 1) require_lws_config(LWS_ROLE_H1 1 requirements) +require_lws_config(LWS_WITH_FTS 1 requirements) require_lws_config(LWS_WITHOUT_SERVER 0 requirements) if (requirements) diff --git a/minimal-examples/selftests.sh b/minimal-examples/selftests.sh index d7992ceea..77fbdd434 100755 --- a/minimal-examples/selftests.sh +++ b/minimal-examples/selftests.sh @@ -24,24 +24,30 @@ MINEX=`dirname $0` MINEX=`realpath $MINEX` TESTS=0 for i in `find $MINEX -name selftest.sh` ; do - C=`cat $i | grep COUNT_TESTS= | cut -d= -f2` - TESTS=$(( $TESTS + $C )) + BN=`echo -n "$i" | sed "s/\/[^\/]*\$//g" | sed "s/.*\///g"` + if [ -e `pwd`/bin/lws-$BN ] ; then + C=`cat $i | grep COUNT_TESTS= | cut -d= -f2` + TESTS=$(( $TESTS + $C )) + fi done FAILS=0 WH=1 for i in `find $MINEX -name selftest.sh` ; do - C=`cat $i | grep COUNT_TESTS= | cut -d= -f2` - sh $i `pwd`/bin $LOGGING_PATH $WH $TESTS $MINEX - FAILS=$(( $FAILS + $? )) - - L=`ps fax | grep lws- | cut -d' ' -f2` - kill $L 2>/dev/null - kill -9 $L 2>/dev/null - wait $L 2>/dev/null - - WH=$(( $WH + $C )) + BN=`echo -n "$i" | sed "s/\/[^\/]*\$//g" | sed "s/.*\///g"` + if [ -e `pwd`/bin/lws-$BN ] ; then + C=`cat $i | grep COUNT_TESTS= | cut -d= -f2` + sh $i `pwd`/bin $LOGGING_PATH $WH $TESTS $MINEX + FAILS=$(( $FAILS + $? )) + + L=`ps fax | grep lws- | cut -d' ' -f2` + kill $L 2>/dev/null + kill -9 $L 2>/dev/null + wait $L 2>/dev/null + + WH=$(( $WH + $C )) + fi done if [ $FAILS -eq 0 ] ; then