Mirrors/libwebsockets
1
0
Fork 0
mirror of https://github.com/warmcat/libwebsockets.git synced 2025-03-09 00:00:04 +01:00
Code Issues Releases Wiki Activity

lws_diskcache: split generic parts from gitohashi into lws

Browse source
This commit is contained in:
Andy Green 2018-11-12 15:24:42 +08:00
parent c5d6255487
commit aa4143aebd
12 changed files with 772 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
CMakeLists.txt
View file

@ -17,6 +17,7 @@ set(LWS_WITH_POLL 1)
# Select features recommended for PC distro packaging
#
option(LWS_WITH_DISTRO_RECOMMENDED "Enable features recommended for distro packaging" OFF)
option(LWS_FOR_GITOHASHI "Enable features recommended for use with gitohashi" OFF)
#
# Major individual features
@ -24,13 +25,13 @@ option(LWS_WITH_DISTRO_RECOMMENDED "Enable features recommended for distro packa
option(LWS_ROLE_H1 "Compile with support for http/1 (needed for ws)" ON)
option(LWS_ROLE_WS "Compile with support for websockets" ON)
option(LWS_ROLE_DBUS "Compile with support for DBUS" OFF)
option(LWS_WITH_HTTP2 "Compile with server support for HTTP/2" OFF)
option(LWS_WITH_HTTP2 "Compile with server support for HTTP/2" ON)
option(LWS_WITH_LWSWS "Libwebsockets Webserver" OFF)
option(LWS_WITH_CGI "Include CGI (spawn process with network-connected stdin/out/err) APIs" OFF)
option(LWS_IPV6 "Compile with support for ipv6" OFF)
option(LWS_UNIX_SOCK "Compile with support for UNIX domain socket" ON)
option(LWS_UNIX_SOCK "Compile with support for UNIX domain socket" OFF)
option(LWS_WITH_PLUGINS "Support plugins for protocols and extensions" OFF)
option(LWS_WITH_HTTP_PROXY "Support for HTTP proxying" ON)
option(LWS_WITH_HTTP_PROXY "Support for HTTP proxying" OFF)
option(LWS_WITH_ZIP_FOPS "Support serving pre-zipped files" OFF)
option(LWS_WITH_SOCKS5 "Allow use of SOCKS5 proxy on client connections" OFF)
option(LWS_WITH_GENERIC_SESSIONS "With the Generic Sessions plugin" OFF)
@ -38,12 +39,12 @@ option(LWS_WITH_PEER_LIMITS "Track peers and restrict resources a single peer ca
option(LWS_WITH_ACCESS_LOG "Support generating Apache-compatible access logs" OFF)
option(LWS_WITH_RANGES "Support http ranges (RFC7233)" OFF)
option(LWS_WITH_SERVER_STATUS "Support json + jscript server monitoring" OFF)
option(LWS_WITH_THREADPOOL "Managed worker thread pool support (relies on pthreads)" ON)
option(LWS_WITH_THREADPOOL "Managed worker thread pool support (relies on pthreads)" OFF)
option(LWS_WITH_HTTP_STREAM_COMPRESSION "Support HTTP stream compression" OFF)
option(LWS_WITH_HTTP_BROTLI "Also offer brotli http stream compression (requires LWS_WITH_HTTP_STREAM_COMPRESSION)" OFF)
option(LWS_WITH_ACME "Enable support for ACME automatic cert acquisition + maintenance (letsencrypt etc)" OFF)
option(LWS_WITH_HUBBUB "Enable libhubbub rewriting support" OFF)
option(LWS_WITH_FTS "Full Text Search support" ON)
option(LWS_WITH_FTS "Full Text Search support" OFF)
#
# TLS library options... all except mbedTLS are basically OpenSSL variants.
#
@ -112,11 +113,22 @@ option(LWS_WITH_EXPORT_LWSTARGETS "Export libwebsockets CMake targets. Disable
option(LWS_REPRODUCIBLE "Build libwebsockets reproducible. It removes the build user and hostname from the build" ON)
option(LWS_WITH_MINIMAL_EXAMPLES "Also build the normally standalone minimal examples, for QA" OFF)
option(LWS_WITH_LWSAC "lwsac Chunk Allocation api" ON)
option(LWS_WITH_DISKCACHE "Hashed cache directory with lazy LRU deletion to size limit" OFF)
option(LWS_WITH_ASAN "Build with gcc runtime sanitizer options enabled (needs libasan)" OFF)
#
# End of user settings
#
if (LWS_FOR_GITOHASHI)
set(LWS_WITH_THREADPOOL 1)
set(LWS_WITH_HTTP2 1)
set(LWS_UNIX_SOCK 1)
set(LWS_WITH_HTTP_PROXY 1)
set(LWS_WITH_FTS 1)
set(LWS_WITH_DISKCACHE 1)
set(LWS_WITH_LWSAC 1)
set(LWS_WITH_LEJP_CONF 1)
endif()
if(LWS_WITH_DISTRO_RECOMMENDED)
set(LWS_WITH_HTTP2 1)
@ -135,6 +147,12 @@ if(LWS_WITH_DISTRO_RECOMMENDED)
set(LWS_WITHOUT_EXTENSIONS 0)
set(LWS_ROLE_DBUS 1)
set(LWS_WITH_FTS 1)
set(LWS_WITH_THREADPOOL 1)
set(LWS_UNIX_SOCK 1)
set(LWS_WITH_HTTP_PROXY 1)
set(LWS_WITH_DISKCACHE 1)
set(LWS_WITH_LWSAC 1)
set(LWS_WITH_LEJP_CONF 1)
endif()
# do you care about this? Then send me a patch where it disables it on travis
@ -227,6 +245,11 @@ if(GIT_EXECUTABLE)
endif()
# translate old functionality enables to set up ROLE enables so nothing changes
if (LWS_WITH_HTTP2 AND LWS_WITHOUT_SERVER)
set(LWS_WITH_HTTP2 0)
message("HTTP2 disabled due to LWS_WITHOUT_SERVER")
endif()
if (LWS_WITH_HTTP2)
set(LWS_ROLE_H2 1)
endif()
@ -238,10 +261,6 @@ if (NOT LWS_ROLE_WS)
set(LWS_WITHOUT_EXTENSIONS 1)
endif()
if (LWS_WITH_HTTP2 AND LWS_WITHOUT_SERVER)
message(FATAL_ERROR "HTTP2 can only be used with server at the moment")
endif()
include_directories(include plugins)
@ -767,10 +786,14 @@ CHECK_C_SOURCE_COMPILES("#include <stdint.h>
return 0;
}" LWS_HAS_INTPTR_T)
# These don't work Cross...
#CHECK_TYPE_SIZE(pid_t PID_T_SIZE)
#CHECK_TYPE_SIZE(size_t SIZE_T_SIZE)
#CHECK_TYPE_SIZE("void *" LWS_SIZEOFPTR LANGUAGE C)
set(CMAKE_REQUIRED_FLAGS "-pthread")
CHECK_C_SOURCE_COMPILES("#define _GNU_SOURCE
#include <pthread.h>
int main(void) {
pthread_t th = 0;
pthread_setname_np(th, NULL);
return 0;
}" LWS_HAS_PTHREAD_SETNAME_NP)
if (NOT PID_T_SIZE)
set(pid_t int)
@ -896,6 +919,11 @@ if (LWS_WITH_FTS)
lib/misc/fts/trie-fd.c)
endif()
if (LWS_WITH_DISKCACHE)
list(APPEND SOURCES
lib/misc/diskcache.c)
endif()
if (NOT LWS_WITHOUT_CLIENT)
list(APPEND SOURCES
lib/core/connect.c

2
cmake/lws_config_private.h.in
View file

@ -120,3 +120,5 @@
#cmakedefine inline ${inline}
#cmakedefine LWS_WITH_ZLIB
#cmakedefine LWS_HAS_PTHREAD_SETNAME_NP

1
include/libwebsockets.h
View file

@ -411,6 +411,7 @@ struct lws;
#include <libwebsockets/lws-tokenize.h>
#include <libwebsockets/lws-lwsac.h>
#include <libwebsockets/lws-fts.h>
#include <libwebsockets/lws-diskcache.h>
#if defined(LWS_WITH_TLS)

185
include/libwebsockets/lws-diskcache.h Normal file
View file

@ -0,0 +1,185 @@
/*
* libwebsockets - disk cache helpers
*
* Copyright (C) 2010-2018 Andy Green <andy@warmcat.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation:
* version 2.1 of the License.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301 USA
*
* included from libwebsockets.h
*/
/*! \defgroup diskcache LWS disk cache
* ## Disk cache API
*
* Lws provides helper apis useful if you need a disk cache containing hashed
* files and need to delete files from it on an LRU basis to keep it below some
* size limit.
*
* The API `lws_diskcache_prepare()` deals with creating the cache dir and
* 256 subdirs, which are used according to the first two chars of the hex
* hash of the cache file.
*
* `lws_diskcache_create()` and `lws_diskcache_destroy()` allocate and free
* an opaque struct that represents the disk cache.
*
* `lws_diskcache_trim()` should be called at eg, 1s intervals to perform the
* cache dir monitoring and LRU autodelete in the background lazily. It can
* be done in its own thread or on a timer... it monitors the directories in a
* stateful way that stats one or more file in the cache per call, and keeps
* a list of the oldest files as it goes. When it completes a scan, if the
* aggregate size is over the limit, it will delete oldest files first to try
* to keep it under the limit.
*
* The cache size monitoring is extremely efficient in time and memory even when
* the cache directory becomes huge.
*
* `lws_diskcache_query()` is used to determine if the file already exists in
* the cache, or if it must be created. If it must be created, then the file
* is opened using a temp name that must be converted to a findable name with
* `lws_diskcache_finalize_name()` when the generation of the file contents are
* complete. Aborted cached files that did not complete generation will be
* flushed by the LRU eventually. If the file already exists, it is 'touched'
* to make it new again and the fd returned.
*
*/
///@{
struct lws_diskcache_scan;
/**
* lws_diskcache_create() - creates an opaque struct representing the disk cache
*
* \param cache_dir_base: The cache dir path, eg `/var/cache/mycache`
* \param cache_size_limit: maximum size on disk the cache is allowed to use
*
* This returns an opaque `struct lws_diskcache_scan *` which represents the
* disk cache, the trim scanning state and so on. You should use
* `lws_diskcache_destroy()` to free it to destroy it.
*/
LWS_VISIBLE LWS_EXTERN struct lws_diskcache_scan *
lws_diskcache_create(const char *cache_dir_base, uint64_t cache_size_limit);
/**
* lws_diskcache_destroy() - destroys the pointer returned by ...create()
*
* \param lds: pointer to the pointer returned by lws_diskcache_create()
*
* Frees *lds and any allocations it did, and then sets *lds to NULL and
* returns.
*/
LWS_VISIBLE LWS_EXTERN void
lws_diskcache_destroy(struct lws_diskcache_scan **lds);
/**
* lws_diskcache_prepare() - ensures the cache dir structure exists on disk
*
* \param cache_base_dir: The cache dir path, eg `/var/cache/mycache`
* \param mode: octal dir mode to enforce, like 0700
* \param uid: uid the cache dir should belong to
*
* This should be called while your app is still privileged. It will create
* the cache directory structure on disk as necessary, enforce the given access
* mode on it and set the given uid as the owner. It won't make any trouble
* if the cache already exists.
*
* Typically the mode is 0700 and the owner is the user that your application
* will transition to use when it drops root privileges.
*/
LWS_VISIBLE LWS_EXTERN int
lws_diskcache_prepare(const char *cache_base_dir, int mode, int uid);
#define LWS_DISKCACHE_QUERY_NO_CACHE 0
#define LWS_DISKCACHE_QUERY_EXISTS 1
#define LWS_DISKCACHE_QUERY_CREATING 2
#define LWS_DISKCACHE_QUERY_ONGOING 3 /* something else is creating it */
/**
* lws_diskcache_query() - ensures the cache dir structure exists on disk
*
* \param lds: The opaque struct representing the disk cache
* \param is_bot: nonzero means the request is from a bot. Don't create new cache contents if so.
* \param hash_hex: hex string representation of the cache object hash
* \param _fd: pointer to the fd to be set
* \param cache: destination string to take the cache filepath
* \param cache_len: length of the buffer at `cache`
* \param extant_cache_len: pointer to a size_t to take any extant cached file size
*
* This function is called when you want to find if the hashed name already
* exists in the cache. The possibilities for the return value are
*
* - LWS_DISKCACHE_QUERY_NO_CACHE: It's not in the cache and you can't create
* it in the cache for whatever reason.
* - LWS_DISKCACHE_QUERY_EXISTS: It exists in the cache. It's open RDONLY and
* *_fd has been set to the file descriptor. *extant_cache_len has been set
* to the size of the cached file in bytes. cache has been set to the
* full filepath of the cached file. Closing _fd is your responsibility.
* - LWS_DISKCACHE_QUERY_CREATING: It didn't exist, but a temp file has been
* created in the cache and *_fd set to a file descriptor opened on it RDWR.
* You should create the contents, and call `lws_diskcache_finalize_name()`
* when it is done. Closing _fd is your responsibility.
* - LWS_DISKCACHE_QUERY_ONGOING: not returned by this api, but you may find it
* desirable to make a wrapper function which can handle another asynchronous
* process that is already creating the cached file. This can be used to
* indicate that situation externally... how to determine the same thing is
* already being generated is out of scope of this api.
*/
LWS_VISIBLE LWS_EXTERN int
lws_diskcache_query(struct lws_diskcache_scan *lds, int is_bot,
const char *hash_hex, int *_fd, char *cache, int cache_len,
size_t *extant_cache_len);
/**
* lws_diskcache_query() - ensures the cache dir structure exists on disk
*
* \param cache: The cache file temp name returned with LWS_DISKCACHE_QUERY_CREATING
*
* This renames the cache file you are creating to its final name. It should
* be called on the temp name returned by `lws_diskcache_query()` if it gave a
* LWS_DISKCACHE_QUERY_CREATING return, after you have filled the cache file and
* closed it.
*/
LWS_VISIBLE LWS_EXTERN int
lws_diskcache_finalize_name(char *cache);
/**
* lws_diskcache_trim() - performs one or more file checks in the cache for size management
*
* \param lds: The opaque object representing the cache
*
* This should be called periodically to statefully walk the cache on disk
* collecting the oldest files. When it has visited every file, if the cache
* is oversize it will delete the oldest files until it's back under size again.
*
* Each time it's called, it will look at one or more dir in the cache. If
* called when the cache is oversize, it increases the amount of work done each
* call until it is reduced again. Typically it will take 256 calls before it
* deletes anything, so if called once per second, it will delete files once
* every 4 minutes. Each call is very inexpensive both in memory and time.
*/
LWS_VISIBLE LWS_EXTERN int
lws_diskcache_trim(struct lws_diskcache_scan *lds);
/**
* lws_diskcache_secs_to_idle() - see how long to idle before calling trim
*
* \param lds: The opaque object representing the cache
*
* If the cache is undersize, there's no need to monitor it immediately. This
* suggests how long to "sleep" before calling `lws_diskcache_trim()` again.
*/
LWS_VISIBLE LWS_EXTERN int
lws_diskcache_secs_to_idle(struct lws_diskcache_scan *lds);

6
lib/core/libwebsockets.c
View file

@ -1382,7 +1382,11 @@ lws_get_network_wsi(struct lws *wsi)
return NULL;
#if defined(LWS_WITH_HTTP2)
if (!wsi->http2_substream && !wsi->client_h2_substream)
if (!wsi->http2_substream
#if !defined(LWS_NO_CLIENT)
&& !wsi->client_h2_substream
#endif
)
return wsi;
while (wsi->h2.parent_wsi)

476
lib/misc/diskcache.c Normal file
View file

@ -0,0 +1,476 @@
/*
* libwebsockets - disk cache helpers
*
* Copyright (C) 2018 Andy Green <andy@warmcat.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation:
* version 2.1 of the License.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301 USA
*/
#define _GNU_SOURCE
#include <pthread.h>
#include "core/private.h"
#include <string.h>
#include <stdio.h>
#include <unistd.h>
#include <fcntl.h>
#include <dirent.h>
#include <time.h>
#include <errno.h>
#include <stdarg.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
struct file_entry {
lws_list_ptr sorted;
lws_list_ptr prev;
char name[64];
time_t modified;
size_t size;
};
struct lws_diskcache_scan {
struct file_entry *batch;
const char *cache_dir_base;
lws_list_ptr head;
time_t last_scan_completed;
uint64_t agg_size;
uint64_t cache_size_limit;
uint64_t avg_size;
uint64_t cache_tries;
uint64_t cache_hits;
int cache_subdir;
int batch_in_use;
int agg_file_count;
int secs_waiting;
};
#define KIB (1024)
#define MIB (KIB * KIB)
#define lp_to_fe(p, _n) lws_list_ptr_container(p, struct file_entry, _n)
static const char *hex = "0123456789abcdef";
#define BATCH_COUNT 128
static int
fe_modified_sort(lws_list_ptr a, lws_list_ptr b)
{
struct file_entry *p1 = lp_to_fe(a, sorted), *p2 = lp_to_fe(b, sorted);
return p2->modified - p1->modified;
}
struct lws_diskcache_scan *
lws_diskcache_create(const char *cache_dir_base, uint64_t cache_size_limit)
{
struct lws_diskcache_scan *lds = lws_malloc(sizeof(*lds), "cachescan");
if (!lds)
return NULL;
memset(lds, 0, sizeof(*lds));
lds->cache_dir_base = cache_dir_base;
lds->cache_size_limit = cache_size_limit;
return lds;
}
void
lws_diskcache_destroy(struct lws_diskcache_scan **lds)
{
if ((*lds)->batch)
lws_free((*lds)->batch);
lws_free(*lds);
*lds = NULL;
}
int
lws_diskcache_prepare(const char *cache_base_dir, int mode, int uid)
{
char dir[256];
int n, m;
(void)mkdir(cache_base_dir, mode);
if (chown(cache_base_dir, uid, -1))
lwsl_err("%s: %s: unable to chown %d\n", __func__,
cache_base_dir, uid);
for (n = 0; n < 16; n++) {
lws_snprintf(dir, sizeof(dir), "%s/%c", cache_base_dir, hex[n]);
(void)mkdir(dir, mode);
if (chown(dir, uid, -1))
lwsl_err("%s: %s: unable to chown %d\n", __func__,
dir, uid);
for (m = 0; m < 16; m++) {
lws_snprintf(dir, sizeof(dir), "%s/%c/%c",
cache_base_dir, hex[n], hex[m]);
(void)mkdir(dir, mode);
if (chown(dir, uid, -1))
lwsl_err("%s: %s: unable to chown %d\n",
__func__, dir, uid);
}
}
return 0;
}
/* copies and then truncates the incoming name, and renames the file at the
* untruncated path to have the new truncated name */
int
lws_diskcache_finalize_name(char *cache)
{
char ren[256], *p;
strncpy(ren, cache, sizeof(ren) - 1);
ren[sizeof(ren) - 1] = '\0';
p = strchr(cache, '~');
if (p) {
*p = '\0';
if (rename(ren, cache)) {
lwsl_err("%s: problem renaming %s to %s\n", __func__,
ren, cache);
return 1;
}
return 0;
}
return 1;
}
int
lws_diskcache_query(struct lws_diskcache_scan *lds, int is_bot,
const char *hash_hex, int *_fd, char *cache, int cache_len,
size_t *extant_cache_len)
{
struct stat s;
int n;
/* caching is disabled? */
if (!lds->cache_dir_base)
return LWS_DISKCACHE_QUERY_NO_CACHE;
if (!is_bot)
lds->cache_tries++;
n = lws_snprintf(cache, cache_len, "%s/%c/%c/%s", lds->cache_dir_base,
hash_hex[0], hash_hex[1], hash_hex);
lwsl_info("%s: job cache %s\n", __func__, cache);
*_fd = open(cache, O_RDONLY);
if (*_fd >= 0) {
int fd;
if (!is_bot)
lds->cache_hits++;
if (fstat(*_fd, &s)) {
close(*_fd);
return LWS_DISKCACHE_QUERY_NO_CACHE;
}
*extant_cache_len = (size_t)s.st_size;
/* "touch" the hit cache file so it's last for LRU now */
fd = open(cache, O_RDWR);
if (fd >= 0)
close(fd);
return LWS_DISKCACHE_QUERY_EXISTS;
}
/* bots are too random to pollute the cache with their antics */
if (is_bot)
return LWS_DISKCACHE_QUERY_NO_CACHE;
/* let's create it first with a unique temp name */
lws_snprintf(cache + n, cache_len - n, "~%d-%p", (int)getpid(),
extant_cache_len);
*_fd = open(cache, O_RDWR | O_CREAT | O_TRUNC, 0600);
if (*_fd < 0) {
/* well... ok... we will proceed without cache then... */
lwsl_notice("%s: Problem creating cache %s: errno %d\n",
__func__, cache, errno);
return LWS_DISKCACHE_QUERY_NO_CACHE;
}
return LWS_DISKCACHE_QUERY_CREATING;
}
int
lws_diskcache_secs_to_idle(struct lws_diskcache_scan *lds)
{
return lds->secs_waiting;
}
/*
* The goal is to collect the oldest BATCH_COUNT filepaths and filesizes from
* the dirs under the cache dir. Since we don't need or want a full list of
* files in there in memory at once, we restrict the linked-list size to
* BATCH_COUNT entries, and once it is full, simply ignore any further files
* that are newer than the newest one on that list. Files older than the
* newest guy already on the list evict the newest guy already on the list
* and are sorted into the correct order. In this way no matter the number
* of files to be processed the memory requirement is fixed at BATCH_COUNT
* struct file_entry-s.
*
* The oldest subset of BATCH_COUNT files are sorted into the cd->batch
* allocation in more recent -> least recent order.
*
* We want to track the total size of all files we saw as well, so we know if
* we need to actually do anything yet to restrict how much space it's taking
* up.
*
* And we want to do those things statefully and incrementally instead of one
* big atomic operation, since the user may want a huge cache, so we look in
* one cache dir at a time and track state in the repodir struct.
*
* When we have seen everything, we add the doubly-linked prev pointers and then
* if we are over the limit, start deleting up to BATCH_COUNT files working back
* from the end.
*/
int
lws_diskcache_trim(struct lws_diskcache_scan *lds)
{
size_t cache_size_limit = lds->cache_size_limit;
char dirpath[132], filepath[132 + 32];
lws_list_ptr lp, op = NULL;
int files_trimmed = 0;
struct file_entry *p;
int fd, n, ret = -1;
size_t trimmed = 0;
struct dirent *de;
struct stat s;
DIR *dir;
if (!lds->cache_subdir) {
if (lds->last_scan_completed + lds->secs_waiting > time(NULL))
return 0;
lds->batch = lws_malloc(sizeof(struct file_entry) *
BATCH_COUNT, "cache_trim");
if (!lds->batch) {
lwsl_err("%s: OOM\n", __func__);
return 1;
}
lds->agg_size = 0;
lds->head = NULL;
lds->batch_in_use = 0;
lds->agg_file_count = 0;
}
lws_snprintf(dirpath, sizeof(dirpath), "%s/%c/%c",
lds->cache_dir_base, hex[(lds->cache_subdir >> 4) & 15],
hex[lds->cache_subdir & 15]);
dir = opendir(dirpath);
if (!dir) {
lwsl_err("Unable to walk repo dir '%s'\n",
lds->cache_dir_base);
return -1;
}
do {
de = readdir(dir);
if (!de)
break;
if (de->d_type != DT_REG)
continue;
lds->agg_file_count++;
lws_snprintf(filepath, sizeof(filepath), "%s/%s", dirpath,
de->d_name);
fd = open(filepath, O_RDONLY);
if (fd < 0) {
lwsl_err("%s: cannot open %s\n", __func__, filepath);
continue;
}
n = fstat(fd, &s);
close(fd);
if (n) {
lwsl_notice("%s: cannot stat %s\n", __func__, filepath);
continue;
}
lds->agg_size += s.st_size;
if (lds->batch_in_use == BATCH_COUNT) {
/*
* once we filled up the batch with candidates, we don't
* need to consider any files newer than the newest guy
* on the list...
*/
if (lp_to_fe(lds->head, sorted)->modified < s.st_mtime)
continue;
/*
* ... and if we find an older file later, we know it
* will be replacing the newest guy on the list, so use
* that directly...
*/
p = lds->head;
lds->head = p->sorted;
} else
/* we are still accepting anything to fill the batch */
p = &lds->batch[lds->batch_in_use++];
p->sorted = NULL;
strncpy(p->name, de->d_name, sizeof(p->name) - 1);
p->name[sizeof(p->name) - 1] = '\0';
p->modified = s.st_mtime;
p->size = s.st_size;
lws_list_ptr_insert(&lds->head, &p->sorted, fe_modified_sort);
} while (de);
ret = 0;
lds->cache_subdir++;
if (lds->cache_subdir != 0x100)
goto done;
/* we completed the whole scan... */
/* if really no guidence, then 256MiB */
if (!cache_size_limit)
cache_size_limit = 256 * 1024 * 1024;
if (lds->agg_size > cache_size_limit) {
/* apply prev pointers to make the list doubly-linked */
lp = lds->head;
while (lp) {
p = lp_to_fe(lp, sorted);
p->prev = op;
op = &p->prev;
lp = p->sorted;
}
/*
* reverse the list (start from tail, now traverse using
* .prev)... it's oldest-first now...
*/
lp = op;
while (lp && lds->agg_size > cache_size_limit) {
p = lp_to_fe(lp, prev);
lws_snprintf(filepath, sizeof(filepath), "%s/%c/%c/%s",
lds->cache_dir_base, p->name[0],
p->name[1], p->name);
if (!unlink(filepath)) {
lds->agg_size -= p->size;
trimmed += p->size;
files_trimmed++;
} else
lwsl_notice("%s: Failed to unlink %s\n",
__func__, filepath);
lp = p->prev;
}
if (files_trimmed)
lwsl_notice("%s: %s: trimmed %d files totalling "
"%lldKib, leaving %lldMiB\n", __func__,
lds->cache_dir_base, files_trimmed,
((unsigned long long)trimmed) / KIB,
((unsigned long long)lds->agg_size) / MIB);
}
if (lds->agg_size && lds->agg_file_count)
lds->avg_size = lds->agg_size / lds->agg_file_count;
/*
* estimate how long we can go before scanning again... default we need
* to start again immediately
*/
lds->last_scan_completed = time(NULL);
lds->secs_waiting = 1;
if (lds->agg_size < cache_size_limit) {
uint64_t avg = 4096, capacity, projected;
/* let's use 80% of the real average for margin */
if (lds->agg_size && lds->agg_file_count)
avg = ((lds->agg_size * 8) / lds->agg_file_count) / 10;
/*
* if we collected BATCH_COUNT files of the average size,
* how much can we clean up in 256s?
*/
capacity = avg * BATCH_COUNT;
/*
* if the cache grew by 10%, would we hit the limit even then?
*/
projected = (lds->agg_size * 11) / 10;
if (projected < cache_size_limit)
/* no... */
lds->secs_waiting = (256 / 2) * ((cache_size_limit -
projected) / capacity);
/*
* large waits imply we may not have enough info yet, so
* check once an hour at least.
*/
if (lds->secs_waiting > 3600)
lds->secs_waiting = 3600;
} else
lds->secs_waiting = 0;
lwsl_info("%s: cache %s: %lldKiB / %lldKiB, next scan %ds\n", __func__,
lds->cache_dir_base,
(unsigned long long)lds->agg_size / KIB,
(unsigned long long)cache_size_limit / KIB,
lds->secs_waiting);
lws_free(lds->batch);
lds->batch = NULL;
lds->cache_subdir = 0;
done:
closedir(dir);
return ret;
}

14
lib/misc/threadpool/threadpool.c
View file

@ -19,9 +19,11 @@
* MA 02110-1301 USA
*/
#define _GNU_SOURCE
#include <pthread.h>
#include "core/private.h"
#include <pthread.h>
#include <string.h>
#include <stdio.h>
@ -668,14 +670,22 @@ lws_threadpool_create(struct lws_context *context,
pthread_cond_init(&tp->wake_idle, NULL);
for (n = 0; n < args->threads; n++) {
#if defined(LWS_HAS_PTHREAD_SETNAME_NP)
char name[16];
#endif
tp->pool_list[n].tp = tp;
tp->pool_list[n].worker_index = n;
pthread_mutex_init(&tp->pool_list[n].lock, NULL);
if (pthread_create(&tp->pool_list[n].thread, NULL,
lws_threadpool_worker, &tp->pool_list[n])) {
lwsl_err("thread creation failed\n");
} else
} else {
#if defined(LWS_HAS_PTHREAD_SETNAME_NP)
lws_snprintf(name, sizeof(name), "%s-%d", tp->name, n);
pthread_setname_np(tp->pool_list[n].thread, name);
#endif
tp->threads_in_pool++;
}
}
return tp;

20
lib/roles/h2/http2.c
View file

@ -245,7 +245,9 @@ lws_wsi_h2_adopt(struct lws *parent_wsi, struct lws *wsi)
/* sid is set just before issuing the headers, ensuring monoticity */
wsi->seen_nonpseudoheader = 0;
#if !defined(LWS_NO_CLIENT)
wsi->client_h2_substream = 1;
#endif
wsi->h2.initialized = 1;
wsi->h2.parent_wsi = parent_wsi;
@ -848,7 +850,11 @@ lws_h2_parse_frame_header(struct lws *wsi)
else {
/* if it's data, either way no swsi means CLOSED state */
if (h2n->type == LWS_H2_FRAME_TYPE_DATA) {
if (h2n->sid <= h2n->highest_sid_opened && wsi->client_h2_alpn) {
if (h2n->sid <= h2n->highest_sid_opened
#if !defined(LWS_NO_CLIENT)
&& wsi->client_h2_alpn
#endif
) {
lwsl_notice("ignoring straggling data\n");
h2n->type = LWS_H2_FRAME_TYPE_COUNT; /* ie, IGNORE */
} else {
@ -1338,6 +1344,7 @@ lws_h2_parse_end_of_frame(struct lws *wsi)
lwsl_info("http req, wsi=%p, h2n->swsi=%p\n", wsi, h2n->swsi);
h2n->swsi->hdr_parsing_completed = 1;
#if !defined(LWS_NO_CLIENT)
if (h2n->swsi->client_h2_substream) {
if (lws_client_interpret_server_handshake(h2n->swsi)) {
lws_h2_rst_stream(h2n->swsi, H2_ERR_STREAM_CLOSED,
@ -1345,6 +1352,7 @@ lws_h2_parse_end_of_frame(struct lws *wsi)
break;
}
}
#endif
if (lws_hdr_extant(h2n->swsi, WSI_TOKEN_HTTP_CONTENT_LENGTH)) {
h2n->swsi->http.rx_content_length = atoll(
@ -1401,10 +1409,12 @@ lws_h2_parse_end_of_frame(struct lws *wsi)
break;
}
#if !defined(LWS_NO_CLIENT)
if (h2n->swsi->client_h2_substream) {
lwsl_info("%s: headers: client path\n", __func__);
break;
}
#endif
if (!lws_hdr_total_length(h2n->swsi, WSI_TOKEN_HTTP_COLON_PATH) ||
!lws_hdr_total_length(h2n->swsi, WSI_TOKEN_HTTP_COLON_METHOD) ||
@ -1465,6 +1475,7 @@ lws_h2_parse_end_of_frame(struct lws *wsi)
h2n->swsi->h2.h2_state == LWS_H2_STATE_HALF_CLOSED_LOCAL)
lws_h2_state(h2n->swsi, LWS_H2_STATE_CLOSED);
#if !defined(LWS_NO_CLIENT)
/*
* client... remote END_STREAM implies we weren't going to
* send anything else anyway.
@ -1494,6 +1505,7 @@ lws_h2_parse_end_of_frame(struct lws *wsi)
lwsl_debug("tx completed returned close\n");
}
}
#endif
break;
case LWS_H2_FRAME_TYPE_PING:
@ -1788,7 +1800,7 @@ lws_h2_parser(struct lws *wsi, unsigned char *in, lws_filepos_t inlen,
n = h2n->length - h2n->count + 1;
lwsl_debug("---- restricting len to %d vs %ld\n", n, (long)inlen + 1);
}
#if !defined(LWS_NO_CLIENT)
if (h2n->swsi->client_h2_substream) {
m = user_callback_handle_rxflow(
@ -1807,7 +1819,9 @@ lws_h2_parser(struct lws *wsi, unsigned char *in, lws_filepos_t inlen,
}
break;
} else {
} else
#endif
{
if (lwsi_state(h2n->swsi) == LRS_DEFERRING_ACTION) {
// lwsl_notice("appending because we are in LRS_DEFERRING_ACTION\n");

15
lib/roles/h2/ops-h2.c
View file

@ -605,7 +605,11 @@ rops_close_kill_connection_h2(struct lws *wsi, enum lws_close_status reason)
} lws_end_foreach_llp(w, h2.sibling_list);
}
if (wsi->upgraded_to_http2 || wsi->http2_substream || wsi->client_h2_substream) {
if (wsi->upgraded_to_http2 || wsi->http2_substream
#if !defined(LWS_NO_CLIENT)
|| wsi->client_h2_substream
#endif
) {
lwsl_info("closing %p: parent %p\n", wsi, wsi->h2.parent_wsi);
if (wsi->h2.child_list && lwsl_visible(LLL_INFO)) {
@ -645,7 +649,11 @@ rops_close_kill_connection_h2(struct lws *wsi, enum lws_close_status reason)
wsi->h2.h2n->pps = NULL;
}
if ((wsi->client_h2_substream || wsi->http2_substream) &&
if ((
#if !defined(LWS_NO_CLIENT)
wsi->client_h2_substream ||
#endif
wsi->http2_substream) &&
wsi->h2.parent_wsi) {
lwsl_info(" %p: disentangling from siblings\n", wsi);
lws_start_foreach_llp(struct lws **, w,
@ -732,8 +740,9 @@ rops_callback_on_writable_h2(struct lws *wsi)
/* for network action, act only on the network wsi */
wsi = network_wsi;
if (already && !wsi->client_h2_alpn
if (already
#if !defined(LWS_NO_CLIENT)
&& !wsi->client_h2_alpn
&& !wsi->client_h2_substream
#endif
)

2
lib/roles/http/client/client.c
View file

@ -352,7 +352,9 @@ start_ws_handshake:
* So this is it, we are an h2 master client connection
* now, not an h1 client connection.
*/
#if defined (LWS_WITH_TLS)
lws_tls_server_conn_alpn(wsi);
#endif
/* send the H2 preface to legitimize the connection */
if (lws_h2_issue_preface(wsi)) {

1
minimal-examples/http-server/minimal-http-server-fulltext-search/CMakeLists.txt
View file

@ -67,6 +67,7 @@ ENDMACRO()
set(requirements 1)
require_lws_config(LWS_ROLE_H1 1 requirements)
require_lws_config(LWS_WITH_FTS 1 requirements)
require_lws_config(LWS_WITHOUT_SERVER 0 requirements)
if (requirements)

30
minimal-examples/selftests.sh
View file

@ -24,24 +24,30 @@ MINEX=`dirname $0`
MINEX=`realpath $MINEX`
TESTS=0
for i in `find $MINEX -name selftest.sh` ; do
C=`cat $i | grep COUNT_TESTS= | cut -d= -f2`
TESTS=$(( $TESTS + $C ))
BN=`echo -n "$i" | sed "s/\/[^\/]*\$//g" | sed "s/.*\///g"`
if [ -e `pwd`/bin/lws-$BN ] ; then
C=`cat $i | grep COUNT_TESTS= | cut -d= -f2`
TESTS=$(( $TESTS + $C ))
fi
done
FAILS=0
WH=1
for i in `find $MINEX -name selftest.sh` ; do
C=`cat $i | grep COUNT_TESTS= | cut -d= -f2`
sh $i `pwd`/bin $LOGGING_PATH $WH $TESTS $MINEX
FAILS=$(( $FAILS + $? ))
L=`ps fax | grep lws- | cut -d' ' -f2`
kill $L 2>/dev/null
kill -9 $L 2>/dev/null
wait $L 2>/dev/null
WH=$(( $WH + $C ))
BN=`echo -n "$i" | sed "s/\/[^\/]*\$//g" | sed "s/.*\///g"`
if [ -e `pwd`/bin/lws-$BN ] ; then
C=`cat $i | grep COUNT_TESTS= | cut -d= -f2`
sh $i `pwd`/bin $LOGGING_PATH $WH $TESTS $MINEX
FAILS=$(( $FAILS + $? ))
L=`ps fax | grep lws- | cut -d' ' -f2`
kill $L 2>/dev/null
kill -9 $L 2>/dev/null
wait $L 2>/dev/null
WH=$(( $WH + $C ))
fi
done
if [ $FAILS -eq 0 ] ; then