From 2224146b2b3a300e80f956c6623e9d6b569ddc8e Mon Sep 17 00:00:00 2001 From: Adam Sutton Date: Tue, 22 May 2012 13:56:23 +0100 Subject: [PATCH] Some further updates to xmltv importer, also added parsing stats. --- src/epg.c | 22 ++++++++---- src/epg.h | 15 +++++--- src/epggrab.c | 43 ++++++++++++++++++----- src/epggrab.h | 21 +++++++++++- src/epggrab/pyepg.c | 25 +++++++------- src/epggrab/xmltv.c | 84 +++++++++++++++++++++++++++++++-------------- 6 files changed, 152 insertions(+), 58 deletions(-) diff --git a/src/epg.c b/src/epg.c index d4a466e4..2120d7bb 100644 --- a/src/epg.c +++ b/src/epg.c @@ -214,12 +214,11 @@ void epg_init ( void ) remain = st.st_size; while ( remain > 4 ) { int msglen = (rp[0] << 24) | (rp[1] << 16) | (rp[2] << 8) | rp[3]; - printf("msglen = %d\n", msglen); remain -= 4; rp += 4; htsmsg_t *m = htsmsg_binary_deserialize(rp, msglen, NULL); if(m) { - htsmsg_print(m); + //htsmsg_print(m); htsmsg_destroy(m); } rp += msglen; @@ -258,7 +257,8 @@ void epg_rem_channel ( channel_t *ch ) * Brand * *************************************************************************/ -epg_brand_t* epg_brand_find_by_uri ( const char *id, int create ) +epg_brand_t* epg_brand_find_by_uri + ( const char *id, int create, int *save ) { epg_brand_t *eb; static epg_brand_t* skel = NULL; @@ -279,6 +279,7 @@ epg_brand_t* epg_brand_find_by_uri ( const char *id, int create ) eb = skel; skel = NULL; eb->eb_uri = strdup(id); + *save |= 1; } } @@ -400,7 +401,8 @@ htsmsg_t *epg_brand_serialize ( epg_brand_t *brand ) * Season * *************************************************************************/ -epg_season_t* epg_season_find_by_uri ( const char *id, int create ) +epg_season_t* epg_season_find_by_uri + ( const char *id, int create, int *save ) { epg_season_t *es; static epg_season_t* skel = NULL; @@ -421,6 +423,7 @@ epg_season_t* epg_season_find_by_uri ( const char *id, int create ) es = skel; skel = NULL; es->es_uri = strdup(id); + *save |= 1; } } @@ -529,7 +532,8 @@ htsmsg_t *epg_season_serialize ( epg_season_t *season ) * Episode * *************************************************************************/ -epg_episode_t* epg_episode_find_by_uri ( const char *id, int create ) +epg_episode_t* epg_episode_find_by_uri + ( const char *id, int create, int *save ) { epg_episode_t *ee; static epg_episode_t* skel = NULL; @@ -550,6 +554,7 @@ epg_episode_t* epg_episode_find_by_uri ( const char *id, int create ) ee = skel; skel = NULL; ee->ee_uri = strdup(id); + *save |= 1; } } @@ -745,7 +750,7 @@ static int _epg_broadcast_idx = 0; // // Note: do we need to pass in stop? epg_broadcast_t* epg_broadcast_find_by_time - ( epg_channel_t *channel, time_t start, time_t stop, int create ) + ( epg_channel_t *channel, time_t start, time_t stop, int create, int *save ) { epg_broadcast_t *eb; static epg_broadcast_t *skel = NULL; @@ -771,6 +776,7 @@ epg_broadcast_t* epg_broadcast_find_by_time if ( eb == NULL ) { eb = skel; skel = NULL; + *save |= 1; _epg_broadcast_idx++; } } @@ -839,7 +845,8 @@ static void _epg_channel_link ( epg_channel_t *ec ) } } -epg_channel_t* epg_channel_find_by_uri ( const char *id, int create ) +epg_channel_t* epg_channel_find_by_uri + ( const char *id, int create, int *save ) { epg_channel_t *ec; static epg_channel_t *skel = NULL; @@ -861,6 +868,7 @@ epg_channel_t* epg_channel_find_by_uri ( const char *id, int create ) skel = NULL; ec->ec_uri = strdup(id); LIST_INSERT_HEAD(&epg_unlinked_channels1, ec, ec_ulink); + *save |= 1; } } diff --git a/src/epg.h b/src/epg.h index 354ba9be..b642a17a 100644 --- a/src/epg.h +++ b/src/epg.h @@ -62,7 +62,8 @@ typedef struct epg_brand } epg_brand_t; /* Lookup */ -epg_brand_t *epg_brand_find_by_uri ( const char *uri, int create ); +epg_brand_t *epg_brand_find_by_uri + ( const char *uri, int create, int *save ); /* Mutators */ int epg_brand_set_title ( epg_brand_t *b, const char *title ) @@ -107,7 +108,8 @@ typedef struct epg_season } epg_season_t; /* Lookup */ -epg_season_t *epg_season_find_by_uri ( const char *uri, int create ); +epg_season_t *epg_season_find_by_uri + ( const char *uri, int create, int *save ); /* Mutators */ int epg_season_set_summary ( epg_season_t *s, const char *summary ) @@ -158,7 +160,8 @@ typedef struct epg_episode } epg_episode_t; /* Lookup */ -epg_episode_t *epg_episode_find_by_uri ( const char *uri, int create ); +epg_episode_t *epg_episode_find_by_uri + ( const char *uri, int create, int *save ); /* Mutators */ int epg_episode_set_title ( epg_episode_t *e, const char *title ) @@ -227,7 +230,8 @@ typedef struct epg_broadcast } epg_broadcast_t; /* Lookup */ -epg_broadcast_t *epg_broadcast_find_by_time ( epg_channel_t *ch, time_t start, time_t stop, int create ); +epg_broadcast_t *epg_broadcast_find_by_time + ( epg_channel_t *ch, time_t start, time_t stop, int create, int *save ); epg_broadcast_t *epg_broadcast_find_by_id ( int id ); /* Mutators */ @@ -263,7 +267,8 @@ typedef struct epg_channel } epg_channel_t; /* Lookup */ -epg_channel_t *epg_channel_find_by_uri ( const char *uri, int create ); +epg_channel_t *epg_channel_find_by_uri + ( const char *uri, int create, int *save ); /* Mutators */ int epg_channel_set_name ( epg_channel_t *c, const char *n ) diff --git a/src/epggrab.c b/src/epggrab.c index 9196d1a5..8245bf5f 100644 --- a/src/epggrab.c +++ b/src/epggrab.c @@ -40,9 +40,9 @@ void epggrab_init ( void ) { /* Defaults */ epggrab_advanced = 0; - epggrab_eit = 1; // on air grab enabled - epggrab_interval = 12; // hours - epggrab_module = NULL; // disabled + epggrab_eit = 1; // on air grab enabled + epggrab_interval = 12 * 3600; // hours + epggrab_module = NULL; // disabled /* Initialise modules */ epggrab_module_pyepg = pyepg_init(); @@ -64,6 +64,8 @@ static void _epggrab_module_run ( epggrab_module_t *mod, const char *opts ) int save = 0; time_t tm1, tm2; htsmsg_t *data; + epggrab_stats_t stats; + memset(&stats, 0, sizeof(stats)); /* Check */ if ( !mod ) return; @@ -72,15 +74,40 @@ static void _epggrab_module_run ( epggrab_module_t *mod, const char *opts ) time(&tm1); data = mod->grab(opts); time(&tm2); - if ( !data ) { - tvhlog(LOG_WARNING, mod->name(), "grab returned no data"); - } else { + + /* Process */ + if ( data ) { + //htsmsg_print(data); tvhlog(LOG_DEBUG, mod->name(), "grab took %d seconds", tm2 - tm1); pthread_mutex_lock(&global_lock); - save = mod->parse(data); - if (save) epg_updated(); + time(&tm1); + save = mod->parse(data, &stats); + time(&tm2); pthread_mutex_unlock(&global_lock); htsmsg_destroy(data); + tvhlog(LOG_DEBUG, mod->name(), "parse took %d seconds", tm2 - tm1); + tvhlog(LOG_DEBUG, mod->name(), " channels tot=%5d new=%5d mod=%5d", + stats.channels.total, stats.channels.created, + stats.channels.modified); + tvhlog(LOG_DEBUG, mod->name(), " brands tot=%5d new=%5d mod=%5d", + stats.brands.total, stats.brands.created, + stats.brands.modified); + tvhlog(LOG_DEBUG, mod->name(), " seasons tot=%5d new=%5d mod=%5d", + stats.seasons.total, stats.seasons.created, + stats.seasons.modified); + tvhlog(LOG_DEBUG, mod->name(), " episodes tot=%5d new=%5d mod=%5d", + stats.episodes.total, stats.episodes.created, + stats.episodes.modified); + tvhlog(LOG_DEBUG, mod->name(), " broadcasts tot=%5d new=%5d mod=%5d", + stats.broadcasts.total, stats.broadcasts.created, + stats.broadcasts.modified); + + /* Updated */ + // TODO: should epg_updated happen inside or outside the lock? + if (save) epg_updated(); + + } else { + tvhlog(LOG_WARNING, mod->name(), "grab returned no data"); } } diff --git a/src/epggrab.h b/src/epggrab.h index 7968d07e..2f4eefa4 100644 --- a/src/epggrab.h +++ b/src/epggrab.h @@ -8,6 +8,25 @@ * Type definitions * *************************************************************************/ +/* + * Grab statistics + */ +typedef struct epggrab_stats_part +{ + int created; + int modified; + int total; +} epggrab_stats_part_t; + +typedef struct epggrab_stats +{ + epggrab_stats_part_t channels; + epggrab_stats_part_t brands; + epggrab_stats_part_t seasons; + epggrab_stats_part_t episodes; + epggrab_stats_part_t broadcasts; +} epggrab_stats_t; + /* * Grabber base class */ @@ -17,7 +36,7 @@ typedef struct epggrab_module void (*enable) ( void ); void (*disable) ( void ); htsmsg_t* (*grab) ( const char *opts ); - int (*parse) ( htsmsg_t *data ); + int (*parse) ( htsmsg_t *data, epggrab_stats_t *stats ); } epggrab_module_t; /* diff --git a/src/epggrab/pyepg.c b/src/epggrab/pyepg.c index 4362daef..ed223cec 100644 --- a/src/epggrab/pyepg.c +++ b/src/epggrab/pyepg.c @@ -43,7 +43,7 @@ static int _pyepg_parse_channel ( htsmsg_t *data ) if ((tags = htsmsg_get_map(data, "tags")) == NULL) return 0; /* Find channel */ - if ((channel = epg_channel_find_by_uri(id, 1)) == NULL) return 0; + if ((channel = epg_channel_find_by_uri(id, 1, &save)) == NULL) return 0; // TODO: need to save if created /* Set name */ @@ -69,7 +69,7 @@ static int _pyepg_parse_brand ( htsmsg_t *data ) if ((tags = htsmsg_get_map(data, "tags")) == NULL) return 0; /* Find brand */ - if ((brand = epg_brand_find_by_uri(str, 1)) == NULL) return 0; + if ((brand = epg_brand_find_by_uri(str, 1, &save)) == NULL) return 0; // TODO: do we need to save if created? /* Set title */ @@ -113,12 +113,12 @@ static int _pyepg_parse_season ( htsmsg_t *data ) if ((tags = htsmsg_get_map(data, "tags")) == NULL) return 0; /* Find series */ - if ((season = epg_season_find_by_uri(str, 1)) == NULL) return 0; + if ((season = epg_season_find_by_uri(str, 1, &save)) == NULL) return 0; // TODO: do we need to save if created? /* Set brand */ if ((str = htsmsg_get_str(attr, "brand"))) { - if ((brand = epg_brand_find_by_uri(str, 0))) { + if ((brand = epg_brand_find_by_uri(str, 0, NULL))) { save |= epg_season_set_brand(season, brand, 1); } } @@ -170,19 +170,19 @@ static int _pyepg_parse_episode ( htsmsg_t *data ) if ((tags = htsmsg_get_map(data, "tags")) == NULL) return 0; /* Find episode */ - if ((episode = epg_episode_find_by_uri(str, 1)) == NULL) return 0; + if ((episode = epg_episode_find_by_uri(str, 1, &save)) == NULL) return 0; // TODO: do we need to save if created? /* Set brand */ if ((str = htsmsg_get_str(attr, "brand"))) { - if ((brand = epg_brand_find_by_uri(str, 0))) { + if ((brand = epg_brand_find_by_uri(str, 0, NULL))) { save |= epg_episode_set_brand(episode, brand, 1); } } /* Set season */ if ((str = htsmsg_get_str(attr, "series"))) { - if ((season = epg_season_find_by_uri(str, 0))) { + if ((season = epg_season_find_by_uri(str, 0, NULL))) { save |= epg_episode_set_season(episode, season, 1); } } @@ -236,14 +236,14 @@ static int _pyepg_parse_broadcast ( htsmsg_t *data, epg_channel_t *channel ) if ((stop = htsmsg_get_str(attr, "stop")) == NULL ) return 0; /* Find episode */ - if ((episode = epg_episode_find_by_uri(id, 1)) == NULL) return 0; + if ((episode = epg_episode_find_by_uri(id, 1, &save)) == NULL) return 0; /* Parse times */ if (!_pyepg_parse_time(start, &tm_start)) return 0; if (!_pyepg_parse_time(stop, &tm_stop)) return 0; /* Find broadcast */ - broadcast = epg_broadcast_find_by_time(channel, tm_start, tm_stop, 1); + broadcast = epg_broadcast_find_by_time(channel, tm_start, tm_stop, 1, &save); if ( broadcast == NULL ) return 0; /* Set episode */ @@ -266,7 +266,7 @@ static int _pyepg_parse_schedule ( htsmsg_t *data ) if ((attr = htsmsg_get_map(data, "attrib")) == NULL) return 0; if ((str = htsmsg_get_str(attr, "channel")) == NULL) return 0; - if ((channel = epg_channel_find_by_uri(str, 0)) == NULL) return 0; + if ((channel = epg_channel_find_by_uri(str, 0, NULL)) == NULL) return 0; if ((tags = htsmsg_get_map(data, "tags")) == NULL) return 0; HTSMSG_FOREACH(f, tags) { @@ -303,7 +303,8 @@ static int _pyepg_parse_epg ( htsmsg_t *data ) return save; } -static int _pyepg_parse ( htsmsg_t *data ) +// TODO: add stats updating +static int _pyepg_parse ( htsmsg_t *data, epggrab_stats_t *stats ) { htsmsg_t *tags, *epg; epggrab_module_t *mod; @@ -319,7 +320,7 @@ static int _pyepg_parse ( htsmsg_t *data ) /* XMLTV format */ if ((epg = htsmsg_get_map(tags, "tv")) != NULL) { mod = epggrab_module_find_by_name("xmltv"); - if (mod) return mod->parse(epg); + if (mod) return mod->parse(epg, stats); } return 0; diff --git a/src/epggrab/xmltv.c b/src/epggrab/xmltv.c index 004012ed..00672453 100644 --- a/src/epggrab/xmltv.c +++ b/src/epggrab/xmltv.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "htsmsg_xml.h" #include "settings.h" @@ -38,6 +39,24 @@ * Parsing * *************************************************************************/ +/** + * Hash the description to get a URI for episode + * + * Note: converts to an ASCII format + */ +static const char *_xmltv_hash ( const char *str ) +{ + size_t i; + static char ret[(MD5_DIGEST_LENGTH*2)+1]; + static unsigned char md5[MD5_DIGEST_LENGTH]; + (void)MD5((const unsigned char*)str, strlen(str), md5); + for ( i = 0; i < MD5_DIGEST_LENGTH; i++ ) { + sprintf(&ret[i*2], "%02X", md5[i]); + } + ret[MD5_DIGEST_LENGTH*2] = '\0'; + return ret; +} + /** * */ @@ -213,20 +232,28 @@ get_episode_info(htsmsg_t *tags, epg_episode_t *ee) */ static int _xmltv_parse_programme_tags(epg_channel_t *xc, htsmsg_t *tags, - time_t start, time_t stop) + time_t start, time_t stop, epggrab_stats_t *stats) { - int save = 0; + int save = 0, save2 = 0; epg_broadcast_t *ebc; epg_episode_t *ee; + const char *uri = NULL; const char *title = htsmsg_xml_get_cdata_str(tags, "title"); const char *desc = htsmsg_xml_get_cdata_str(tags, "desc"); #if TODO_EPG_GENRE const char *category = htsmsg_xml_get_cdata_str(tags, "category"); #endif - /* Generate an episode */ - // TODO: hash the description? - ee = epg_episode_find_by_uri("TODO", 1); + /* Generate a URI */ + if ( desc ) + uri = _xmltv_hash(desc); + // TODO: what to do otherwise? + if ( !uri ) return 0; + + /* Find episode */ + ee = epg_episode_find_by_uri(uri, 1, &save); + stats->episodes.total++; + if (save) stats->episodes.created++; if (title) save |= epg_episode_set_title(ee, title); if (desc) save |= epg_episode_set_description(ee, desc); #if TODO_EPG_GENRE @@ -235,12 +262,18 @@ _xmltv_parse_programme_tags(epg_channel_t *xc, htsmsg_t *tags, #if TODO_XMLTV_EP_NUMBERING _get_episode_info(tags, &episode); #endif + if (save) stats->episodes.modified++; /* Create broadcast */ - ebc = epg_broadcast_find_by_time(xc, start, stop, 1); - save |= epg_broadcast_set_episode(ebc, ee, 1); + ebc = epg_broadcast_find_by_time(xc, start, stop, 1, &save2); + if ( ebc != NULL ) { + stats->broadcasts.total++; + if (save2) stats->broadcasts.created++; + save2 |= epg_broadcast_set_episode(ebc, ee, 1); + if (save2) stats->broadcasts.modified++; + } - return save; + return save | save2; } @@ -248,7 +281,7 @@ _xmltv_parse_programme_tags(epg_channel_t *xc, htsmsg_t *tags, * Parse a tag from xmltv */ static int -_xmltv_parse_programme(htsmsg_t *body) +_xmltv_parse_programme(htsmsg_t *body, epggrab_stats_t *stats) { int save = 0; htsmsg_t *attribs, *tags; @@ -261,7 +294,7 @@ _xmltv_parse_programme(htsmsg_t *body) if((attribs = htsmsg_get_map(body, "attrib")) == NULL) return 0; if((tags = htsmsg_get_map(body, "tags")) == NULL) return 0; if((chid = htsmsg_get_str(attribs, "channel")) == NULL) return 0; - if((xc = epg_channel_find_by_uri(chid, 0)) == NULL) return 0; + if((xc = epg_channel_find_by_uri(chid, 0, NULL)) == NULL) return 0; if((s = htsmsg_get_str(attribs, "start")) == NULL) return 0; start = _xmltv_str2time(s); if((s = htsmsg_get_str(attribs, "stop")) == NULL) return 0; @@ -269,7 +302,7 @@ _xmltv_parse_programme(htsmsg_t *body) if(stop <= start || stop < dispatch_clock) return 0; - _xmltv_parse_programme_tags(xc, tags, start, stop); + save |= _xmltv_parse_programme_tags(xc, tags, start, stop, stats); return save; } @@ -277,26 +310,27 @@ _xmltv_parse_programme(htsmsg_t *body) * Parse a tag from xmltv */ static int -_xmltv_parse_channel(htsmsg_t *body) +_xmltv_parse_channel(htsmsg_t *body, epggrab_stats_t *stats) { int save =0; htsmsg_t *attribs, *tags;/*, *subtag;*/ - const char *id;/*TODO, *name, *icon;*/ + const char *id, *name; /*,icon;*/ epg_channel_t *xc; if(body == NULL) return 0; if((attribs = htsmsg_get_map(body, "attrib")) == NULL) return 0; if((id = htsmsg_get_str(attribs, "id")) == NULL) return 0; - if((xc = epg_channel_find_by_uri(id, 1)) == NULL) return 0; if((tags = htsmsg_get_map(body, "tags")) == NULL) return 0; - // TODO: save in find_by_uri + if((xc = epg_channel_find_by_uri(id, 1, &save)) == NULL) return 0; + stats->channels.total++; + if (save) stats->channels.created++; -#if TODO_EPG_CHANNEL_META - if((name = xmltv_get_cdata_by_tag(tags, "display-name")) != NULL) { + if((name = htsmsg_xml_get_cdata_str(tags, "display-name")) != NULL) { save |= epg_channel_set_name(xc, name); } +#if TODO_EPG_CHANNEL_META if((subtag = htsmsg_get_map(tags, "icon")) != NULL && (attribs = htsmsg_get_map(subtag, "attrib")) != NULL && (icon = htsmsg_get_str(attribs, "src")) != NULL) { @@ -310,6 +344,7 @@ _xmltv_parse_channel(htsmsg_t *body) } } #endif + if (save) stats->channels.modified++; return save; } @@ -317,7 +352,7 @@ _xmltv_parse_channel(htsmsg_t *body) * */ static int -_xmltv_parse_tv(htsmsg_t *body) +_xmltv_parse_tv(htsmsg_t *body, epggrab_stats_t *stats) { int save = 0; htsmsg_t *tags; @@ -328,19 +363,18 @@ _xmltv_parse_tv(htsmsg_t *body) HTSMSG_FOREACH(f, tags) { if(!strcmp(f->hmf_name, "channel")) { - save |= _xmltv_parse_channel(htsmsg_get_map_by_field(f)); + save |= _xmltv_parse_channel(htsmsg_get_map_by_field(f), stats); } else if(!strcmp(f->hmf_name, "programme")) { - save |= _xmltv_parse_programme(htsmsg_get_map_by_field(f)); + save |= _xmltv_parse_programme(htsmsg_get_map_by_field(f), stats); } } return save; } - /** * */ -static int _xmltv_parse ( htsmsg_t *data ) +static int _xmltv_parse ( htsmsg_t *data, epggrab_stats_t *stats ) { htsmsg_t *tags, *tv; @@ -350,7 +384,7 @@ static int _xmltv_parse ( htsmsg_t *data ) if((tv = htsmsg_get_map(tags, "tv")) == NULL) return 0; - return _xmltv_parse_tv(tv); + return _xmltv_parse_tv(tv, stats); } /* ************************************************************************ @@ -358,6 +392,7 @@ static int _xmltv_parse ( htsmsg_t *data ) * ***********************************************************************/ // TODO: config +// TODO: remove use of hardcoded xmltv script static epggrab_module_t xmltv_module; @@ -371,13 +406,12 @@ static htsmsg_t* _xmltv_grab ( const char *iopts ) size_t outlen; char *outbuf; char errbuf[100]; - const char *cmd = "/usr/bin/tv_grab_uk_rt"; + const char *cmd = "/home/aps/tmp/epg.sh";//usr/bin/tv_grab_uk_rt"; /* Debug */ tvhlog(LOG_DEBUG, "xmltv", "grab %s", cmd); /* Grab */ - /* TODO: using hardcoded xmltv command at the moment */ outlen = spawn_and_store_stdout(cmd, NULL, &outbuf); if ( outlen < 1 ) { tvhlog(LOG_ERR, "xmltv", "no output detected");