Some further updates to xmltv importer, also added parsing stats.

This commit is contained in:
Adam Sutton 2012-05-22 13:56:23 +01:00
parent 988eb8133a
commit 2224146b2b
6 changed files with 152 additions and 58 deletions

View file

@ -214,12 +214,11 @@ void epg_init ( void )
remain = st.st_size;
while ( remain > 4 ) {
int msglen = (rp[0] << 24) | (rp[1] << 16) | (rp[2] << 8) | rp[3];
printf("msglen = %d\n", msglen);
remain -= 4;
rp += 4;
htsmsg_t *m = htsmsg_binary_deserialize(rp, msglen, NULL);
if(m) {
htsmsg_print(m);
//htsmsg_print(m);
htsmsg_destroy(m);
}
rp += msglen;
@ -258,7 +257,8 @@ void epg_rem_channel ( channel_t *ch )
* Brand
* *************************************************************************/
epg_brand_t* epg_brand_find_by_uri ( const char *id, int create )
epg_brand_t* epg_brand_find_by_uri
( const char *id, int create, int *save )
{
epg_brand_t *eb;
static epg_brand_t* skel = NULL;
@ -279,6 +279,7 @@ epg_brand_t* epg_brand_find_by_uri ( const char *id, int create )
eb = skel;
skel = NULL;
eb->eb_uri = strdup(id);
*save |= 1;
}
}
@ -400,7 +401,8 @@ htsmsg_t *epg_brand_serialize ( epg_brand_t *brand )
* Season
* *************************************************************************/
epg_season_t* epg_season_find_by_uri ( const char *id, int create )
epg_season_t* epg_season_find_by_uri
( const char *id, int create, int *save )
{
epg_season_t *es;
static epg_season_t* skel = NULL;
@ -421,6 +423,7 @@ epg_season_t* epg_season_find_by_uri ( const char *id, int create )
es = skel;
skel = NULL;
es->es_uri = strdup(id);
*save |= 1;
}
}
@ -529,7 +532,8 @@ htsmsg_t *epg_season_serialize ( epg_season_t *season )
* Episode
* *************************************************************************/
epg_episode_t* epg_episode_find_by_uri ( const char *id, int create )
epg_episode_t* epg_episode_find_by_uri
( const char *id, int create, int *save )
{
epg_episode_t *ee;
static epg_episode_t* skel = NULL;
@ -550,6 +554,7 @@ epg_episode_t* epg_episode_find_by_uri ( const char *id, int create )
ee = skel;
skel = NULL;
ee->ee_uri = strdup(id);
*save |= 1;
}
}
@ -745,7 +750,7 @@ static int _epg_broadcast_idx = 0;
//
// Note: do we need to pass in stop?
epg_broadcast_t* epg_broadcast_find_by_time
( epg_channel_t *channel, time_t start, time_t stop, int create )
( epg_channel_t *channel, time_t start, time_t stop, int create, int *save )
{
epg_broadcast_t *eb;
static epg_broadcast_t *skel = NULL;
@ -771,6 +776,7 @@ epg_broadcast_t* epg_broadcast_find_by_time
if ( eb == NULL ) {
eb = skel;
skel = NULL;
*save |= 1;
_epg_broadcast_idx++;
}
}
@ -839,7 +845,8 @@ static void _epg_channel_link ( epg_channel_t *ec )
}
}
epg_channel_t* epg_channel_find_by_uri ( const char *id, int create )
epg_channel_t* epg_channel_find_by_uri
( const char *id, int create, int *save )
{
epg_channel_t *ec;
static epg_channel_t *skel = NULL;
@ -861,6 +868,7 @@ epg_channel_t* epg_channel_find_by_uri ( const char *id, int create )
skel = NULL;
ec->ec_uri = strdup(id);
LIST_INSERT_HEAD(&epg_unlinked_channels1, ec, ec_ulink);
*save |= 1;
}
}

View file

@ -62,7 +62,8 @@ typedef struct epg_brand
} epg_brand_t;
/* Lookup */
epg_brand_t *epg_brand_find_by_uri ( const char *uri, int create );
epg_brand_t *epg_brand_find_by_uri
( const char *uri, int create, int *save );
/* Mutators */
int epg_brand_set_title ( epg_brand_t *b, const char *title )
@ -107,7 +108,8 @@ typedef struct epg_season
} epg_season_t;
/* Lookup */
epg_season_t *epg_season_find_by_uri ( const char *uri, int create );
epg_season_t *epg_season_find_by_uri
( const char *uri, int create, int *save );
/* Mutators */
int epg_season_set_summary ( epg_season_t *s, const char *summary )
@ -158,7 +160,8 @@ typedef struct epg_episode
} epg_episode_t;
/* Lookup */
epg_episode_t *epg_episode_find_by_uri ( const char *uri, int create );
epg_episode_t *epg_episode_find_by_uri
( const char *uri, int create, int *save );
/* Mutators */
int epg_episode_set_title ( epg_episode_t *e, const char *title )
@ -227,7 +230,8 @@ typedef struct epg_broadcast
} epg_broadcast_t;
/* Lookup */
epg_broadcast_t *epg_broadcast_find_by_time ( epg_channel_t *ch, time_t start, time_t stop, int create );
epg_broadcast_t *epg_broadcast_find_by_time
( epg_channel_t *ch, time_t start, time_t stop, int create, int *save );
epg_broadcast_t *epg_broadcast_find_by_id ( int id );
/* Mutators */
@ -263,7 +267,8 @@ typedef struct epg_channel
} epg_channel_t;
/* Lookup */
epg_channel_t *epg_channel_find_by_uri ( const char *uri, int create );
epg_channel_t *epg_channel_find_by_uri
( const char *uri, int create, int *save );
/* Mutators */
int epg_channel_set_name ( epg_channel_t *c, const char *n )

View file

@ -40,9 +40,9 @@ void epggrab_init ( void )
{
/* Defaults */
epggrab_advanced = 0;
epggrab_eit = 1; // on air grab enabled
epggrab_interval = 12; // hours
epggrab_module = NULL; // disabled
epggrab_eit = 1; // on air grab enabled
epggrab_interval = 12 * 3600; // hours
epggrab_module = NULL; // disabled
/* Initialise modules */
epggrab_module_pyepg = pyepg_init();
@ -64,6 +64,8 @@ static void _epggrab_module_run ( epggrab_module_t *mod, const char *opts )
int save = 0;
time_t tm1, tm2;
htsmsg_t *data;
epggrab_stats_t stats;
memset(&stats, 0, sizeof(stats));
/* Check */
if ( !mod ) return;
@ -72,15 +74,40 @@ static void _epggrab_module_run ( epggrab_module_t *mod, const char *opts )
time(&tm1);
data = mod->grab(opts);
time(&tm2);
if ( !data ) {
tvhlog(LOG_WARNING, mod->name(), "grab returned no data");
} else {
/* Process */
if ( data ) {
//htsmsg_print(data);
tvhlog(LOG_DEBUG, mod->name(), "grab took %d seconds", tm2 - tm1);
pthread_mutex_lock(&global_lock);
save = mod->parse(data);
if (save) epg_updated();
time(&tm1);
save = mod->parse(data, &stats);
time(&tm2);
pthread_mutex_unlock(&global_lock);
htsmsg_destroy(data);
tvhlog(LOG_DEBUG, mod->name(), "parse took %d seconds", tm2 - tm1);
tvhlog(LOG_DEBUG, mod->name(), " channels tot=%5d new=%5d mod=%5d",
stats.channels.total, stats.channels.created,
stats.channels.modified);
tvhlog(LOG_DEBUG, mod->name(), " brands tot=%5d new=%5d mod=%5d",
stats.brands.total, stats.brands.created,
stats.brands.modified);
tvhlog(LOG_DEBUG, mod->name(), " seasons tot=%5d new=%5d mod=%5d",
stats.seasons.total, stats.seasons.created,
stats.seasons.modified);
tvhlog(LOG_DEBUG, mod->name(), " episodes tot=%5d new=%5d mod=%5d",
stats.episodes.total, stats.episodes.created,
stats.episodes.modified);
tvhlog(LOG_DEBUG, mod->name(), " broadcasts tot=%5d new=%5d mod=%5d",
stats.broadcasts.total, stats.broadcasts.created,
stats.broadcasts.modified);
/* Updated */
// TODO: should epg_updated happen inside or outside the lock?
if (save) epg_updated();
} else {
tvhlog(LOG_WARNING, mod->name(), "grab returned no data");
}
}

View file

@ -8,6 +8,25 @@
* Type definitions
* *************************************************************************/
/*
* Grab statistics
*/
typedef struct epggrab_stats_part
{
int created;
int modified;
int total;
} epggrab_stats_part_t;
typedef struct epggrab_stats
{
epggrab_stats_part_t channels;
epggrab_stats_part_t brands;
epggrab_stats_part_t seasons;
epggrab_stats_part_t episodes;
epggrab_stats_part_t broadcasts;
} epggrab_stats_t;
/*
* Grabber base class
*/
@ -17,7 +36,7 @@ typedef struct epggrab_module
void (*enable) ( void );
void (*disable) ( void );
htsmsg_t* (*grab) ( const char *opts );
int (*parse) ( htsmsg_t *data );
int (*parse) ( htsmsg_t *data, epggrab_stats_t *stats );
} epggrab_module_t;
/*

View file

@ -43,7 +43,7 @@ static int _pyepg_parse_channel ( htsmsg_t *data )
if ((tags = htsmsg_get_map(data, "tags")) == NULL) return 0;
/* Find channel */
if ((channel = epg_channel_find_by_uri(id, 1)) == NULL) return 0;
if ((channel = epg_channel_find_by_uri(id, 1, &save)) == NULL) return 0;
// TODO: need to save if created
/* Set name */
@ -69,7 +69,7 @@ static int _pyepg_parse_brand ( htsmsg_t *data )
if ((tags = htsmsg_get_map(data, "tags")) == NULL) return 0;
/* Find brand */
if ((brand = epg_brand_find_by_uri(str, 1)) == NULL) return 0;
if ((brand = epg_brand_find_by_uri(str, 1, &save)) == NULL) return 0;
// TODO: do we need to save if created?
/* Set title */
@ -113,12 +113,12 @@ static int _pyepg_parse_season ( htsmsg_t *data )
if ((tags = htsmsg_get_map(data, "tags")) == NULL) return 0;
/* Find series */
if ((season = epg_season_find_by_uri(str, 1)) == NULL) return 0;
if ((season = epg_season_find_by_uri(str, 1, &save)) == NULL) return 0;
// TODO: do we need to save if created?
/* Set brand */
if ((str = htsmsg_get_str(attr, "brand"))) {
if ((brand = epg_brand_find_by_uri(str, 0))) {
if ((brand = epg_brand_find_by_uri(str, 0, NULL))) {
save |= epg_season_set_brand(season, brand, 1);
}
}
@ -170,19 +170,19 @@ static int _pyepg_parse_episode ( htsmsg_t *data )
if ((tags = htsmsg_get_map(data, "tags")) == NULL) return 0;
/* Find episode */
if ((episode = epg_episode_find_by_uri(str, 1)) == NULL) return 0;
if ((episode = epg_episode_find_by_uri(str, 1, &save)) == NULL) return 0;
// TODO: do we need to save if created?
/* Set brand */
if ((str = htsmsg_get_str(attr, "brand"))) {
if ((brand = epg_brand_find_by_uri(str, 0))) {
if ((brand = epg_brand_find_by_uri(str, 0, NULL))) {
save |= epg_episode_set_brand(episode, brand, 1);
}
}
/* Set season */
if ((str = htsmsg_get_str(attr, "series"))) {
if ((season = epg_season_find_by_uri(str, 0))) {
if ((season = epg_season_find_by_uri(str, 0, NULL))) {
save |= epg_episode_set_season(episode, season, 1);
}
}
@ -236,14 +236,14 @@ static int _pyepg_parse_broadcast ( htsmsg_t *data, epg_channel_t *channel )
if ((stop = htsmsg_get_str(attr, "stop")) == NULL ) return 0;
/* Find episode */
if ((episode = epg_episode_find_by_uri(id, 1)) == NULL) return 0;
if ((episode = epg_episode_find_by_uri(id, 1, &save)) == NULL) return 0;
/* Parse times */
if (!_pyepg_parse_time(start, &tm_start)) return 0;
if (!_pyepg_parse_time(stop, &tm_stop)) return 0;
/* Find broadcast */
broadcast = epg_broadcast_find_by_time(channel, tm_start, tm_stop, 1);
broadcast = epg_broadcast_find_by_time(channel, tm_start, tm_stop, 1, &save);
if ( broadcast == NULL ) return 0;
/* Set episode */
@ -266,7 +266,7 @@ static int _pyepg_parse_schedule ( htsmsg_t *data )
if ((attr = htsmsg_get_map(data, "attrib")) == NULL) return 0;
if ((str = htsmsg_get_str(attr, "channel")) == NULL) return 0;
if ((channel = epg_channel_find_by_uri(str, 0)) == NULL) return 0;
if ((channel = epg_channel_find_by_uri(str, 0, NULL)) == NULL) return 0;
if ((tags = htsmsg_get_map(data, "tags")) == NULL) return 0;
HTSMSG_FOREACH(f, tags) {
@ -303,7 +303,8 @@ static int _pyepg_parse_epg ( htsmsg_t *data )
return save;
}
static int _pyepg_parse ( htsmsg_t *data )
// TODO: add stats updating
static int _pyepg_parse ( htsmsg_t *data, epggrab_stats_t *stats )
{
htsmsg_t *tags, *epg;
epggrab_module_t *mod;
@ -319,7 +320,7 @@ static int _pyepg_parse ( htsmsg_t *data )
/* XMLTV format */
if ((epg = htsmsg_get_map(tags, "tv")) != NULL) {
mod = epggrab_module_find_by_name("xmltv");
if (mod) return mod->parse(epg);
if (mod) return mod->parse(epg, stats);
}
return 0;

View file

@ -24,6 +24,7 @@
#include <unistd.h>
#include <stdlib.h>
#include <dirent.h>
#include <openssl/md5.h>
#include "htsmsg_xml.h"
#include "settings.h"
@ -38,6 +39,24 @@
* Parsing
* *************************************************************************/
/**
* Hash the description to get a URI for episode
*
* Note: converts to an ASCII format
*/
static const char *_xmltv_hash ( const char *str )
{
size_t i;
static char ret[(MD5_DIGEST_LENGTH*2)+1];
static unsigned char md5[MD5_DIGEST_LENGTH];
(void)MD5((const unsigned char*)str, strlen(str), md5);
for ( i = 0; i < MD5_DIGEST_LENGTH; i++ ) {
sprintf(&ret[i*2], "%02X", md5[i]);
}
ret[MD5_DIGEST_LENGTH*2] = '\0';
return ret;
}
/**
*
*/
@ -213,20 +232,28 @@ get_episode_info(htsmsg_t *tags, epg_episode_t *ee)
*/
static int
_xmltv_parse_programme_tags(epg_channel_t *xc, htsmsg_t *tags,
time_t start, time_t stop)
time_t start, time_t stop, epggrab_stats_t *stats)
{
int save = 0;
int save = 0, save2 = 0;
epg_broadcast_t *ebc;
epg_episode_t *ee;
const char *uri = NULL;
const char *title = htsmsg_xml_get_cdata_str(tags, "title");
const char *desc = htsmsg_xml_get_cdata_str(tags, "desc");
#if TODO_EPG_GENRE
const char *category = htsmsg_xml_get_cdata_str(tags, "category");
#endif
/* Generate an episode */
// TODO: hash the description?
ee = epg_episode_find_by_uri("TODO", 1);
/* Generate a URI */
if ( desc )
uri = _xmltv_hash(desc);
// TODO: what to do otherwise?
if ( !uri ) return 0;
/* Find episode */
ee = epg_episode_find_by_uri(uri, 1, &save);
stats->episodes.total++;
if (save) stats->episodes.created++;
if (title) save |= epg_episode_set_title(ee, title);
if (desc) save |= epg_episode_set_description(ee, desc);
#if TODO_EPG_GENRE
@ -235,12 +262,18 @@ _xmltv_parse_programme_tags(epg_channel_t *xc, htsmsg_t *tags,
#if TODO_XMLTV_EP_NUMBERING
_get_episode_info(tags, &episode);
#endif
if (save) stats->episodes.modified++;
/* Create broadcast */
ebc = epg_broadcast_find_by_time(xc, start, stop, 1);
save |= epg_broadcast_set_episode(ebc, ee, 1);
ebc = epg_broadcast_find_by_time(xc, start, stop, 1, &save2);
if ( ebc != NULL ) {
stats->broadcasts.total++;
if (save2) stats->broadcasts.created++;
save2 |= epg_broadcast_set_episode(ebc, ee, 1);
if (save2) stats->broadcasts.modified++;
}
return save;
return save | save2;
}
@ -248,7 +281,7 @@ _xmltv_parse_programme_tags(epg_channel_t *xc, htsmsg_t *tags,
* Parse a <programme> tag from xmltv
*/
static int
_xmltv_parse_programme(htsmsg_t *body)
_xmltv_parse_programme(htsmsg_t *body, epggrab_stats_t *stats)
{
int save = 0;
htsmsg_t *attribs, *tags;
@ -261,7 +294,7 @@ _xmltv_parse_programme(htsmsg_t *body)
if((attribs = htsmsg_get_map(body, "attrib")) == NULL) return 0;
if((tags = htsmsg_get_map(body, "tags")) == NULL) return 0;
if((chid = htsmsg_get_str(attribs, "channel")) == NULL) return 0;
if((xc = epg_channel_find_by_uri(chid, 0)) == NULL) return 0;
if((xc = epg_channel_find_by_uri(chid, 0, NULL)) == NULL) return 0;
if((s = htsmsg_get_str(attribs, "start")) == NULL) return 0;
start = _xmltv_str2time(s);
if((s = htsmsg_get_str(attribs, "stop")) == NULL) return 0;
@ -269,7 +302,7 @@ _xmltv_parse_programme(htsmsg_t *body)
if(stop <= start || stop < dispatch_clock) return 0;
_xmltv_parse_programme_tags(xc, tags, start, stop);
save |= _xmltv_parse_programme_tags(xc, tags, start, stop, stats);
return save;
}
@ -277,26 +310,27 @@ _xmltv_parse_programme(htsmsg_t *body)
* Parse a <channel> tag from xmltv
*/
static int
_xmltv_parse_channel(htsmsg_t *body)
_xmltv_parse_channel(htsmsg_t *body, epggrab_stats_t *stats)
{
int save =0;
htsmsg_t *attribs, *tags;/*, *subtag;*/
const char *id;/*TODO, *name, *icon;*/
const char *id, *name; /*,icon;*/
epg_channel_t *xc;
if(body == NULL) return 0;
if((attribs = htsmsg_get_map(body, "attrib")) == NULL) return 0;
if((id = htsmsg_get_str(attribs, "id")) == NULL) return 0;
if((xc = epg_channel_find_by_uri(id, 1)) == NULL) return 0;
if((tags = htsmsg_get_map(body, "tags")) == NULL) return 0;
// TODO: save in find_by_uri
if((xc = epg_channel_find_by_uri(id, 1, &save)) == NULL) return 0;
stats->channels.total++;
if (save) stats->channels.created++;
#if TODO_EPG_CHANNEL_META
if((name = xmltv_get_cdata_by_tag(tags, "display-name")) != NULL) {
if((name = htsmsg_xml_get_cdata_str(tags, "display-name")) != NULL) {
save |= epg_channel_set_name(xc, name);
}
#if TODO_EPG_CHANNEL_META
if((subtag = htsmsg_get_map(tags, "icon")) != NULL &&
(attribs = htsmsg_get_map(subtag, "attrib")) != NULL &&
(icon = htsmsg_get_str(attribs, "src")) != NULL) {
@ -310,6 +344,7 @@ _xmltv_parse_channel(htsmsg_t *body)
}
}
#endif
if (save) stats->channels.modified++;
return save;
}
@ -317,7 +352,7 @@ _xmltv_parse_channel(htsmsg_t *body)
*
*/
static int
_xmltv_parse_tv(htsmsg_t *body)
_xmltv_parse_tv(htsmsg_t *body, epggrab_stats_t *stats)
{
int save = 0;
htsmsg_t *tags;
@ -328,19 +363,18 @@ _xmltv_parse_tv(htsmsg_t *body)
HTSMSG_FOREACH(f, tags) {
if(!strcmp(f->hmf_name, "channel")) {
save |= _xmltv_parse_channel(htsmsg_get_map_by_field(f));
save |= _xmltv_parse_channel(htsmsg_get_map_by_field(f), stats);
} else if(!strcmp(f->hmf_name, "programme")) {
save |= _xmltv_parse_programme(htsmsg_get_map_by_field(f));
save |= _xmltv_parse_programme(htsmsg_get_map_by_field(f), stats);
}
}
return save;
}
/**
*
*/
static int _xmltv_parse ( htsmsg_t *data )
static int _xmltv_parse ( htsmsg_t *data, epggrab_stats_t *stats )
{
htsmsg_t *tags, *tv;
@ -350,7 +384,7 @@ static int _xmltv_parse ( htsmsg_t *data )
if((tv = htsmsg_get_map(tags, "tv")) == NULL)
return 0;
return _xmltv_parse_tv(tv);
return _xmltv_parse_tv(tv, stats);
}
/* ************************************************************************
@ -358,6 +392,7 @@ static int _xmltv_parse ( htsmsg_t *data )
* ***********************************************************************/
// TODO: config
// TODO: remove use of hardcoded xmltv script
static epggrab_module_t xmltv_module;
@ -371,13 +406,12 @@ static htsmsg_t* _xmltv_grab ( const char *iopts )
size_t outlen;
char *outbuf;
char errbuf[100];
const char *cmd = "/usr/bin/tv_grab_uk_rt";
const char *cmd = "/home/aps/tmp/epg.sh";//usr/bin/tv_grab_uk_rt";
/* Debug */
tvhlog(LOG_DEBUG, "xmltv", "grab %s", cmd);
/* Grab */
/* TODO: using hardcoded xmltv command at the moment */
outlen = spawn_and_store_stdout(cmd, NULL, &outbuf);
if ( outlen < 1 ) {
tvhlog(LOG_ERR, "xmltv", "no output detected");