#!/usr/bin/perl -w

eval 'exec /usr/bin/perl -w -S $0 ${1+"$@"}'
    if 0; # not running under some shell

use warnings;
use strict;
use XMLTV::ProgressBar;
use XMLTV::Memoize; XMLTV::Memoize::check_argv('get_octets');
use XMLTV::DST;
use XMLTV::Supplement qw/GetSupplement SetSupplementRoot/;
use XMLTV::Options qw/ParseOptions/;
use XMLTV::Configure::Writer;
use XMLTV::Ask;
use XMLTV::Lineup;
use File::Path;
use File::Basename;
use LWP::UserAgent;
use HTTP::Cache::Transparent;
use Getopt::Long;
use Encode qw/decode encode/;
use Date::Manip; Date_Init('TZ=UTC'); # UTC required for XMLTV::DST
use HTML::Entities;

##############################################
#################### TODO ####################
##############################################

# - refactor title processing routines to take \%prog
#
# - case insensitive matching during title
#   processing
#
# - use hash lookups where possible instead
#   of iterating over lists of text to match
#
# - only run specific procesing routines where
#   sensible and skip further processing once a 
#   replacement has been made
#
# - ensure that title processing has finished
#   before concatenating episode numbering
#
# - split out UTF-8 handling routines to take \$page
#
# - revisit handling of DST changeovers to ensure
#   both changeovers are handled appropriately
#   and that attempts are made to handle incorrectly
#   flagged programmes
#
# - look forward more than 1 programme when
#   looking for and correcting overlapping
#   programmes
#
# - include actor role data where available
#
# - include review details where available
#
# - ensure lang output for relevant elements

###############################################
################## VARIABLES ##################
###############################################

# Grabber name
my $grabber_name = 'tv_grab_uk_rt';

# Grabber version
my $grabber_cvs_id = '$Id: tv_grab_uk_rt.in,v 1.286 2010/04/21 05:24:40 knowledgejunkie Exp $';
my $grabber_version;

if ( $grabber_cvs_id =~ m!\$Id: [^,]+,v (\S+) ([0-9/: -]+)! ) {
    $grabber_version = "$1, $2";
}
else {
    $grabber_version = "Unknown";
}

# Set up LWP::UserAgent
my $ua = LWP::UserAgent->new;
$ua->agent("xmltv/$XMLTV::VERSION");

# Lineup writer
my $lineup_writer;

#type id source-data-url generator-info-name generator-info-url
my %xmltv_lineup_attributes = (
    'type'                => 'DVB-T',
    'version'             => '1.00',
    'id'                  => 'freeview.co.uk',
    'source-data-url'     => 'tv_grab_uk_rt FreeView channels',
    'generator-info-name' => "XMLTV/$XMLTV::VERSION, $grabber_name $grabber_version",
    'generator-info-url'  => 'http://www.xmltv.org',
);

# Required to be displayed by Radio Times
my $rt_copyright 
      = "\n"
      . "     +-----------------------------------------------------+     \n"
      . "     | In accessing this XML feed, you agree that you will |     \n"
      . "     | only access its contents for your own personal and  |     \n"
      . "     |  non-commercial use and not for any commercial or   |     \n"
      . "     |  other purposes, including advertising or selling   |     \n"
      . "     |  any goods or services, including any third-party   |     \n"
      . "     |   software applications available to the general    |     \n"
      . "     |           public. <xmltv.radiotimes.com>            |     \n"
      . "     +-----------------------------------------------------+     \n"
      . "\n";

# Location of Radio Times channel index file
my $rt_root_dir = 'http://xmltv.radiotimes.com/xmltv';
my $rt_channels_uri = "$rt_root_dir/channels.dat";

# The format of the Radio Times source data (set to strict UTF-8)
my $source_encoding = "utf-8";
# Default XML output encoding to use (set to strict UTF-8)
my $xml_encoding    = "utf-8";

my %tv_attributes = (
    'source-info-name'    => 'Radio Times XMLTV Service',
    'source-info-url'     => 'http://www.radiotimes.com',
    'source-data-url'     => "$rt_channels_uri",
    'generator-info-name' => "XMLTV/$XMLTV::VERSION, $grabber_name $grabber_version",
    'generator-info-url'  => 'http://www.xmltv.org',
);

# Get default location to store cached listings data
my $default_cachedir = get_default_cachedir();

# Reciprocal XMLTV/RT ID hashes for the required channel_ids fields, allowing
# RT ID -> XMLTV ID and XMLTV ID -> RT ID lookups
my (%rt_to_xmltv, %xmltv_to_rt);
# Hashes for the optional channel_ids fields, keyed by XMLTV ID
my (%extra_dn, %icon_urls, %channel_offset, %broadcast_hours, %video_quality);

# Do the progress bars need a final update?
my $need_final_update;

# Read all command line options 
my ( $opt, $conf ) = ParseOptions( {
    grabber_name => "$grabber_name",
    version => "$grabber_cvs_id",
    description => "United Kingdom/Republic of Ireland (Radio Times)",
    capabilities => [qw/baseline manualconfig cache preferredmethod tkconfig apiconfig lineups/],
    stage_sub => \&config_stage,
    listchannels_sub => \&list_channels,
    load_old_config_sub => \&load_old_config,
    preferredmethod => 'allatonce',
    defaults => { days => 15, offset => 0, quiet => 0, debug => 0 },
    list_lineups_sub => \&list_lineups,
    get_lineup_sub => \&get_lineup,
} );

# At this point, grabbing routines take over from ParseOptions

die "Error: You cannot specify --quiet with --debug, exiting"
    if ($opt->{quiet} && $opt->{debug});

# Make sure that we are _really_ using a GMT (+0000) timezone
# check_xmltv_tz();

if (not defined( $conf->{'channel'} )) {
    print STDERR "No channels selected in configfile " .
                 $opt->{'config-file'} . "\n" .
                 "Please run the grabber with --configure.\n";
    exit 1;
}

# New-style config files must include a cachedir entry
if (not defined( $conf->{cachedir} )) {
    print STDERR "No cachedir defined in configfile " .
                 $opt->{'config-file'} . "\n" .
                 "Please run the grabber with --configure.\n";
    exit 1;
}

# Update encoding if seen in new-style config file
if (defined( $conf->{encoding} )) {
    $xml_encoding = $conf->{encoding}->[0];
}

# Enable title processing? Enable it by default if not explicitly disabled
my $title_processing;
if (defined( $conf->{'title-processing'} )) {
    $title_processing = $conf->{'title-processing'}->[0];
}
else {
    $title_processing = 'enabled';
}

# Initialise the cache-directory
init_cachedir( $conf->{cachedir}->[0] );

# Set cache options
#
# MaxAge set to 15 days as Radio Times provides 14 days of listings
# NoUpdate set to 1hr as Radio Times data updated once per day
#
HTTP::Cache::Transparent::init( {
    BasePath       => $conf->{cachedir}->[0],
    MaxAge         => 15*24,
    NoUpdate       => 60*60,
    Verbose        => $opt->{debug},
    ApproveContent => \&check_content_length,
    }
);

# Variables for programme title manipulation
my $have_title_data = 0;
my (@non_title_info,       @mixed_title_subtitle, 
    @mixed_subtitle_title, @reversed_title_subtitle) = ();
my %replacement_titles;       # key = old title, value = replacement title
my %replacement_episodes;     # key = title, value = hash (where key = old ep, value = new ep)
my %replacement_cats;         # key = title, value = category
my %replacement_title_eps;    # key = 'old_title . old_ep', value = (new_title, new_ep)
my %flagged_title_eps;        # key = old_title from title fixup routine 8
my %replacement_ep_from_desc; # key = title, value = hash (where key = desc, value = new ep)
my %uc_prog_titles;           # key = title, value = title

# Create global hash to store the programme titles for all programmes on all
# channels, as we will process this last after grabbing to determine any
# 'manufactured' titles which may include temporary 'seasonal' information
my %prog_titles;

# Array to store any overlapping programmes that are detected
my @overlapping_progs;

# Hash to store bad character strings and their replacments that are used when
# processing the source data to remove mis-encoded UTF-8 characters
my %utf8_fixups;

# Create hashes to store names/urls of channels with occurences of mis-encoded
# UTF-8 data after our replacement routines have run
my %hasC27FBFchars;
my %hadEFBFBD;
my %hadC3AFC2BFC2BD;

# Create hashes to store uncategories programmes and available categories
# to potentially use for such programmes
my %uncategorised_progs;
my %categories;
my %cats_per_prog;

# Create hashes to store episode details that may still contain series, episode
# or part numbering after processing to handle these has been carried out
my %possible_series_nums;
my %possible_episode_nums;
my %possible_part_nums;

# Hash to store titles containing text that should likely be removed
my %title_text_to_remove;

# Hash to store details of empty source listings
my %empty_listings;

# Track problems during listings retrieval. Currently we exit(1) only if
# listings data is missing for any requested channels
my $warnings = 0;

# Output XMLTV library and grabber versions
if (!$opt->{quiet}) {
    say("Program/library version information:\n");
    say("XMLTV library version: $XMLTV::VERSION");
    say("$grabber_name version: $grabber_version");
    say("  libwww-perl version: $LWP::VERSION");
    say("  Date::Manip version: " . DateManipVersion(1) . "\n");
}

# Determine the modification time of the source data on the RT servers
my $rt_mod_time = get_mod_time($rt_channels_uri);
if ( $rt_mod_time ) {
    say("\nSource data last updated on: " . $rt_mod_time . "\n") if (!$opt->{quiet});
    $tv_attributes{'date'} = $rt_mod_time;
}

# Retrieve list of all channels currently available
my $available_channels = load_available_channels();
# Now ensure configured channels are still available to download
my $wanted_chs = check_configured_channels($available_channels);

###############################################
############### GRAB THE DATA #################
###############################################

# Configure output and write XMLTV data - header, channels, listings, and footer
my $writer;
setup_xmltv_writer();
write_xmltv_header();
write_channel_list($available_channels, $wanted_chs);
write_listings_data($available_channels, $wanted_chs);
write_xmltv_footer();

# Print out optional debug info for titles, bad utf-8 chars and categories
if ($opt->{debug}) {
    if ($title_processing eq 'enabled') {
        print_titles_with_colons();
        print_titles_with_hyphens();
    }
    #print_uc_titles_pre();
    print_uc_titles_post();
    print_title_variants();
    print_flagged_title_eps();

    print_possible_prog_numbering();

    print_categories();
    print_uncategorised_progs();
    print_cats_per_prog();

    print_overlapping_progs();
    
    print_empty_listings();
    print_misencoded_utf8_data();
}

# Give a useful exit status if data for some channels was not downloaded
if ($warnings) {
    if (!$opt->{quiet}) {
        say("\nFinished, but listings for some channels are missing. " .
            "Check error log.\n");
    }
    exit(1);
}
else {
    if (!$opt->{quiet}) {
        say("\nFinished!\n");
    }
    exit(0);
}

###############################################
################ SUBROUTINES ##################
###############################################

# Convenience debugging method
sub t {
    my ($message) = @_;
    if ($opt->{debug}) {
        print STDERR $message . "\n";
    }
}

# Convenience method for use with XMLTV::Memoize. Only return content
# after a successful response. We require access to the raw octets via
# $resp->content in order to be able to process the data for double and
# mis-encoded UTF-8 content. Calling $resp->decoded_content or using
# LWP::Simple::get() (versions of LWP >=5.827) would not permit this.
sub get_octets {
    my $resp = $ua->get(shift @_);
    if ($resp->is_error) {
        return undef;
    }
    else {
        return $resp->content;
    }
}

# Get the last-modified time of a successful HTTP Response object. Return
# undef on error
sub get_mod_time {
    my $resp = $ua->get(shift @_);
    if ($resp->is_error) {
        return undef;
    }
    else {
        return $resp->header('Last-Modified');
    }
}

# Return the digit equivalent of its word, i.e. "one" -> "1",
# or return the word if it appears to consist of only digits
sub word_to_digit {

    my $word = shift;
    return undef if !defined $word;
    return $word if $word =~ /\d+/;
    for (lc $word) { 
        if    (/^one$/)       { return 1 }
        elsif (/^two$/)       { return 2 }
        elsif (/^three$/)     { return 3 }
        elsif (/^four$/)      { return 4 }
        elsif (/^five$/)      { return 5 }
        elsif (/^six$/)       { return 6 }
        elsif (/^seven$/)     { return 7 }
        elsif (/^eight$/)     { return 8 }
        elsif (/^nine$/)      { return 9 }
        elsif (/^ten$/)       { return 10 }
        elsif (/^eleven$/)    { return 11 }
        elsif (/^twelve$/)    { return 12 }
        elsif (/^thirteen$/)  { return 13 }
        elsif (/^fourteen$/)  { return 14 }
        elsif (/^fifteen$/)   { return 15 }
        elsif (/^sixteen$/)   { return 16 }
        elsif (/^seventeen$/) { return 17 }
        elsif (/^eighteen$/)  { return 18 }
        elsif (/^nineteen$/)  { return 19 }
        elsif (/^twenty$/)    { return 20 }
        # handle 1-8 in roman numberals
        elsif (/^i$/)         { return 1 }
        elsif (/^ii$/)        { return 2 }
        elsif (/^iii$/)       { return 3 }
        elsif (/^iv$/)        { return 4 }
        elsif (/^v$/)         { return 5 }
        elsif (/^vi$/)        { return 6 }
        elsif (/^vii$/)       { return 7 }
        elsif (/^viii$/)      { return 8 }
        # return undef if input unhandled
        else                  { return undef }
    }
}

# Ensure timezone used for utc_offset date procesing is UTC (+0000)
sub check_xmltv_tz {
    die("XMLTV requires a Date::Manip timezone of +0000 to work properly.\n"
        . "Current Date::Manip timezone is " . Date_TimeZone() . ".\n")
        if (Date_TimeZone ne '+0000');
}

# Display required copyright message from Radio Times
sub display_copyright {
    say("$rt_copyright");
}

sub get_default_cachedir {
    my $winhome = undef;
    if (defined $ENV{HOMEDRIVE} && defined $ENV{HOMEPATH}) {
        $winhome = $ENV{HOMEDRIVE} . $ENV{HOMEPATH};
    }
        
    my $home = $ENV{HOME} || $winhome || ".";
    my $dir = "$home/.xmltv/cache";
    t("Using '$dir' as cache-directory for XMLTV listings");
    return $dir;
}

sub init_cachedir {
    my $path = shift @_;
    if (! -d $path) {
        if (mkpath($path)) {
            t("Created cache-directory '$path'");
        }
        else {
            die "Error: Failed to create cache-directory $path: $@, exiting";
        }
    }
}

# Check whether data files on the RT website are empty but still online, or
# contain HTML/XML from the Radio Times' error page.
#
# These files will have a good HTTP response header as they exist, but they
# contain no data. Caching via HCT without checking for a non-zero content_size
# beforehand will therefore overwrite good data with bad. Any file having a
# content_length of 0 or seen to contain DOCTYPE info will not be cached and the 
# existing cached copy of the file will be used instead.
#
# Support for this functionality requires using at least the 1.0 version of
# HTTP::Cache::Transparent, which can be obtained from CPAN.
#
sub check_content_length {
    my $rt_file = shift @_;
    if ($rt_file->is_success) {
        # reject an empty (but available) file
        if ($rt_file->content_length == 0) {
            return 0;
        }
        # an empty source file containing only the RT disclaimer has a length
        # of 307 bytes
        elsif ($rt_file->content_length == 307) {
            return 0;
        }
        # reject a likely HTML error page
        elsif ($rt_file->content =~ /DOCTYPE/) {
            return 0;
        }
        # cache a likely good file
        else {
            return 1;
        }
    }
    # reject file if retrieval failed
    else {
        return 0;
    }
}

sub config_stage {
    my ( $stage, $conf ) = @_;

    my $result;
    my $writer = new XMLTV::Configure::Writer( OUTPUT => \$result,
                                               encoding => $xml_encoding );

    $writer->start( { grabber => "$grabber_name" } );
    
    if ($stage eq 'start') {

        $writer->start_selectone( {
            id => 'encoding',
            title => [ [ 'Encoding', 'en' ] ],
            description => [
                [ "Select which output format to use",
                'en' ] ],
        } );
        $writer->write_option( {
            value => 'utf-8',
            text => [ [ 'UTF-8 (Unicode)', 'en' ] ],
        } );
        $writer->write_option( {
            value => 'iso-8859-1',
            text => [ [ 'ISO-8859-1 (Latin-1)', 'en' ] ],
        } );
        $writer->end_selectone();

        $writer->end('select-cachedir');
    }
    elsif ($stage eq 'select-cachedir') {
        $writer->write_string( {
            id => 'cachedir',
            title => [ [ 'Enter the directory to store the listings cache in', 'en' ] ],
            description => [
                [ "$grabber_name uses a cache with files that it has already " .
                'downloaded. Please specify where the cache shall be stored.',
                'en' ] ],
            default => $default_cachedir,
        } );
        $writer->end('select-title-processing');
    }
    elsif ($stage eq 'select-title-processing') {

        $writer->start_selectone( {
            id => 'title-processing',
            title => [ [ 'Enable title processing?', 'en' ] ],
            description => [
                [ "In a bid to provide more consistent listings data, $grabber_name " .
                'can further process programme and episode titles.',
                'en' ] ],
        } );
        $writer->write_option( {
            value => 'enabled',
            text => [ [ 'Enable title processing', 'en' ] ],
        } );
        $writer->write_option( {
            value => 'disabled',
            text => [ [ 'Disable title processing', 'en' ] ],
        } );
        $writer->end_selectone();

        $writer->end('select-postcode');
    }
    elsif ($stage eq 'select-postcode') {

        $writer->write_string( {
            id => 'postcode',
            title => [ [ 'Enter the first part of your postcode', 'en' ] ],
            description => [
                [ "$grabber_name can use the first part of your postcode in " .
                'order to determine which regional channels to display ' .
                'during configuration. Republic of Ireland users should enter ' .
                'the pseudo-postcode "EIRE". Entering "none" will disable this ' .
                'feature.',
                'en' ] ],
            default => 'none',
        } );
        $writer->end('select-platform');
    }
    elsif ($stage eq 'select-platform') {
        $writer->start_selectone( {
            id => 'platform',
            title => [ [ 'Select which TV platform you use', 'en' ] ],
            description => [
                [ "When choosing which channels to download listings for, $grabber_name " .
                'can show only those channels available on your TV platform.',
                'en' ] ],
        } );
        $writer->write_option( {
            value => 'none',
            text => [ [ 'None of these, I\'ll choose my channels', 'en' ] ],
        } );
        $writer->write_option( {
            value => 'freeview',
            text => [ [ 'Freeview', 'en' ] ],
        } );
        $writer->write_option( {
            value => 'skydigital',
            text => [ [ 'Sky Digital', 'en' ] ],
        } );
        $writer->write_option( {
            value => 'virgintv',
            text => [ [ 'VirginMedia TV', 'en' ] ],
        } );
        $writer->write_option( {
            value => 'freesat',
            text => [ [ 'FreeSat', 'en' ] ],
        } );
        $writer->write_option( {
            value => 'freesatfromsky',
            text => [ [ 'FreeSat from Sky', 'en' ] ],
        } );
        $writer->write_option( {
            value => 'analogue',
            text => [ [ 'Analogue', 'en' ] ],
        } );
        $writer->end_selectone();

        # The select-channels stage must be the last stage called
        $writer->end( 'select-channels' );
    }
    else {
        die "Unknown stage $stage";
    }
    
    return $result;
}

sub list_channels {
    my ( $conf, $opt ) = @_;

    my $channels = load_available_channels();
    my $platform = $conf->{platform}->[0];
    my $postcode = $conf->{postcode}->[0];

    # During configuration or listing available channels, we can also filter
    # those channels which are available on the user's TV platform.
    #
    # If we find a usable platform identifier, we retrieve details on which 
    # channels are available on that platform, and remove those which are not 
    # available from the list of available channels
    if (defined $platform && lc $platform ne 'none') {
        # Retrieve hash of matched/unmatched channels 
        my $platform_chans = get_channels_by_platform($platform);

        # Have we matched any channels?
        if (defined $platform_chans->{'matched'} 
                && scalar @{ $platform_chans->{'matched'} } >= 1) {
            # Flag the matched channels as being available
            foreach my $matched_chan (@{ $platform_chans->{'matched'} } ) {
                $channels->{$matched_chan}{'available'} = 1;
            }
            # Remove channels not flagged as available from %channels hash
            foreach my $chan (keys %{$channels}) {
                if (defined $channels->{$chan}{'available'} 
                                && $channels->{$chan}{'available'} == 1) {
                    delete $channels->{$chan}{'available'};
                }
                else {
                    delete $channels->{$chan};
                }
            }
        }
    }

    # During configuration or listing available channels, we can only include
    # regional channels which are available in the configured postcode area.
    #
    # If we find a configured postcode, we retrieve details on which channels
    # are available, and remove those which are not available from the list of
    # available channels
    if (defined $postcode && lc $postcode ne 'none') {
        # Retrieve hash of matched/unmatched channels 
        my $reg_chans = get_channels_by_postcode($postcode);

        # Have we matched any channels?
        if (defined $reg_chans->{'matched'} 
                && scalar @{ $reg_chans->{'matched'} } >= 1) {
            # Remove the ummatched regional channels from the %channels hash
            foreach my $unmatched_chan (@{ $reg_chans->{'unmatched'} } ) {
                delete $channels->{$unmatched_chan};
            }
        }
    }

    my $result="";
    my $fh = new IO::Scalar \$result;
    my $oldfh = select( $fh );

    my %g_args = (OUTPUT => $fh);

    # Write XMLTV to $result, rather than STDOUT
    my $writer = new XMLTV::Writer(%g_args, encoding => $xml_encoding);
    $writer->start(\%tv_attributes);

    # It is perhaps better to sort the list of available channels by
    # display-name, rather than xmltv_id. First create a hash to store the
    # id->name mapping
    my %chan_id_to_name;

    # Only add the non-RT sourced timeshifted channels during configuration,
    # otherwise the configuration could include both Radio Times-sourced
    # timeshifted data, and the timeshifted data we create internally from a
    #regular +0 channel
    my $chan_name;
    foreach my $chan_id (keys % {$channels}) {
        $chan_name = $channels->{$chan_id}->{'display-name'}->[0]->[0];
        if ($chan_name !~ /\(RT\)$/) {
            $chan_id_to_name{$chan_id} = $chan_name;
        }
    }

    # Create a sorted list of xmltv_ids in ascending order of the
    # corresponding display name (case-insensitive)
    my @chan_ids = sort {uc($chan_id_to_name{$a}) cmp uc($chan_id_to_name{$b})}
                        keys %chan_id_to_name;

    foreach my $channel (@chan_ids) {
        delete $channels->{$channel}{'rt_id'};
        $writer->write_channel( $channels->{$channel} );
    }
    
    $writer->end;
    select( $oldfh );
    $fh->close();

    return $result;
}

sub load_old_config {
    my ( $config_file ) = @_;
 
    if (!$opt->{quiet}) {
        say("Using old-style config file");
    }

    my @config_entries = XMLTV::Config_file::read_lines( $config_file );

    my $conf = {};
    # Use default cachedir as there was no support for choosing an alternative
    # cache directory before ParseOptions support was added to the grabber.
    $conf->{cachedir}->[0] = $default_cachedir;
    $conf->{channel} = [];

    CONFIG_ENTRY:
    foreach my $config_entry (@config_entries)
    {
        next CONFIG_ENTRY if (!defined $config_entry);
        next CONFIG_ENTRY if ($config_entry =~ '^#' || $config_entry =~ '^$');
        if ($config_entry !~ /^channel\s+(\S+)$/) {
            if (!$opt->{quiet}) {
                say("Bad line '$config_entry' in config file, skipping");
            }
            next CONFIG_ENTRY;
        }

        my( $command, $param ) = split( /\s+/, $config_entry, 2 );
        $param =~ tr/\n\r//d;
        $param =~ s/\s+$//;

        # We only support channel entries in the old-style config
        if ( $command =~ /^\s*channel\s*$/ ) {
            push @{$conf->{channel}}, $param;
        }
        else {
            die "Unknown command '$command' in config file $config_file"
        }
    }

    return $conf;
}

# Determine all currently available channels by reading the current Radio
# Times list of channels, and adding additional information from the
# grabber's channel_ids file. The content of both of these files is
# required in order to proceed with listings retrieval.
#
sub load_available_channels {
    # First we read in the XMLTV channel_ids file to provide additional
    # information (icon, display name) about available channels, and also
    # provide the information necessary for timeshifted and part-time channel
    # support.
    #
    # We use the hashes %rt_to_xmltv and %xmltv_to_rt to lookup the Radio 
    # Times and XMLTV channel IDs. These will deal sensibly with a new RT
    # channel that isn't yet mentioned in the channel_ids file.
    
    # Provide statistics for the number of usable, unusable, timeshifted,
    # part-time, and part-time timeshifted channels listed in channel_ids.
    my ($num_good_channels, $num_bad_channels, 
        $num_ts_channels, $num_pt_channels, $num_pt_ts_channels);

    # Retrieve grabber's channel_ids file via XMLTV::Supplement
    my $xmltv_channel_ids = GetSupplement("$grabber_name", 'channel_ids');

    die "Error: XMLTV channel_ids data is missing, exiting" 
        if (!defined $xmltv_channel_ids || $xmltv_channel_ids eq '');

    my @lines = split /[\n\r]+/, $xmltv_channel_ids;

    t("\nExtended XMLTV channel information:\n");

    XMLTV_CHANID_ENTRY:
    foreach my $line (@lines) {
        # Skip blank lines. Comments are allowed if they are at the start 
        # of the line.
        next XMLTV_CHANID_ENTRY if ($line =~ '^#' || $line =~ '^$');
        my @fields = split /\|/, $line;
        # We need at least 2 fields (xmltv_id,rt_id) to run the grabber.
        # No check on maximum number of fields to support future updates
        # to channel_ids now we are using XMLTV::Supplement.
        if (scalar @fields < 2) {
            t("Wrong number of fields in XMLTV channel_ids entry:\n"
                    . "\t" . $line);
            next XMLTV_CHANID_ENTRY;
        }

        # The channel_ids fields are:
        # 1) XMLTV ID
        # 2) RT ID
        # 3) Channel name
        # 4) Channel icon URL
        # 5) Timeshift offset
        # 6) Broadcast hours
        # 7) Video quality
        #
        # The RT channels.dat provides a channel name, but it may be out of
        # date - here we provide an alternative or updated name if the 
        # channel name has changed
        my ($xmltv_id, $rt_id,          $extra_dn, 
            $icon_url, $channel_offset, $broadcast_hours,
            $video_quality) = @fields;

        # Flag timeshifted and part-time channels for stats
        my ($is_timeshift, $is_parttime);

        # Check for required XMLTV ID and RT ID fields, skip if missing
        if (!defined $xmltv_id  || $xmltv_id eq '') {
            t("Undefined XMLTV ID seen in channel_ids, skipping");
            next XMLTV_CHANID_ENTRY;
        }
        if ($xmltv_id !~ /\w+\.\w+.*/) {
            t("Invalid XMLTV ID seen in channel_ids, skipping");
            next XMLTV_CHANID_ENTRY;
        }
        if (!defined $rt_id || $rt_id eq '') {
            t("Undefined RT ID seen in channel_ids, skipping");
            next XMLTV_CHANID_ENTRY;
        }
        if ($rt_id !~ /^\d+$/) {
            t("Invalid RT ID seen in channel_ids, skipping");
            next XMLTV_CHANID_ENTRY;
        }
        
        # Check for duplicate RT IDs having same associated XMLTV ID. As part of
        # timeshifted/part-time channel support, we associate the same RT ID
        # with different XMLTV IDs
        foreach my $id (@{$rt_to_xmltv{$rt_id}}) {
            if (defined $id && $id eq $xmltv_id) {
                t("Radio Times ID '$rt_id' already seen in XMLTV " 
                  . "channel_ids file, skipping");
                next XMLTV_CHANID_ENTRY;
            }
        }

        # Check whether current XMLTV ID has already been seen
        if (defined $xmltv_to_rt{$xmltv_id}) {
            t("XMLTV ID '$xmltv_id' already seen in XMLTV channel_ids file, skipping");
            next XMLTV_CHANID_ENTRY;
        }

        # Store the XMLTV channel description, report if it is missing
        if (defined $extra_dn) {
            if ($extra_dn eq '' || $extra_dn !~ /\w+/) {
                $extra_dn = undef;
                t("No XMLTV channel name associated with '$xmltv_id'");
            }
            else {
                $extra_dn{$xmltv_id} = $extra_dn;
                # t("Channel '$xmltv_id' has XMLTV name '$extra_dn'");
            }
        }
        
        # Check for channel icon
        if (defined $icon_url) {
            if ($icon_url eq '' || $icon_url !~ /^http/) {
                $icon_url = undef;
                t("No channel icon associated with '$xmltv_id'");
            }
            else {
                $icon_urls{$xmltv_id} = $icon_url;
                # t("Channel '$xmltv_id' has icon '$icon_url'");
            }
        }
        
        # Check for valid timeshift offset
        if (defined $channel_offset) {
            if ($channel_offset eq '' || $channel_offset !~ /^(\+|\-)/) {
                $channel_offset = undef;
            }
            else {
                $channel_offset{$xmltv_id} = $channel_offset;
                t("Channel '$xmltv_id' has timeshift of '$channel_offset'");
                $is_timeshift = 1;
            }
        }
        
        # Check for correct broadcast hours format (HHMM-HHMM)
        if (defined $broadcast_hours) {
            if ($broadcast_hours eq '' || $broadcast_hours !~ /\d{4}-\d{4}/) {
                $broadcast_hours = undef;
            }
            else {
                $broadcast_hours{$xmltv_id} = $broadcast_hours;
                t("Channel '$xmltv_id' is on air '$broadcast_hours'");
                $is_parttime = 1;
            }
        }

        # Check for presence of video quality information (SDTV or HDTV)
        if (defined $video_quality) {
            if ($video_quality eq '' || $video_quality !~ /SDTV|HDTV/) {
                $video_quality = undef;
            }
            else {
                $video_quality{$xmltv_id} = $video_quality;
            }
        }

        # Handle multiple XMLTV IDs associated with a single RT ID. Required
        # after introduction of timeshifted and part-time channel support,
        # which map multiple XMLTV IDs to a single RT ID.
        push @{$rt_to_xmltv{$rt_id}}, $xmltv_id;
        $xmltv_to_rt{$xmltv_id} = $rt_id;

        # Update the counts of part-time and timeshifted channels
        if ($is_timeshift && $is_parttime) {
            $num_pt_ts_channels++;
        }
        elsif ($is_timeshift) {
            $num_ts_channels++;
        }
        elsif ($is_parttime) {
            $num_pt_channels++;
        }

        # Finally, update count of good/bad channels
        if ($extra_dn =~ /\(Do\ Not\ Use\)/) {
            $num_bad_channels++;
        }
        else {
            $num_good_channels++;
        }
    }
    t("\n");
    # channel_ids processing finished

    die "Error: No usable XMLTV channel definitions seen in channel_ids, exiting"
        if (!defined $num_good_channels || $num_good_channels < 1);


    # Read in the Radio Times channels.dat file, which is supplied in UTF-8
    # format. We process the list of available channels and check for 
    # presence of duplicate IDs or names.
    #
    # Grab the octets
    t("Retrieving channel list from Radio Times website");
    my $rt_channels_dat = get_octets( $rt_channels_uri );

    die "Error: Radio Times channels.dat data is missing, exiting\n"
        . "Please check $rt_channels_uri"
        if (!defined $rt_channels_dat || $rt_channels_dat eq '');
        
    # Decode source UTF-8 octets, process for HTML entities, and encode 
    # into configured output encoding
    my $decoded_rt_channels_dat;
    t("\nDecoding channel data from $source_encoding octets into Perl's internal format");
    $decoded_rt_channels_dat = decode($source_encoding, $rt_channels_dat);
    t("Processing for HTML entities seen in the channel data");
    decode_entities($decoded_rt_channels_dat);
    t("Encoding channel data from Perl's internal format into $xml_encoding octets\n");
    $rt_channels_dat = encode($xml_encoding, $decoded_rt_channels_dat);

    my @rt_channels = split /\n/, $rt_channels_dat;
    my $num_rt_channels = scalar @rt_channels;

    $need_final_update = 0;
    my $chans_bar;
    if (!$opt->{quiet} && !$opt->{debug}) {
        $chans_bar = new XMLTV::ProgressBar({name   => 'Retrieving channels',
                                             count  => $num_rt_channels,
                                             ETA    => 'linear', });
    }

    # Hash to store details for <channel> elements
    my %channels;
    my (%seen_rt_id, %seen_name);
    my $num_good_rt_channels = 0;
    my $to_say = "";

    RT_CHANDAT_ENTRY:
    foreach my $rt_channel (@rt_channels) {
        chomp $rt_channel;
        # ignore empty line and disclaimer at start of file
        if ($rt_channel =~ /^\s*$/ || $rt_channel =~ /^In accessing this XML feed/) {
            next RT_CHANDAT_ENTRY;
        }

        if ($rt_channel !~ /^(\d+)\|(.+)/) {
            t("Bad entry '$rt_channel' seen in RT channels.dat, skipping");
            next RT_CHANDAT_ENTRY;
        }

        my ($rt_id, $rt_name) = ($1, $2);
        if ($seen_rt_id{$rt_id}++) {
            t("Duplicate channnel ID '$rt_id' seen in RT channels.dat, skipping");
            next RT_CHANDAT_ENTRY;
        }

        if ($seen_name{$rt_name}++) {
            t("Another channel name '$rt_name' seen in RT channels.dat, skipping");
            next RT_CHANDAT_ENTRY;
        }
        
        # Check whether there is at least one XMLTV ID associated with the RT ID
        #
        # If the current RT channel has a known XMLTV ID, check it against known bad
        # channels and skip it if required. If the channel does not have an 
        # XMLTV ID, create one and continue.
        #
        my $xmltv_id = $rt_to_xmltv{$rt_id}[0];
        if (defined $xmltv_id) {
            # Skip any RT entries which have been flagged as bad in channel_ids file
            if ($extra_dn{ $rt_to_xmltv{$rt_id}[0] } =~ /\(Do\ Not\ Use\)/) {
                t("Channel '$rt_name' ($rt_id) flagged as bad, skipping");
                $need_final_update = 1;
                next RT_CHANDAT_ENTRY;
            }
        }
        else {
            # Handle new channels available on RT site unknown to channel_ids file
            $to_say .= "Channel '$rt_name' ($rt_id) unknown to XMLTV, configuring\n";
            t("Will use XMLTV ID 'C$rt_id.radiotimes.com' during configuration\n");
            push @{$rt_to_xmltv{$rt_id}}, "C$rt_id.radiotimes.com";
        }

        foreach my $id (@{$rt_to_xmltv{$rt_id}}) {
            # Use a name for the channel if defined in our channel_ids file,
            # otherwise use the name supplied by the Radio Times.
            my @names = ();
            if (defined $extra_dn{$id}) {
                @names = ([ $extra_dn{$id} ]);
            }
            else {
                @names = ([ $rt_name ]);
            }

            # Add a URL for a channel icon if available.
            my @icon;
            my $icon_url = $icon_urls{$id};
            if ($icon_url) {
                @icon = { 'src' => $icon_url };
            }

            # Add the channel's details to the %channels hash, adding icon
            # details if available.
            if (@icon) {
                $channels{$id} = {
                    id             => $id,
                    rt_id          => $rt_id,
                    'display-name' => \@names,
                    'icon'         => \@icon,
                };
            }
            else {
                $channels{$id} = {
                    id             => $id,
                    rt_id          => $rt_id,
                    'display-name' => \@names,
                };
            }
        }
        # We have a usable channel definition at this point
        $num_good_rt_channels++;

        # Update the progres bar by one increment
        if (defined $chans_bar) {
            $chans_bar->update();
        }
    }

    die "Error: No usable Radio Times channel definitions available, exiting"
        if ($num_good_rt_channels < 1);

    if (defined $chans_bar) {
        # Only update the progress bar to 100% if we need to
        if ($need_final_update) {
            $chans_bar->update($num_rt_channels);
        }
        $chans_bar->finish();
        if (!$opt->{quiet}) {
            say( "\n" );
        }
    }

    if (!$opt->{quiet} && $to_say) {
        say( $to_say );
    }

    # Output statistics on the number of channels currently available
    if (!$opt->{quiet}) {
        say("\nThe Radio Times has usable data available for $num_good_rt_channels channels which we\n"
            . "can use to generate TV listings for regular and some timeshifted\n"
            . "channels. The tv_grab_uk_rt software also has support for an additional\n"
            . "$num_ts_channels timeshifted, $num_pt_channels part-time, and $num_pt_ts_channels part-time timeshifted channels\n"
            . "based on the Radio Times data.\n\n"
            . "In total, tv_grab_uk_rt currently supports $num_good_channels channels.\n");
    }

    # Report any channels listed in channel_ids not seen on the Radio Times
    # site
    if (!$opt->{quiet}) {
        XMLTV_ID:
        foreach my $xmltv_id (keys %xmltv_to_rt) {
            # Ignore channels flagged as bad in channel_ids
            next XMLTV_ID if ($extra_dn{$xmltv_id} =~ /.*Do\ Not\ Use.*/);
            if (!exists $channels{$xmltv_id}) {
                say("XMLTV channel '$xmltv_id' ($xmltv_to_rt{$xmltv_id}) " 
                   . "not seen on RT site\n");
            }
        }
    }

    return \%channels;
}

# Check that the requested channels are available from the Radio Times
#
sub check_configured_channels {
    my ( $available_channels ) = @_;
    
    # List of channel IDs that we can download listings for after checking
    # $opt->{channel} against the current RT/XMLTV channel list.
    my @wanted_chs;

    t("Reading config file channel entries");

    WANTED_CHAN:
    foreach my $chan_id (@{$conf->{channel}}) {
        t("  Read channel '$chan_id'");
        if (!exists ${$available_channels}{$chan_id}) {
            if (!$opt->{quiet}) {
                say("  Configured channel '$chan_id' is unavailable");
            }
            next WANTED_CHAN;
        }
        push @wanted_chs, $chan_id;
    }
    my $num_req_chans = scalar @wanted_chs;
    die "Error: No configured channels are available, exiting"
        if (!defined $num_req_chans || $num_req_chans < 1);
    t("Finished reading $num_req_chans configured channels");

    return \@wanted_chs;
}

# Retrieve and process mappings on channel ID to postcodes for regional 
# channels.
#
# Takes a postcode as an argument and returns i) a list of matching XMLTV IDs
# and ii) a list of the remaining regional XMLTV IDs that were not matched
#
sub get_channels_by_postcode {
    my $conf_postcode = shift;

    my $xmltvids_postcodes = GetSupplement("$grabber_name", 'regional_channels_by_postcode');

    die "Error: XMLTV regional_channels_by_postcode data is missing, exiting" 
        if (!defined $xmltvids_postcodes || $xmltvids_postcodes eq '');

    my @lines = split /[\n\r]+/, $xmltvids_postcodes;

    # Hash to hold matched and unmatched XMLTV IDs
    my %reg_chans;

    XMLTV_POSTCODE_ENTRY:
    foreach my $line (@lines) {
        # Skip blank lines. Comments are allowed if they are at the start 
        # of the line.
        next XMLTV_POSTCODE_ENTRY if ($line =~ '^#' || $line =~ '^$');
        my @fields = split /\|/, $line;
        # We need 2 fields (xmltv_id,postcodes).
        if (scalar @fields != 2) {
            t("Wrong number of fields in XMLTV regional_channels_by_postcode entry:\n"
                    . "\t" . $line);
            next XMLTV_POSTCODE_ENTRY;
        }
        my ( $xmltv_id, $postcodes ) = @fields;

        # Check for required XMLTV ID and postcode fields, skip if missing
        if (!defined $xmltv_id  || $xmltv_id eq '') {
            t("Undefined XMLTV ID seen in regional_channels_by_postcode, skipping");
            next XMLTV_POSTCODE_ENTRY;
        }
        if ($xmltv_id !~ /\w+\.\w+.*/) {
            t("Invalid XMLTV ID seen in regional_channels_by_postcode, skipping");
            next XMLTV_POSTCODE_ENTRY;
        }
        if (!defined $postcodes || $postcodes eq '') {
            t("Undefined postcode entry seen in regional_channels_by_postcode, skipping");
            next XMLTV_POSTCODE_ENTRY;
        }
        if (defined $xmltv_to_rt{$xmltv_id}) {
            t("Channel '$xmltv_id' has region-specific information available");
        }
        else {
            t("Channel '$xmltv_id' is not available, skipping ");
            next XMLTV_POSTCODE_ENTRY;
        }

        my @postcodes = split /,/, $postcodes;
        foreach my $postcode (@postcodes) {
            if (uc $postcode eq uc $conf_postcode) {
                push @{$reg_chans{matched}}, $xmltv_id;
                # match made, process next channel
                next XMLTV_POSTCODE_ENTRY;
            }
        }
        # No match made, so add to unmatched list
        push @{ $reg_chans{unmatched} }, $xmltv_id;
    }

    return \%reg_chans;
}

# Retrieve and process mappings on channel ID to TV platforms.
#
# Takes a platform ID as an argument and returns i) a list of matching XMLTV IDs
# and ii) a list of the remaining regional XMLTV IDs that were not matched
#
sub get_channels_by_platform {
    my $conf_platform = shift;

    my $xmltvids_platforms = GetSupplement("$grabber_name", 'channels_platforms');

    die "Error: XMLTV channels_platforms data is missing, exiting" 
        if (!defined $xmltvids_platforms || $xmltvids_platforms eq '');

    my @lines = split /[\n\r]+/, $xmltvids_platforms;

    # Hash to hold matched and unmatched XMLTV IDs
    my %platform_chans;

    XMLTV_PLATFORM_ENTRY:
    foreach my $line (@lines) {
        # Skip blank lines. Comments are allowed if they are at the start 
        # of the line.
        next XMLTV_PLATFORM_ENTRY if ($line =~ '^#' || $line =~ '^$');
        my @fields = split /\|/, $line;
        # We need 2 fields (xmltv_id,platform(s)).
        if (scalar @fields != 2) {
            t("Wrong number of fields in XMLTV channels_platforms entry:\n"
                    . "\t" . $line);
            next XMLTV_PLATFORM_ENTRY;
        }
        my ( $xmltv_id, $platforms ) = @fields;

        # Check for required XMLTV ID and platform fields, skip if missing
        if (!defined $xmltv_id  || $xmltv_id eq '') {
            t("Undefined XMLTV ID seen in channels_platforms, skipping");
            next XMLTV_PLATFORM_ENTRY;
        }
        if ($xmltv_id !~ /\w+\.\w+.*/) {
            t("Invalid XMLTV ID seen in channels_platforms, skipping");
            next XMLTV_PLATFORM_ENTRY;
        }
        if (!defined $platforms || $platforms eq '') {
            t("Undefined platform entry seen in channels_platforms, skipping");
            next XMLTV_PLATFORM_ENTRY;
        }
        if (defined $xmltv_to_rt{$xmltv_id}) {
            t("Channel '$xmltv_id' has platform information available");
        }
        else {
            t("Channel '$xmltv_id' is not available, skipping ");
            next XMLTV_PLATFORM_ENTRY;
        }

        my @platforms = split /,/, $platforms;
        foreach my $platform (@platforms) {
            if (uc $platform eq uc $conf_platform) {
                push @{$platform_chans{matched}}, $xmltv_id;
                # match made, process next channel
                next XMLTV_PLATFORM_ENTRY;
            }
        }
        # No match made, so add to unmatched list
        push @{ $platform_chans{unmatched} }, $xmltv_id;
    }

    return \%platform_chans;
}

# Determine options for, and create XMLTV::Writer object
sub setup_xmltv_writer {
    # output options
    my %g_args = ();
    if (defined $opt->{output}) {
        t("\nOpening XML output file '$opt->{output}'\n");
        my $fh = new IO::File ">$opt->{output}";
        die "Error: Cannot write to '$opt->{output}', exiting" if (!$fh);
        %g_args = (OUTPUT => $fh);
    }

    # Determine how many days of listings are required and range-check, applying
    # default values if impossible. If --days or --offset is specified we must
    # ensure that values for days, offset and cutoff are passed to XMLTV::Writer
    my %d_args = ();
    if (defined $opt->{days} || defined $opt->{offset}) {
        if (defined $opt->{days}) {
            if ($opt->{days} < 1 || $opt->{days} > 15) {
                if (!$opt->{quiet}) {
                    say("Specified --days option is not possible (1-15). "
                      . "Retrieving all available listings.");
                }
                $opt->{days} = 15
            }
        }
        else {
            $opt->{days} = 15;
        }

        if (defined $opt->{offset}) {
            if ($opt->{offset} < 0 || $opt->{offset} > 14) {
                if (!$opt->{quiet}) {
                    say("Specified --offset option is not possible (0-14). "
                      . "Retrieving all available listings.");
                }
                $opt->{offset} = 0;
            }
        }
        else {
            $opt->{offset} = 0;
        }
        $d_args{days} = $opt->{days};
        $d_args{offset} = $opt->{offset};
        # We currently don't provide a --cutoff option
        $d_args{cutoff} = "000000";
    }

    t("Setting up XMLTV::Writer using \"" . $xml_encoding . "\" for output");
    $writer = new XMLTV::Writer(%g_args, %d_args, encoding => $xml_encoding);
}

sub write_xmltv_header {
    t("Writing XMLTV header");
    $writer->start(\%tv_attributes);
}

sub write_channel_list {
    my ( $available_channels, $wanted_chs ) = @_;
    
    t("Started writing <channel> elements");
    foreach my $chan_id (@{$wanted_chs}) {
        my %h = %{ ${$available_channels}{$chan_id} };
        delete $h{rt_id};
        $writer->write_channel(\%h);
    }
    t("Finished writing <channel> elements");
}

# Read in the prog_titles_to_process file
sub load_prog_titles_to_process {
    my $prog_titles_to_process = undef;
    # Retrieve prog_titles_to_process via XMLTV::Supplement
    $prog_titles_to_process 
            = GetSupplement("$grabber_name", 'prog_titles_to_process');

    if (defined $prog_titles_to_process) {
        my @prog_titles_to_process = split /[\n\r]+/, $prog_titles_to_process;

        t("\nTitle processing information:\n");
        PROG_TITLE_ENTRY:
        foreach my $line (@prog_titles_to_process) {
            # Comments are allowed if they are at the start of the line
            next PROG_TITLE_ENTRY if ($line =~ /^#/);
            my @fields = split /\|/, $line;
            # Each entry requires 2 fields
            if (scalar @fields != 2) {
                t("Wrong number of fields in XMLTV prog_titles_to_process entry:\n"
                        . "\t" . $line);
                next PROG_TITLE_ENTRY;
            }
            # The prog_titles_to_process fields are:
            # 1) procesing code
            # 2) title/non-title text to process
            #
            my ($code, $process_text) = @fields;
            if (!defined $code || $code eq '' || $code !~ /\d+/) {
                t("Invalid title processing code: " . $line . "'");
                next PROG_TITLE_ENTRY;
            }

            if (!defined $process_text || $process_text eq '' 
                                       || $process_text !~ /\w+/) {
                t("Invalid title processing text: " . $line . "'");
                next PROG_TITLE_ENTRY;
            }

            # processing codes are documented in prog_titles_to_process file
            if ($code eq '1')  {
                push @non_title_info, $process_text;
                t("[1] Will remove '" . $process_text . "' from title " 
                  . "if found");
                $have_title_data = 1;
            }
            elsif ($code eq '2') {
                push @mixed_title_subtitle, $process_text;
                t("[2] Will check for subtitle after title for '" 
                  . $process_text . "'");
                $have_title_data = 1;
                next PROG_TITLE_ENTRY;
            }
            elsif ($code eq '3') {
                push @mixed_subtitle_title, $process_text;
                t("[3] Will check for subtitle before title for '" 
                  . $process_text . "'");
                $have_title_data = 1;
                next PROG_TITLE_ENTRY;
            }
            elsif ($code eq '4') {
                push @reversed_title_subtitle, $process_text;
                t("[4] Will check for reversed title/subtitle for '" 
                  . $process_text . "'");
                next PROG_TITLE_ENTRY;
            }
            elsif ($code eq '5') {
                my @fields = split( /~/, $process_text, 2);
                if (scalar @fields != 2) {
                    t("[5] Invalid number of fields (need 2) in processing text: '" . $process_text . "'");
                    next PROG_TITLE_ENTRY;
                }
                my( $old_title, $new_title ) = @fields;
                $replacement_titles{$old_title} = $new_title;
                t("[5] Will check for inconsistent title '" 
                  . $old_title . "'");
                next PROG_TITLE_ENTRY;
            }
            elsif ($code eq '6') {
                my @fields = split( /~/, $process_text, 2);
                if (scalar @fields != 2) {
                    t("[6] Invalid number of fields (need 2) in processing text: '" . $process_text . "'");
                    next PROG_TITLE_ENTRY;
                }
                my( $uncat_title, $cat ) = @fields;
                $replacement_cats{$uncat_title} = $cat;
                t("[6] Will assign title '" . $uncat_title 
                    . "' to category '" . $cat . "'");
                next PROG_TITLE_ENTRY;
            }
            elsif ($code eq '7') {
                my @fields = split( /~/, $process_text, 3);
                if (scalar @fields != 3) {
                    t("[7] Invalid number of fields (need 3) in processing text: '" . $process_text . "'");
                    next PROG_TITLE_ENTRY;
                }
                my( $ep_title, $old_ep, $new_ep ) = @fields;
                $replacement_episodes{$ep_title}->{$old_ep} = $new_ep;
                t("[7] Will check for inconsistent episode data '" . $old_ep 
                    . "' for title '" . $ep_title . "'");
                next PROG_TITLE_ENTRY;
            }
            elsif ($code eq '8') {
                my @fields = split( /~/, $process_text, 4);
                if (scalar @fields != 4) {
                    t("[8] Invalid number of fields (need 4) in processing text: '" . $process_text . "'");
                    next PROG_TITLE_ENTRY;
                }
                foreach my $field (@fields) {
                    $field = "" if !defined $field;
                }
                my( $old_title, $old_ep, $new_title, $new_ep ) = @fields;
                if ($old_title eq '' or $new_title eq '') {
                    t("[8] Ignoring fixup '" . $process_text . "' as old/new title not given");
                    next PROG_TITLE_ENTRY;
                }
                # remember old title so that we can output a debug list of
                # programmes that may also need to be handled via this fixup
                $flagged_title_eps{$old_title} = $old_title;
                
                my $key = ("" . $old_title . "|" . $old_ep);
                $replacement_title_eps{$key} = [$new_title, $new_ep];
                t("[8] Will update old title/subtitle '" . $old_title . ": " . $old_ep 
                    . "' to '" . $new_title . ": " . $new_ep . "'");
                next PROG_TITLE_ENTRY;
            }
            elsif ($code eq '9') {
                my @fields = split( /~/, $process_text, 3);
                if (scalar @fields != 3) {
                    t("[9] Invalid number of fields (need 3) in processing text: '" . $process_text . "'");
                    next PROG_TITLE_ENTRY;
                }
                my( $title, $episode, $desc ) = @fields;
                $replacement_ep_from_desc{$title}->{$desc} = $episode;
                t("[9] Will update subtitle to '" . $episode . "' for title '" . $title 
                    . "' based on given description '" . $desc . "'");
                next PROG_TITLE_ENTRY;
            }
            else {
                t("Unknown code seen in prog_titles_to_process file," 
                  . " skipping entry '" . $line . "'");
                next PROG_TITLE_ENTRY;
            }
        }
    }
    else {
        if (!$opt->{quiet}) {
            say("Disabling title processing, no information found.");
        }
    }
    if (!$opt->{quiet}) {
        say("\n");
    }
}

# Read in the utf8_fixups file
sub load_utf8_fixups {
    my $utf8_fixups = undef;
    # Retrieve utf8_fixups via XMLTV::Supplement
    $utf8_fixups 
            = GetSupplement("$grabber_name", 'utf8_fixups');

    if (defined $utf8_fixups) {
        my @utf8_fixups = split /[\n\r]+/, $utf8_fixups;

        t("\nLoading UTF-8 fixups\n");
        UTF8_FIXUP_ENTRY:
        foreach my $line (@utf8_fixups) {
            # Comments are allowed if they are at the start of the line
            next UTF8_FIXUP_ENTRY if ($line =~ /^#/);
            my @fields = split /\|/, $line;
            # Each entry requires 2 fields
            if (scalar @fields != 2) {
                t("Wrong number of fields in XMLTV UTF-8 fixup entry:\n"
                        . "\t" . $line);
                next UTF8_FIXUP_ENTRY;
            }
            # The utf8_fixups fields are:
            # 1) bad utf-8 characters to find and replace (as hex)
            # 2) the replacement characters (as hex)
            #
            my ($bad_chars, $replacement) = @fields;
            if (!defined $bad_chars || $bad_chars eq '') {
                t("Invalid UTF-8 fixup regex: '" . $line . "'");
                next UTF8_FIXUP_ENTRY;
            }

            if (!defined $replacement || $replacement eq '') {
                t("Invalid UTF-8 fixup replacement: '" . $line . "'");
                next UTF8_FIXUP_ENTRY;
            }
            if ($bad_chars =~ /\\xEF\\xBF\\xBD/ || $bad_chars =~ /\\xC3\\xAF\\xC2\\xBF\\xC2\\xBD/) {
            }
            else {
                t("Ignoring UTF-8 fixup handled in grabber: '" . $line . "'");
                next UTF8_FIXUP_ENTRY;
            }

            # Remove the \x chars read from the file leaving a simple hex string
            # containing only [0-9A-F] chars
            $replacement =~ s/\\x//g;
            # Now convert each byte (2 hex chars) into its character equivalent
            $replacement =~ s/([0-9A-F][0-9A-F])/chr(hex($1))/eig;

            # Process the regex to get a character string to print. We'll use
            # the preserved hex string during processing later
            my $bad_chars_chr = $bad_chars;
            $bad_chars_chr =~ s/\\x//g;
            $bad_chars_chr =~ s/([0-9A-F][0-9A-F])/chr(hex($1))/eig;

            $utf8_fixups{$bad_chars} = $replacement;
            t("UTF-8 fixup: will replace \"" . $bad_chars_chr . "\" with \"" 
                    . $replacement . "\" if seen");
            next UTF8_FIXUP_ENTRY;
        }
    }
    else {
        if (!$opt->{quiet}) {
            say("No additional UTF-8 fixups were found.");
        }
    }
    if (!$opt->{quiet}) {
        say("\n");
    }
}

# Download listings data for configured channels that are available
sub write_listings_data {
    my ( $available_channels, $wanted_chs ) = @_;
    
    my $num_req_chans = scalar @{$wanted_chs};

    if (!$opt->{quiet}) {
        display_copyright();
    }

    if (!$opt->{quiet}) {
        say("Will download listings for $num_req_chans configured channels\n");
    }

    my $listings_bar;
    if (!$opt->{quiet} && !$opt->{debug}) {
        $listings_bar = new XMLTV::ProgressBar({name  => 'Retrieving listings',
                                                count => $num_req_chans,
                                                ETA   => 'linear', });
    }

    # Was title processing enabled in config file?
    if ($title_processing eq 'enabled') {
        t("Extra title processing is enabled\n");
        load_prog_titles_to_process();
    }
    else {
        t("Extra title processing is disabled\n");
    }

    # Load the UTF-8 fixups
    load_utf8_fixups();

    # Hash to hold warnings of incorrect number of fields. The warning
    # is given once per listings file if noticed more than once
    my %warned_wrong_num_fields;

    # Reset check for final progress bar update
    $need_final_update = 0;

    # Process all of the channels we want listings for
    WANTED_CH:
    foreach my $ch (@{$wanted_chs}) {
        my $c = ${$available_channels}{$ch};
        my $xmltv_id = $c->{id};
        my $rt_id = $c->{rt_id};
        my $rt_name = $c->{'display-name'}->[0]->[0];
        if (!defined $rt_id) {
            t("No Radio Times ID for channel '$rt_name', skipping");
            next WANTED_CH;
        }

        # Create the channel's URL based on ID
        my $rt_listings_uri = "$rt_root_dir/$rt_id.dat";
        # Include the URL in any warn/die messages
        local $SIG{__DIE__} = sub { die "$rt_listings_uri: $_[0]" };
        local $SIG{__WARN__} = sub { warn "$rt_listings_uri: $_[0]" };

        # Read in the listings data for the channel as UTF-8 octets. We will 
        # process the raw octets before decoding them to Perl's internal
        # format below.
        t("\nRetrieving listings for '$rt_name'");
        my $page = get_octets( $rt_listings_uri );

        if (!defined $page || $page eq '') {
            if (!$opt->{quiet}) {
                say("No listings data available for '$rt_name' ($xmltv_id), skipping");
            }
            $warnings++;
            next WANTED_CH;
        }
        if (!$opt->{quiet}) {
            say("Processing listings for '$rt_name' ($xmltv_id)");
        }
        t("");
        if (defined $channel_offset{$xmltv_id}) {
            t("  Detected a channel offset of '$channel_offset{$xmltv_id}'" 
              . " for '$rt_name'");
        }

        # If the Radio Times name for the channel contains timezone information,
        # use it, otherwise set the timezone to default of UTC
        my $base_tz;
        if ($rt_name =~ /\((UTC|GMT|CET)\)\s*$/) {
            $base_tz = $1;
            t("  Base timezone for utc_offset set to '$base_tz' (via channel name)\n");
        }
        else {
            $base_tz = 'UTC';
            t("  Base timezone for utc_offset set to 'UTC' (default)\n");
        }

        # Tidy up any bad characters in the Radio Times source data. Although the
        # data is provided in UTF-8 format the text may contain mis-encoded UTF-8
        # or the NULL character which should be corrected as far as possible 
        # before processing continues.
        #
        # Double-encoded UTF-8 characters (4 bytes)
        # =========================================
        #
        # The ISO-8859-1 charset contains 256 codepoints (0x00-0xFF). When encoded
        # into UTF-8, either 1 or 2 bytes are required to encode these characters
        # as follows:
        # 
        # ISO-8859-1            UTF-8        Chars in    Bytes      Notes
        #    range          byte(s) range     Range     Required
        #
        #  0x00-0x1F      [00]-[1F]             32         1        Non-printing
        #  0x20-0x7F      [20]-[7F]             96         1        Printing
        #  0x80-0x9F      [C2][80]-[C2][9F]     32         2        Non-printing
        #  0xA0-0xBF      [C2][A0]-[C2][BF]     32         2        Printing
        #  0xC0-0xFF      [C3][80]-[C3][BF]     64         2        Printing
        #
        # A double-encoded UTF-8 character that uses 4 bytes (but should use only 2
        # if properly encoded) uses the first 2 bytes to contain the UTF-8
        # representation of the first byte of the proper UTF-8 representation of the
        # character, and the second 2 bytes to contain the UTF-8 representation of the second
        # byte.
        #
        # E.g.:
        #
        # The data contains a double-encoded UTF-8 encoding of the A-grave character using 
        # 4 bytes. The correct UTF-8 encoding of this character is [C3][80]. The data 
        # actually contains the 4 bytes [C3][83][C2][80]. [C3][83] is the UTF-8 encoding 
        # of [C3], and [C2][80] is the UTF-8 encoding of [80]. We therefore replace this 
        # 4-byte double-encoding with [C3][80] which is valid UTF-8 and can be successfully
        # encoded into other character encodings if required.
        #
        # The range of Unicode codepoints encoded into 2 bytes in UTF-8 lie in the 
        # range [C2-DF][80-BF].
        #
        # http://en.wikipedia.org/wiki/ISO/IEC_8859-1
        # http://en.wikipedia.org/wiki/UTF-8
        # http://www.eki.ee/letter/
        #
        # Double-encoded UTF-8 General Punctuation characters (6 bytes)
        # =============================================================
        #
        # Occasionally in the listings we see double-encoded characters from
        # the Unicode General Punctuation range of characters. When encoded into
        # UTF-8 these characters should require 3 bytes. However, when double-encoded 
        # they take 6 bytes. During their handling we replace them with their ASCII
        # equivalents which are how the characters are usually included in the 
        # listings.
        #
        # Mis-encoded UTF-8 characters (6/8 bytes)
        # ========================================
        #
        # Frequently seen in the data (especially in film listings) are completely 
        # mis-encoded sequences of UTF-8 characters. Each sequence of bad bytes 
        # starts with a correctly encoded 2 byte UTF-8 character but it then
        # followed by 2 or 3 mis-encoded ASCII-range characters. When encoded into
        # UTF-8 these ASCII chars should take 1 byte each, but in this situation
        # use 2 bytes which then fail to decode or display correctly.
        #
        # Unicode Replacement Character (U+FFFD)
        # ======================================
        #
        # The UTF-8 source data may also contain the bytes [EF][BF][BD] which 
        # are the UTF-8 encoding of the Unicode Replacement Character U+FFFD.
        # It is likely that these are introduced during preparation of the 
        # listings data by the Radio Times, as any characters that cannot be 
        # understood are replaced by this character.
        #
        # Double-encoded Unicode Replacement Character (6 bytes)
        # ======================================================
        #
        # The correct encoding for the Unicode Replacement Character is [EF][BF][BD],
        # however it has been seen double-encoded in the listings data as 
        # [C3][AF][C2][BF][C2][BD]. As with the normal replacement character, there 
        # is no way to determine which replacement character to use in this case, 
        # so we substitute a '?' char if we cannot handle the specific occurence. 
        # This error needs to have been seen at least once in source data to be able 
        # to construct a suitable fixup.
        #
        t("  Checking '$rt_name' listings data for bad UTF-8 chars...");
        for ($page) {
            # Programme entries containing RT reviews or updated information
            # may have erroneous CR+SP characters which we tidy up here
            t("    Looking for CR+SP characters...");
            if ( s/\x0D\x20//g ) {
                t("      Removed CR+SP characters from '$rt_name' listings data");
            }


            # Fixup any double-encoded UTF-8 chars (4 bytes)
            t("    Looking for double-encoded UTF-8 characters...");
            if ( /[\xC3][\x82-\x83][\xC2][\x80-\xBF]/ ) {
                # first capture each set of double-encoded UTF-8 bytes 
                # (4 in total, 2 for each "real" UTF-8 char) into a list
                my @double_bytes = ($page =~ /[\xC3][\x82-\x83][\xC2][\x80-\xBF]/g);

                # get a unique list of the different doubly encoded bytes
                my %unique_double_bytes;
                foreach(@double_bytes) {
                    $unique_double_bytes{$_} = $_;
                }
                # Get a list of unique 4-byte sequences
                @double_bytes = sort values %unique_double_bytes;
                foreach (@double_bytes) {
                    t("      Found double-encoded bytes: " . $_);
                }
                # process the list, reading 2 pairs of bytes in each iteration
                foreach (@double_bytes) {
                    /([\xC3][\x82-\x83])([\xC2][\x80-\xBF])/;
                    my $badbytes_1 = $1;
                    my $badbytes_2 = $2;
                    # convert each pair of bytes from UTF-8 to ISO-8859-1 to get a single 
                    # byte from the original pair
                    my $goodbyte_1 = encode("iso-8859-1", decode("utf-8", $badbytes_1) );
                    my $goodbyte_2 = encode("iso-8859-1", decode("utf-8", $badbytes_2) );
                    # finally globally replace each group of 4 bad bytes with 
                    # the 2 correct replacement bytes
                    $page =~ s/$badbytes_1$badbytes_2/$goodbyte_1$goodbyte_2/g;
                    t("      Replaced bad bytes '" . $badbytes_1 . $badbytes_2 
                                    . "' with good bytes '" . $goodbyte_1 . $goodbyte_2 . "'");
                }
            }

            # Fixup any double-encoded UTF-8 General Punctuation chars (6 bytes) 
            # with their ASCII equivalents (1 byte)
            t("    Looking for double-encoded UTF-8 General Punctuation characters...");
            if ( /[\xC3][\xA2][\xC2][\x80-\x81]/ ) {
                t("      Replaced double-encoded 6-byte UTF-8 General Punctuation chars");
                s/\xC3\xA2\xC2\x80\xC2\x90/\x2D/g; # <2D> -> -
                s/\xC3\xA2\xC2\x80\xC2\x91/\x2D/g; # <2D> -> -
                s/\xC3\xA2\xC2\x80\xC2\x92/\x2D/g; # <2D> -> -
                s/\xC3\xA2\xC2\x80\xC2\x93/\x2D/g; # <2D> -> -
                s/\xC3\xA2\xC2\x80\xC2\x94/\x2D/g; # <2D> -> -
                s/\xC3\xA2\xC2\x80\xC2\x95/\x2D/g; # <2D> -> -
                s/\xC3\xA2\xC2\x80\xC2\x98/\x27/g; # <27> -> '
                s/\xC3\xA2\xC2\x80\xC2\x99/\x27/g; # <27> -> '
                s/\xC3\xA2\xC2\x80\xC2\x9A/\x27/g; # <27> -> '
                s/\xC3\xA2\xC2\x80\xC2\x9C/\x22/g; # <22> -> "
                s/\xC3\xA2\xC2\x80\xC2\x9D/\x22/g; # <22> -> "
                s/\xC3\xA2\xC2\x80\xC2\x9E/\x22/g; # <22> -> "
                s/\xC3\xA2\xC2\x80\xC2\x9F/\x22/g; # <22> -> "
                s/\xC3\xA2\xC2\x80\xC2\xA6/\x2E\x2E\x2E/g; # <2E><2E><2E> -> ...
            }

            # Fixup any mis-encoded character sequences in range 
            # [C3][A0-AF][C2][80-BF][C2][80-BF] (6 bytes)
            t("    Looking for mis-encoded [C3][A0-AF] bytes...");
            if ( /[\xC3][\xA0-\xAF][\xC2][\x80-\xBF][\xC2][\x80-\xBF]/ ) {
                # first capture each sequence of mis-encoded UTF-8 bytes 
                # (6 in total)
                my @misencoded_bytes = 
                        ($page =~ /[\xC3][\xA0-\xAF][\xC2][\x80-\xBF][\xC2][\x80-\xBF]/g);
                # get a unique list of the different mis-encoded byte sequences
                my %unique_misencoded_bytes;
                MIS_ENC_BYTE:
                foreach (@misencoded_bytes) {
                    # the Unicode Replacement Character is handled below, so ignore here
                    # (when double-encoded, it will match the regex above)
                    if (/\xC3\xAF\xC2\xBF\xC2\xBD/) {
                        t("      Ignoring double-encoded Unicode Replacement Character (handled separately)");
                        next MIS_ENC_BYTE;
                    }
                    $unique_misencoded_bytes{$_} = $_;
                }
                # Get a new list of the unique 6-byte sequences
                @misencoded_bytes = sort values %unique_misencoded_bytes;
                foreach (@misencoded_bytes) {
                    t("      Found mis-encoded bytes: " . $_);
                }
                # process the list, reading 4 bytes in each iteration. Bytes
                # 1 and 2 are correct and left untouched, bytes 4 and 6 are 
                # extracted and corrected before being output
                foreach (@misencoded_bytes) {
                    /([\xC3][\xA0-\xAF])[\xC2]([\x80-\xBF])[\xC2]([\x80-\xBF])/;
                    my $goodbytes = $1; # correct, and used in replacement
                    my $badbyte_1 = $2; # incorrect byte value
                    my $badbyte_2 = $3; # incorrect byte value
                    # the bad bytes require 0x40 (DEC 64) to be subtracted from the char 
                    # value. 0xA0 are a special case and always converted to regular
                    # space char (0x20)
                    my $goodbyte_1;
                    if ($badbyte_1 !~ /\xA0/) {
                        $goodbyte_1 = chr( (ord $badbyte_1) - 64);
                    }
                    else {
                        $goodbyte_1 = "\x20";
                    }
                    my $goodbyte_2;
                    if ($badbyte_2 !~ /\xA0/) {
                        $goodbyte_2 = chr( (ord $badbyte_2) - 64);
                    }
                    else {
                        $goodbyte_2 = "\x20";
                    }
                    # finally globally replace each sequence of bad bytes with 
                    # the correct replacement bytes
                    $page =~ s/$_/$goodbytes$goodbyte_1$goodbyte_2/g;
                    t("      Replaced mis-encoded [C3][A0-AF] bytes '" . $_
                                    . "' with bytes '" 
                                    . $goodbytes . $goodbyte_1 . $goodbyte_2 . "'");
                }
            }

            # Fixup any mis-encoded character sequences in range 
            # [C3][B0-BF][C2][80-BF][C2][80-BF][C2][80-BF] (8 bytes)
            t("    Looking for mis-encoded [C3][B0-BF] bytes...");
            if ( /[\xC3][\xB0-\xBF][\xC2][\x80-\xBF][\xC2][\x80-\xBF][\xC2][\x80-\xBF]/ ) {
                # first capture each sequence of mis-encoded UTF-8 bytes 
                # (8 in total)
                my @misencoded_bytes = 
                        ($page =~ /[\xC3][\xB0-\xBF][\xC2][\x80-\xBF][\xC2][\x80-\xBF][\xC2][\x80-\xBF]/g);
                # get a unique list of the different mis-encoded byte sequences
                my %unique_misencoded_bytes;
                foreach(@misencoded_bytes) {
                    $unique_misencoded_bytes{$_} = $_;
                }
                # Get a new list of the unique 8-byte sequences
                @misencoded_bytes = sort values %unique_misencoded_bytes;
                foreach (@misencoded_bytes) {
                    t("      Found mis-encoded bytes: " . $_);
                }
                # process the list, reading 5 bytes in each iteration. Bytes
                # 1 and 2 are correct and left untouched, bytes 4, 6 and 8 are 
                # extracted and corrected before being output
                foreach (@misencoded_bytes) {
                    /([\xC3][\xB0-\xBF])[\xC2]([\x80-\xBF])[\xC2]([\x80-\xBF])[\xC2]([\x80-\xBF])/;
                    my $goodbytes = $1; # correct, and used in replacement
                    my $badbyte_1 = $2; # incorrect byte value
                    my $badbyte_2 = $3; # incorrect byte value
                    my $badbyte_3 = $4; # incorrect byte value
                    # the bad bytes require 0x40 (DEC 64) to be subtracted from the char 
                    # value. 0xA0 are a special case and always converted to regular
                    # space char (0x20)
                    my $goodbyte_1;
                    if ($badbyte_1 !~ /\xA0/) {
                        $goodbyte_1 = chr( (ord $badbyte_1) - 64);
                    }
                    else {
                        $goodbyte_1 = "\x20";
                    }
                    my $goodbyte_2;
                    if ($badbyte_2 !~ /\xA0/) {
                        $goodbyte_2 = chr( (ord $badbyte_2) - 64);
                    }
                    else {
                        $goodbyte_2 = "\x20";
                    }
                    my $goodbyte_3;
                    if ($badbyte_3 !~ /\xA0/) {
                        $goodbyte_3 = chr( (ord $badbyte_3) - 64);
                    }
                    else {
                        $goodbyte_3 = "\x20";
                    }
                    # finally globally replace each sequence of bad bytes with 
                    # the correct replacement bytes
                    $page =~ s/$_/$goodbytes$goodbyte_1$goodbyte_2$goodbyte_3/g;
                    t("      Replaced mis-encoded [C3][B0-BF] bytes '" . $_
                                    . "' with bytes '" 
                                    . $goodbytes . $goodbyte_1 . $goodbyte_2 . $goodbyte_3 . "'");
                }
            }

            # Here we replace specific sequences of characters seen in the source 
            # data that cannot be handled automatically above. These include
            # occurences of the Unicode Replace Character (single and double
            # encoded).
            # 
            # We use a supplemental file to store these fixups to allow updating
            # without needing to update the grabber itself.
            #
            t("    Looking for Unicode Replacement Character...");
            if ( /\xEF\xBF\xBD/ || /\xC3\xAF\xC2\xBF\xC2\xBD/ ) {

                if (%utf8_fixups) {
                    foreach my $bad_chars (keys %utf8_fixups) {
                        my $replacement = $utf8_fixups{$bad_chars};
                        # Now search for the regex string and replace with char string
                        if ($page =~ s/$bad_chars/$replacement/g) {
                            t("      Replaced Unicode Replacement Character with \"" 
                                    . $replacement . "\"");
                        }
                    }
                }
                if ( $page =~ s/\xEF\xBF\xBD/\x3F/g ) {
                    t("    After fixups, data for '$rt_name' still contained Unicode "
                            . "Replacement character. Replaced with \"?\"\n");
                    $hadEFBFBD{$rt_name} = $rt_listings_uri;
                }
                if ( $page =~ s/\xC3\xAF\xC2\xBF\xC2\xBD/\x3F/g ) {
                    t("    After fixups, data for '$rt_name' still contained "
                            . "double-encoded Unicode Replacement character. "
                            . "Replaced with \"?\"\n");
                    # track and report any that we've missed
                    $hadC3AFC2BFC2BD{$rt_name} = $rt_listings_uri;
                }
            }

            # after fixups replace any non-breaking space chars with regular spaces
            t("    Looking for non-breaking spaces...");
            if ( s/\xC2\xA0/\x20/g ) {
                t("      Replaced non-breaking space chars with regular spaces");
            }

            # Check for remaining potentially bad UTF-8 chars that we might need
            # to handle (below we remove any in range [C2][7F-9F])
            if ( $opt->{debug} && /\xC2[\x7F-\x9F]/ ) {
                t("    After fixups, data for '$rt_name' still contains " 
                        . "characters in range [C2][7F-9F]\n");
                # track and report any that we've missed
                $hasC27FBFchars{$rt_name} = $rt_listings_uri;
            }

            # Finally, remove any remaining non-printing control characters (keep \t \n and \r)
            # Refer to above table for ISO-8859-1 and UTF-8 Unicode encodings for chars
            #
            # First, chars in UTF-8 range [00-1F] (ISO-8859-1 range [00-1F])
            if ( s/[\x00-\x08\x0B-\x0C\x0E-\x1F]//g ) {
                t("    Removed non-printing characters (range [00]-[1F]) from "
                        . "'$rt_name' listings data");
            }
            # Next, chars in UTF-8 range [C2][7F-9F] (ISO-8859-1 range [7F-9F])
            if ( s/[\xC2][\x7F-\x9F]//g ) {
                t("    Removed non-printing characters (range [C2][7F-9F]) from "
                        . "'$rt_name' listings data");
            }
        }
        # end of UTF-8 fixups

        # Decode source UTF-8 octets, process for HTML entities, and encode 
        # into configured output encoding
        my $decoded_page;
        t("\nDecoding listings data from $source_encoding octets into Perl's internal format");
        $decoded_page = decode($source_encoding, $page);
        t("Processing for HTML entities seen in the listings data");
        decode_entities($decoded_page);
        t("Encoding listings data from Perl's internal format into $xml_encoding octets\n");
        $page = encode($xml_encoding, $decoded_page);

        ##### From this point, $data is in octets #####

        # Start to process individual programme entries found in listings
        t("  Started writing <programme> elements for channel '$rt_name'\n");

        # list to store programme elements for writing when each channel is parsed
        my @programmes = ();

        # Track number of programmes per channel
        my $num_titles = 0;

        # Keep a reference to the previous programme that was processed. Set it
        # to undef initially as programmes are processed in transmission order.
        # We update it whenever a programme is successfully processed below.
        my $previous_programme = undef;

        PROGRAMME:
        foreach my $prog (split /\n/, $page) {

            # ignore empty line and disclaimer at start of each file
            if ( $prog =~ /^\s*$/ || $prog =~ /^In accessing this XML feed/) {
                next PROGRAMME;
            }

            my @fields = split /\~/, $prog;
            if (scalar @fields != 23) {
                if ($warned_wrong_num_fields{$ch}++) {
                    t("  Wrong number of fields in line:\n  $prog\n");
                }
                next PROGRAMME;
            }
            # Remove any spaces at start/end of fields
            foreach my $field (@fields) {
                $field =~ s/^\s+//;
                $field =~ s/\s+$//;
                undef $field if !length $field;
            }
            
            # Description of Radio Times data fields (23 in total):
            #
            # title - the programme title (text)
            # sub_title - infrequently defined - preference is given to episode
            #             if defined in the source data (text)
            # episode - the episode's position in the current series and/or the 
            #           name of the particular episode of the programme (text)
            # year - the year of production (text)
            # director - the programme's director(s) (text)
            # cast - the programme's cast (may include character details) (text)
            # premiere - whether this is a film's first showing (boolean)
            # film - whether the programme is a film (boolean)
            # repeat - whether the programme has been shown before (boolean)
            # subtitles - whether subtitles are available (boolean)
            # widescreen - whether the broadcast is 16:9 widescreen (boolean)
            # new_series - whether the programme is the first episode in a 
            #              series new (boolean)
            # deaf_signed - whether in-vision signing is available (boolean)
            # blank_and_white - whether the broadcast is not in colour (boolean)
            # star_rating - a star rating between 0 and 5 for films (text)
            # certificate - the BBFC certificate for the programme (text)
            # genre - the genre of the programme (text)
            # desc - a description of the programme. Can be a specific review by a
            #        Radio Times reviewer (text)
            # choice - whether the programme is recommended by the 
            #          Radio Times (boolean)
            # date - the transmission date (text)
            # start - the transmission start time for the programme (text)
            # stop - the transmissions stop time for the programme (text)
            # duration_mins - the duration of the programme in minutes (text)
            my ($title,       $sub_title,       $episode,         $year,
                $director,    $cast,            $premiere,        $film,
                $repeat,      $subtitles,       $widescreen,      $new_series,
                $deaf_signed, $black_and_white, $star_rating,     $certificate,
                $genre,       $desc,            $choice,          $date,
                $start,       $stop,            $duration_mins,
            ) = @fields;

            my $updated_listing_info; # placeholder variable for future use

            if (!defined $title) {
                t("  Missing title in entry '$prog', skipping");
                next PROGRAMME;
            }
            t("  Processing programme title '$title'");

            if (!defined $date) {
                t("  Missing date in entry '$prog', skipping");
                next PROGRAMME;
            }

            # Check the true/false fields for valid data
            foreach my $field ($premiere,    $film,            $repeat, 
                               $subtitles,   $widescreen,      $new_series,
                               $deaf_signed, $black_and_white, $choice, ) {
                if (!defined $field) {
                    t("  A required true/false value was undefined for '$title', skipping");
                    next PROGRAMME;
                }
                elsif ($field eq 'true') {
                    $field = 1;
                }
                elsif ($field eq 'false') {
                    $field = 0;
                }
                else {
                    t("  A bad true/false value '$field' was seen for '$title', skipping");
                    next PROGRAMME;
                }
            }

            # Check for any DST-related information the RT may include in the title
            # for a programme. If we find any explicit DST information we store it 
            # for use later and remove it from the title.
            my $explicit_tz;
            if ($title =~ s/^\((GMT|UTC|BST|UTC\+1)\)\s*//) {
                $explicit_tz = $1;
            }

            # Prefer $episode over $sub_title if both given in source data.
            # Retain episode $sub_title data if $episode contains only episode
            # numbering.
            if (defined $sub_title) {
                if (defined $episode) {
                    if ( $episode =~ /^\d+\s*\/\s*\d+$/ ) {
                        t("  Merging sub-title '$sub_title' with episode numbering '$episode' provided separately");
                        $episode = $episode . ": " . $sub_title;
                        $sub_title = undef;
                    }
                    else {
                        t("  Ignoring sub-title '$sub_title' as episode '$episode' provided");
                        $sub_title = undef;
                    }
                }
                else {
                    t("  Using sub-title '$sub_title' as episode not given");
                    $episode = $sub_title;
                    $sub_title = undef;
                }
            }

            # Remove production year information from $episode for films
            if (defined $episode) {
                if ($film && $episode =~ s/Prod Year (\d{4})//i) {
                    t("  Removed production year info from episode details");
                    $episode = undef;
                    if (!defined $year) {
                        $year = $1;
                    }
                }
            }

            # Title and episode processing. We process titles if the user has 
            # not explicitly disabled title processing during configuration
            # and we have supplement data to process programmes against.
            #
            # We run a series of search and replace routines to clean up 
            # the title/episode information. Leaving non-title information in 
            # the title, or having inconsistent title/episode formatting  will 
            # result in PVR applications being unable to consistently match 
            # programme titles and episode information.
            #
            # Some listings appear to be added without being processed upstream 
            # to provide subtitle (episode) information. The titles of these
            # programmes are uppercase and may contain season numbering. Here
            # we monitor these before further title processing is carried out.
            if ($title eq uc($title)) {
                $uc_prog_titles{$title} = $title;
            }
            
            # First, remove any non-title information found in the title. This 
            # information is placed at the start of the 'real' title, separated 
            # by a colon (to date, this has always been the case in the RT
            # listings).
            #
            if ($have_title_data && @non_title_info && $title =~ /:/) {

                NON_TITLE_TEXT:
                foreach my $non_title_info (@non_title_info) {
                    if ($title =~ s/^($non_title_info)\s*:\s*//) {
                        t("  Removed '" . $non_title_info 
                          . "' from title. New title '" . $title . "'");
                        last NON_TITLE_TEXT;
                    }
                }
            }

            # If the title still contains a colon or hyphen, there is a chance it may
            # contain merged title and episode data, which we now process.
            if ($have_title_data && $title =~ /:|-/) {

                # Some programme titles contain both the title and episode,
                # separated by a colon ($title:$episode) or a hyphen 
                # ($title - $episode). Here we reassign the episode to the 
                # $episode element, leaving only the programme's title in the 
                # $title element
                #
                if (@mixed_title_subtitle) {

                    MIXED_TITLE_SUBTITLE:
                    foreach my $mixed_title_subtitle (@mixed_title_subtitle) {
                        if ($title =~ /^($mixed_title_subtitle)\s*(?::|-)\s*(.*)/) {
                            # store the captured text
                            my $new_title = $1;
                            my $new_episode = $2;
                            if (!defined $episode) {
                                t("  Moved '" . $new_episode . "' to sub-title,"
                                  . " new title is '" . $new_title . "'");
                                $title = $new_title;
                                $episode = $new_episode;
                                last MIXED_TITLE_SUBTITLE;
                            }
                            elsif ($episode eq $new_episode) {
                                t("  Sub-title '" . $episode . "' seen in "
                                  . "title already exists, new title is '"
                                  . $new_title . "'");
                                $title = $new_title;
                                last MIXED_TITLE_SUBTITLE;
                            }
                            # concat subtitle after any episode numbering (x/y)
                            elsif ($episode =~ /^\d+\s*(?:&\d+)?\s*\/\s*\d+\s*(?:\.|\/|,|;|:)?\s*-?\s*(?:series\s*(\w+)\s*(?:;|\.)?)?\s*$/) {
                                t("  Concatenating sub-title '" . $new_episode 
                                  . "' seen in title after existing episode numbering '" 
                                  . $episode . "'");
                                $title = $new_title;
                                $episode = $episode . ": " . $new_episode;
                                last MIXED_TITLE_SUBTITLE;
                            }
                            else {
                                t("  Concatenating sub-title '" . $new_episode 
                                  . "' seen in title with existing episode info '" 
                                  . $episode . "'");
                                $title = $new_title;
                                $episode = $new_episode . ": " . $episode;
                                last MIXED_TITLE_SUBTITLE;
                            }
                        }
                    }
                }

                # Some programme titles contain both the episode and title,
                # separated by a colon ($episode:$title) or a hyphen
                # ($episode - $title). Here we reassign the episode to the 
                # $episode element, leaving only the programme's title in the 
                # $title element.
                #
                if (@mixed_subtitle_title) {

                    MIXED_SUBTITLE_TITLE:
                    foreach my $mixed_subtitle_title (@mixed_subtitle_title) {
                        if ($title =~ /^(.*)\s*(?::|-)\s*($mixed_subtitle_title)/) {
                            # store the captured text
                            my $new_title = $2;
                            my $new_episode = $1;
                            if (!defined $episode) {
                                t("  Moved '" . $new_episode . "' to sub-title, " 
                                  . "new title is '" . $new_title . "'");
                                $title = $new_title;
                                $episode = $new_episode;
                                last MIXED_SUBTITLE_TITLE;
                            }
                            elsif ($episode eq $new_episode) {
                                t("  Identical sub-title '" . $episode 
                                  . "' also seen in title, new title is '" 
                                  . $new_title . "'");
                                $title = $new_title;
                                last MIXED_SUBTITLE_TITLE;
                            }
                            # concat subtitle after any episode numbering (x/y)
                            elsif ($episode =~ /^\d+\s*(?:&\d+)?\s*\/\s*\d+\s*(?:\.|\/|,|;|:)?\s*-?\s*(?:series\s*(\w+)\s*(?:;|\.)?)?\s*$/) {
                                t("  Concatenating sub-title '" . $new_episode 
                                  . "' seen in title after existing episode numbering '" 
                                  . $episode . "'");
                                $title = $new_title;
                                $episode = $episode . ": " . $new_episode;
                                last MIXED_SUBTITLE_TITLE;
                            }
                            else {
                                t("  Concatenating sub-title '" . $new_episode 
                                  . "' seen in title with existing episode info '" 
                                  . $episode . "'");
                                $title = $new_title;
                                $episode = $new_episode . ": " . $episode;
                                last MIXED_SUBTITLE_TITLE;
                            }
                        }
                    }
                }
            }

            # Listings for some channels may include programme details which have
            # reversed title and sub-title information ($title = episode and 
            # $episode = title). In order to create more consistent data, we check 
            # for flagged programme titles and reverse the given title and 
            # sub-title
            #
            if ($have_title_data && @reversed_title_subtitle && defined $episode) {

                REVERSED_TITLE_SUBTITLE:
                foreach my $reversed_title_subtitle (@reversed_title_subtitle) {
                    if ($reversed_title_subtitle eq $episode) {
                        t("  Seen reversed title-subtitle for '" 
                          . $title . ":" . $episode . "' - reversing" );
                        $episode = $title;
                        $title = $reversed_title_subtitle;
                        t("  New title is '" . $title . "' and new " 
                          . "sub-title is '" . $episode . "'");
                        last REVERSED_TITLE_SUBTITLE;
                    }
                }
            }

            # After processing title and episode information, using S&R based
            # on the original listings data, we can now process inconsistent 
            # title and episode data content.
            #
            # First, replace inconsistent titles
            #
            if ($have_title_data && %replacement_titles) {
                REPLACEMENT_TITLE:
                foreach my $bad_title (keys %replacement_titles) {
                    if ($title eq $bad_title) {
                        $title = $replacement_titles{$bad_title};
                        t("  Replaced title '" . $bad_title . "' with '"
                          . $title . "' for consistency");
                        last REPLACEMENT_TITLE;
                    }
                }
            }
            #
            # Second, replace inconsistent episodes. The 
            # %replacement_episodes data structure is a hash of hashes.
            #
            if ($have_title_data && %replacement_episodes && defined $episode) {
                REPLACEMENT_EPISODE:
                foreach my $bad_episode_title (keys %replacement_episodes) {
                    # First, check whether we have matched the programme title
                    if ($title eq $bad_episode_title) {
                        # Now look for a specific episode match for the title
                        foreach my $bad_episode (keys %{$replacement_episodes{$bad_episode_title}}) {
                            if ($episode eq $bad_episode) {
                                $episode = $replacement_episodes{$bad_episode_title}->{$bad_episode};
                                t("  Replaced episode info '" . $bad_episode . "' for title '" 
                                  . $bad_episode_title . "' with '" . $episode . "' for consistency");
                                last REPLACEMENT_EPISODE;
                            }
                        }
                    }
                }
            }
            #
            # Allow arbitrary replacement of one title/episode pair with another.
            # To be used where previous title/episode replacement routines
            # do not allow the desired correction (i.e. one-off changes).
            # THIS MUST BE USED WITH CARE!
            #
            if ($have_title_data && %replacement_title_eps) {
                my $tmp_ep;
                my $tmp_ep_num;
                my $tmp_ep_num_text = '';
                # Handle potential undef episode value, as the empty string
                # was used in place of an undef episode during concatenation
                # in the replacement hash
                if (not defined $episode) {
                    $tmp_ep = '';
                }
                # Also handle an episode number that may be present in source
                # data but not in replacement text
                elsif ($episode =~ /^\d+\s*(?:&\d+)?\s*\/\s*\d+\s*(?:\.|\/|,|;|:)?\s*-?\s*(?:series\s*(\w+)\s*(?:;|\.)?)?\s*$/) {
                    $tmp_ep = '';
                    $tmp_ep_num = $episode;
                    $tmp_ep_num_text = " (Preserving existing numbering)";
                }
                else {
                    $tmp_ep = $episode;
                }
                my $key = "" . $title . "|" . $tmp_ep;
                REPLACEMENT_TITLE_EPISODE:
                foreach my $bad_title_episode (keys %replacement_title_eps) {
                    # Check whether we have matched the old programme
                    # title/episode combo
                    if ( $key eq $bad_title_episode) {
                        # Now replace the old title/ep values with new ones
                        my ($old_title, $old_ep) = ($title, $tmp_ep);
                        my ($new_title, $new_ep) = @{$replacement_title_eps{$key}};
                        # update the title
                        $title = $new_title;
                        # if new episode value is empty string, replace with undef;
                        # otherwise use new value
                        if ($new_ep eq '') {
                            if (defined $tmp_ep_num) {
                                $episode = $tmp_ep_num;
                            }
                            else {
                                $episode = undef;
                            }
                        }
                        else {
                            if (defined $tmp_ep_num) {
                                $episode = $tmp_ep_num . ": " . $new_ep;
                            }
                            else {
                                $episode = $new_ep;
                            }
                        }
                        t("  Replaced old title/ep '" . $old_title . ": " . $old_ep
                            . "' with new title/ep '" . $new_title . ": " . $new_ep
                            . "' for consistency" . $tmp_ep_num_text);
                        last REPLACEMENT_TITLE_EPISODE;
                    }
                }
            }
            #
            # Replace inconsistent/missing episode subtitle using a given 
            # description (should be unique per title). The %replacement_ep_from_desc 
            # data structure is a hash of hashes.
            #
            if ($have_title_data && %replacement_ep_from_desc && defined $desc) {
                REPLACEMENT_EPISODE_FROM_DESC:
                foreach my $bad_episode_title (keys %replacement_ep_from_desc) {
                    # First, check whether we have matched the programme title
                    if ($title eq $bad_episode_title) {
                        # Now look for a specific episode match for the title
                        foreach my $bad_ep_desc (keys %{$replacement_ep_from_desc{$bad_episode_title}}) {
                            if ($desc eq $bad_ep_desc) {
                                my $old_ep;
                                (defined $episode) ? ($old_ep = $episode) : ($old_ep = '');
                                $episode = $replacement_ep_from_desc{$bad_episode_title}->{$bad_ep_desc};
                                t("  Updated episode from '" . $old_ep . "' to '" . $episode 
                                    . "' for title '" . $bad_episode_title . "' based on desc '" 
                                    . $bad_ep_desc . "' for consistency");
                                last REPLACEMENT_EPISODE_FROM_DESC;
                            }
                        }
                    }
                }
            }
            #
            # After processing the data for inconsistent title and episode data
            # we now process titles that may not be categorised, or are
            # categorised with various categoriea in the source data.
            # Untitled programmes ("To Be Announced") and films are ignored.
            #
            if ($have_title_data && %replacement_cats && $title !~ /To Be Announced/i && !$film) {
                REPLACEMENT_CAT:
                foreach my $uncat_title (keys %replacement_cats) {
                    if ($title eq $uncat_title) {
                        $genre = $replacement_cats{$uncat_title};
                        t("  Assigned title '" . $title . "' to category '" . $genre . "'");
                        last REPLACEMENT_CAT;
                    }
                }
            }

            # Extract any series/episode/part numbering found in the $episode 
            # information. Series and episode numbering are parsed out of the
            # text and made available in the <episode-num> element. Part
            # numbering is parsed but unused. However, when part numbering is
            # seen in the text it is processed to make its format consistent.
            #
            # First we check for the following groups of series/episode formats:
            #
            # "1/6"
            # "1/6; ...."
            # "1/6 - ...."
            # "1/6. ...."
            # "1/6, series 1; ...."
            # "1/6, series one; ...."
            # "1&2/6"
            #
            # "Episode 1"
            # ".... - Episode 1"
            # ".... ep 2"
            #
            # "series one."
            #
            # "101- ...."
            #
            # "1-3/3 - ..."
            #
            my ($episode_num, $num_episodes, $series_num, $part_num);
            if (defined $episode) {

                t("  Pre-processing title/episode details '" . $title . " / " . $episode . "'");
                my $processed = 0;

                # 1) check for "x/y" format
                if ($episode =~
                    s{
                      ^                    # start at beginning of episode details
                      (\d+)                # CAPTURE the first number(s) found ($episode_num)
                      \s*                  # ignore any whitespace
                      (?:&\d+)?            # check for "&2" details relating to following episode
                      \s*                  # ignore any whitespace
                      \/                   # forward slash
                      \s*                  # ignore any whitespace
                      (\d+)                # CAPTURE the second number(s) found ($num_episodes)
                      \s*                  # ignore any whitespace
                      (?:\.|\/|,|;|:)?     # check for punctuation characters
                      \s*                  # ignore any whitespace
                      -?                   # check for hyphen
                      \s*                  # ignore any whitespace
                      (?:series\s*(\w+|\d+)\s*(?:;|\.)?)? 
                                           # check for series number information ($series_num)
                      \s*                  # ignore any whitespace
                    }
                    {}ix ) {
                        $episode_num = $1 - 1;
                        $num_episodes = $2;
                        t("    Episode number found: episode $1 of $2 (ep, x/y)");
                        if ( defined $3 ) {
                            my $series_digits = word_to_digit($3);
                            if ( defined $series_digits and $series_digits > 0 ) {
                                t("    Series number found: series $series_digits (parsed as $3 from ep, x/y)");
                                $series_num = $series_digits - 1;
                            }
                        }
                        $processed = 1;
                }

                # or 2) check for "Episode x" format
                elsif ($episode =~
                    s{
                      ^                      # start at beginning of episode details
                      (?:Episode|Ep|Epiosde) # ignore "Episode" text
                      \s*                    # ignore any whitespace
                      (\d+)                  # CAPTURE the first number(s) found ($episode_num)
                      $                      # finish at end of episode details
                    }
                    {}ix ) {
                        $episode_num = $1 - 1;
                        t("    Episode number found: episode $1 (ep, episode x)");
                        $processed = 1;
                }

                # or 3) check for "wwww - Episode x" format
                elsif ($episode =~
                    s{
                      \s*                    # ignore any whitespace
                      -?                     # check for hyphen
                      \s*                    # ignore any whitespace
                      (?:Episode|Ep|Epiosde) # ignore "Episode" text
                      \s*                    # ignore any whitespace
                      (\d+)                  # CAPTURE the first number(s) found ($episode_num)
                      $                      # finish at end of episode details
                    }
                    {}ix ) {
                        $episode_num = $1 - 1;
                        t("    Episode number found: episode $1 (ep, wwww - episode x)");
                        $processed = 1;
                }

                # or 4) check for "series wwww" format (number given in words)
                elsif ($episode =~ 
                    s{
                      ^                    # start at beginning of episode details
                      (?:series)           # ignore "series" text
                      \s*                  # ignore any whitespace
                      (\w+)                # CAPTURE series number given as a *word* ($series_num)
                      (?:\.)?              # check for punctuation characters
                      \s*                  # ignore any whitespace
                    }
                    {}ix ) {
                        my $series_digits = word_to_digit($1);
                        if ( defined $series_digits and $series_digits > 0 ) {
                            t("    Series number found: series $series_digits (parsed as $1 from ep, series wwww)");
                            $series_num = $series_digits - 1;
                        }
                        $processed = 1;
                }

                # or 5) check for "nnn- ...." format (parse out and ignore the numbering)
                elsif ($episode =~
                    s{
                      ^                    # start at beginning of episode details
                      (\d{3})              # CAPTURE the 3 digits (unused)
                      (?:-)                # ignore the hyphen
                      \s*                  # ignore any whitespace
                    }
                    {}ix ) {
                        t("    Episode number found (unhandled): episode $1 (ep, nnn-)");
                        $processed = 1;
                }

                # or 6) check for "x-y/z - " format (parse out and ignore the numbering)
                # this numbering is used when two or more episodes are edited together
                # into a single programme (are these technically parts then?)
                elsif ($episode =~
                    s{
                     ^                     # start at beginning of episode details
                     (\d+)                 # CAPTURE the first episode digits (unused)
                     \s*                   # ignore any whitespace
                     (?:-)                 # ignore the hyphen
                     \s*                   # ignore any whitespace
                     (\d+)                 # CAPTURE the last episode digits (unused)
                     \s*                   # ignore any whitespace
                     \/                    # forward slash
                     \s*                   # ignore any whitespace
                     (\d+)                 # CAPTURE the total episode digits (unused)
                     \s*                   # ignore any whitespace
                    }
                    {}ix ) {
                        t("    Episode number found (unhandled): episodes $1-$2 of $3 (ep, x-y/z)");
                        $processed = 1;
                }
                # check for series/episode numbering at end of episode details
                # "Genius Series 1, 4"
                elsif ($episode =~
                    s{
                      ^                      # start at beginning of episode details
                      (.*\b[\x21-\x2F\x3F]?) # CAPTURE the title details before season numbering
                      \s*                    # ignore any whitespace
                      (?:,|;|:|-)?           # check for optional punctuation characters
                      \s*                    # ignore any whitespace
                      (?:\()?                # opening paren
                      (?:Season|Series)      # check for Part/Pt text
                      \s*                    # ignore any whitespace
                      (\d+)                  # CAPTURE season number
                      (?:,)?                 # ignore comma if present
                      \s*                    # ignore any whitespace
                      (\d+)?                 # CAPTURE episode number if present
                      \s*                    # ignore any whitespace
                      (?:\))?                # closing paren
                      $                      # finish at end of episode details
                     }
                     {}ix ) {
                            $title = $1;
                            t("    Season number found: Season $2 (title regex)");
                            $series_num = $2 - 1;
                            if ( defined $3 ) {
                                t("    Episode number found: episode $3 (ep, Series x, y)");
                                $episode_num = $3 - 1;
                            }
                            if ($episode eq $title) {
                                $episode = undef;
                            }
                            $processed = 1;
                }

                # Next we check for part numbering and make its formatting consistent
                #
                # FIXME should we export part number in <episode-num> and remove
                # it from the text?
                #
                if ($episode =~ /(Part|Pt)/i || $episode =~ /\d\s*$/) {
                    # this regex looks for part numbering in parentheses
                    #
                    # "Dead Man's Eleven (Part 1)"
                    # "Dead Man's Eleven - (Part 1)"
                    # "Dead Man's Eleven - (Part 1/2)"
                    # "Dead Man's Eleven (Pt 1)"
                    # "Dead Man's Eleven - (Pt. 1)"
                    # "Dead Man's Eleven - (Pt. 1/2)"
                    if ($episode =~
                        m{
                          ^                      # start at beginning of episode details
                          (.*\b[\x21-\x2F\x3F]?) # CAPTURE the episode details before part numbering
                          \s*                    # ignore any whitespace
                          (?:,|;|:|-)?           # check for optional punctuation characters
                          \s*                    # ignore any whitespace
                          (?:\()                 # opening paren
                          (?:Part|Pt(?:\.)?)     # check for Part/Pt text
                          \s*                    # ignore any whitespace
                          (\d+)                  # CAPTURE part number
                          \s*                    # ignore any whitespace
                          (?:\/\s*\d+)?          # ignore any total part number
                          (?:\))                 # closing paren
                          $                      # finish at end of episode details
                         }ix )
                    {
                                t("    Part number found: part $2 (regex #1)");
                                $episode = $1 . " (Part " . $2 . ")";
                                $part_num = $2 - 1;
                                $processed = 1;
                    }
                    # this regex looks for part numbering with no other episode information
                    #
                    # "Part 1"
                    # "Part 1/3"
                    # "Pt 2"
                    # "Pt 2/3"
                    # "Pt. 3"
                    elsif ($episode =~
                        m{
                          ^                    # start at beginning of episode details
                          (?:Part|Pt(?:\.)?)   # check for Part/Pt text
                          \s*                  # ignore any whitespace
                          (\d+)                # CAPTURE part number
                          \s*                  # ignore any whitespace
                          (?:\/\s*\d+)?        # ignore any total part number
                          $                    # finish at end of episode details
                         }ix )
                    {
                                t("    Part number found: part $1 (regex #2)");
                                $episode = "Part " . $1;
                                $part_num = $1 - 1;
                                $processed = 1;
                    }
                    # this regex looks for bare part numbering after a comma, semicolon, 
                    # colon or hyphen
                    #
                    # "Dead Man's Eleven - Part 1"
                    # "Dead Man's Eleven: Part 1"
                    # "Dead Man's Eleven; Pt 1"
                    # "Dead Man's Eleven, Pt. 1"
                    elsif ($episode =~
                        m{
                          ^                      # start at beginning of episode details
                          (.*\b[\x21-\x2F\x3F]?) # CAPTURE the episode details before part numbering
                          \s*                    # ignore any whitespace
                          (?:,|;|:|-)            # punctuation characters
                          \s*                    # ignore any whitespace
                          (?:Part|Pt(?:\.)?)     # check for Part/Pt text
                          \s*                    # ignore any whitespace
                          (\d+)                  # CAPTURE part number
                          \s*                    # ignore any whitespace
                          (?:\/\s*\d+)?          # ignore any total part number
                          $                      # finish at end of episode details
                         }ix )
                    {
                                t("    Part number found: part $2 (regex #3)");
                                $episode = $1 . " (Part " . $2 . ")";
                                $part_num = $2 - 1;
                                $processed = 1;
                    }
                    # this regex looks for part numbering immediately following episode info
                    #
                    # "Dead Man's Eleven Part 1"
                    # "Dead Man's Eleven Pt 1"
                    # "Dead Man's Eleven Pt 1/2"
                    # "Dead Man's Eleven Pt. 1"
                    elsif ($episode =~
                        m{
                          ^                      # start at beginning of episode details
                          (.*\b[\x21-\x2F\x3F]?) # CAPTURE the episode details before part numbering
                          \s*                    # ignore any whitespace
                          (?:Part|Pt(?:\.)?)     # check for Part/Pt text
                          \s*                    # ignore any whitespace
                          (\d+)                  # CAPTURE part number
                          \s*                    # ignore any whitespace
                          (?:\/\s*\d+)?          # ignore any total part number
                          $                      # finish at end of episode details
                         }ix )
                    {
                                t("    Part number found: part $2 (regex #4)");
                                $episode = $1 . " (Part " . $2 . ")";
                                $part_num = $2 - 1;
                                $processed = 1;
                    }
                    # this regex looks for a digit (conservatively between 1 and 6) following
                    # the episode details, a colon and at least one space
                    #
                    # "Dead Man's Eleven: 1"
                    elsif ($episode =~
                        m{
                          ^                    # start at beginning of episode details
                          (.*)                 # CAPTURE the episode details before part numbering
                          \s*                  # ignore any whitespace
                          (?::)                # colon
                          \s+                  # ignore any whitespace - min 1 space
                          (\d{1})              # CAPTURE single digit part number between 1 and 6
                          $                    # finish at end of episode details
                         }ix )
                    {
                                if ($2 ge 1 && $2 le 6) {
                                    t("    Part number found: part $2 (regex #5, range 1-6)");
                                    $episode = $1 . " (Part " . $2 . ")";
                                    $part_num = $2 - 1;
                                    $processed = 1;
                                }
                    }
                    # this regex looks for worded part numbering with no other episode information
                    #
                    # "Part One"
                    # "Pt Two"
                    # "Pt. Three"
                    elsif ($episode =~
                        m{
                          ^                    # start at beginning of episode details
                          (?:Part|Pt(?:\.)?)   # check for Part/Pt text
                          \s+                  # ignore any whitespace
                          (\w+)                # CAPTURE part number wording
                          $                    # finish at end of episode details
                         }ix )
                    {
                                my $part_digits = word_to_digit($1);
                                if ( defined $part_digits and $part_digits > 0 ) {
                                    t("    Part number found: part $part_digits (regex #6, parsed as $1)");
                                    $episode = "Part " . $part_digits;
                                    $part_num = $part_digits - 1;
                                    $processed = 1;
                                }
                    }
                    # this regex looks for bare part numbering after a comma, semicolon, 
                    # colon or hyphen, where the numbering is given in words
                    #
                    # "Dead Man's Eleven - Part One"
                    # "Dead Man's Eleven: Part One"
                    # "Dead Man's Eleven; Pt One"
                    # "Dead Man's Eleven, Pt. One"
                    elsif ($episode =~
                        m{
                          ^                      # start at beginning of episode details
                          (.*\b[\x21-\x2F\x3F]?) # CAPTURE the episode details before part numbering
                          \s*                    # ignore any whitespace
                          (?:,|;|:|-)            # punctuation characters
                          \s*                    # ignore any whitespace
                          (?:Part|Pt(?:\.)?)     # check for Part/Pt text
                          \s+                    # ignore any whitespace
                          (\w+)                  # CAPTURE part number wording
                          $                      # finish at end of episode details
                         }ix )
                    {
                                my $part_digits = word_to_digit($2);
                                if ( defined $part_digits and $part_digits > 0 ) {
                                    t("    Part number found: part $part_digits (regex #7, parsed as $2)");
                                    $episode = $1 . " (Part " . $part_digits . ")";
                                    $part_num = $part_digits - 1;
                                    $processed = 1;
                                }
                    }
                    # this regex looks for worded part numbering immediately following episode info
                    #
                    # "Dead Man's Eleven Part One"
                    # "Dead Man's Eleven Pt One"
                    # "Dead Man's Eleven Pt. One"
                    elsif ($episode =~
                        m{
                          ^                      # start at beginning of episode details
                          (.*\b[\x21-\x2F\x3F]?) # CAPTURE the episode details before part numbering
                          \s*                    # ignore any whitespace
                          (?:Part|Pt(?:\.)?)     # check for Part/Pt text
                          \s*                    # ignore any whitespace
                          (\w+)                  # CAPTURE part number wording
                          $                      # finish at end of episode details
                         }ix )
                    {
                                my $part_digits = word_to_digit($2);
                                if ( defined $part_digits and $part_digits > 0 ) {
                                    t("    Part number found: part $part_digits (regex #8, parsed as $2)");
                                    $episode = $1 . " (Part " . $part_digits . ")";
                                    $part_num = $part_digits - 1;
                                    $processed = 1;
                                }
                    }
                    # this regex looks for worded part numbering in parentheses
                    #
                    # "Dead Man's Eleven (Part One)"
                    # "Dead Man's Eleven - (Part One)"
                    # "Dead Man's Eleven (Pt One)"
                    # "Dead Man's Eleven - (Pt. One)"
                    elsif ($episode =~
                        m{
                          ^                      # start at beginning of episode details
                          (.*\b[\x21-\x2F\x3F]?) # CAPTURE the episode details before part numbering
                          \s*                    # ignore any whitespace
                          (?:,|;|:|-)?           # check for optional punctuation characters
                          \s*                    # ignore any whitespace
                          (?:\()                 # opening paren
                          (?:Part|Pt(?:\.)?)     # check for Part/Pt text
                          \s*                    # ignore any whitespace
                          (\w+)                  # CAPTURE part number wording
                          \s*                    # ignore any whitespace
                          (?:\))                 # closing paren
                          $                      # finish at end of episode details
                         }ix )
                    {
                                my $part_digits = word_to_digit($2);
                                if ( defined $part_digits and $part_digits > 0 ) {
                                    t("    Part number found: part $part_digits (regex #9, parsed as $2)");
                                    $episode = $1 . " (Part " . $part_digits . ")";
                                    $part_num = $part_digits - 1;
                                    $processed = 1;
                                }
                    }
                    # check for potential part numbering left unprocessed
                    #
                    # we do this at the end of the if-else because the (Part x) text is
                    # not (yet) removed from the episode details, only made consistent
                    elsif ($opt->{debug} && $episode =~ /\b(Part|Pt(\.)?)(\d+|\s+\w+)/i) {
                        t("    Possible part numbering still seen: $episode");
                        $possible_part_nums{$episode} = $episode;
                    }
                }

                # Check for potential season numbering in title
                #
                if ($title =~ /(Season|Series)/i) {
                    # this regex looks for season numbering in title with
                    # in parentheses
                    #
                    # "Wheeler Dealers (Season 1)"
                    # "Wheeler Dealers - (Series 1)"
                    if ($title =~
                        m{
                          ^                      # start at beginning of title details
                          (.*\b[\x21-\x2F\x3F]?) # CAPTURE the title details before season numbering
                          \s*                    # ignore any whitespace
                          (?:,|;|:|-)?           # check for optional punctuation characters
                          \s*                    # ignore any whitespace
                          (?:\()                 # opening paren
                          (?:Season|Series)      # check for Part/Pt text
                          \s*                    # ignore any whitespace
                          (\d+)                  # CAPTURE season number
                          (?:,)?                 # ignore comma if present
                          \s*                    # ignore any whitespace
                          (\d+)?                 # CAPTURE episode number if present
                          \s*                    # ignore any whitespace
                          (?:\))                 # closing paren
                          $                      # finish at end of title details
                         }ix )
                    {
                                $title = $1;
                                if (defined $series_num && $series_num != $2) {
                                    t("    Season number (" . $series_num . ") already defined. "
                                        . "Ignoring different season number (" . $2 . ") in title.");
                                }
                                else {
                                    t("    Season number found: Season $2 (title regex)");
                                    $series_num = $2 - 1;
                                }
                                $episode_num = $3 - 1 if $3;
                                $processed = 1;
                    } 
                    # this regex looks for season numbering in title without
                    # parentheses
                    #
                    # "Wheeler Dealers Series 1"
                    # "Wheeler Dealers Series 1, 3"
                    elsif ($title =~
                        m{
                          ^                      # start at beginning of title details
                          (.*\b[\x21-\x2F\x3F]?) # CAPTURE the title details before season numbering
                          \s*                    # ignore any whitespace
                          (?:,|;|:|-)?           # check for optional punctuation characters
                          \s*                    # ignore any whitespace
                          (?:Season|Series)      # check for Part/Pt text
                          \s*                    # ignore any whitespace
                          (\d+)                  # CAPTURE season number
                          (?:,)?                 # ignore comma if present
                          \s*                    # ignore any whitespace
                          (\d+)?                 # CAPTURE episode number if present
                          \s*                    # ignore any whitespace
                          $                      # finish at end of title details
                         }ix )
                    {
                                $title = $1;
                                if (defined $series_num && $series_num != $2) {
                                    t("    Season number (" . $series_num . ") already defined. "
                                        . "Ignoring different season number (" . $2 . ") in title.");
                                }
                                else {
                                    t("    Season number found: Season $2 (title regex)");
                                    $series_num = $2 - 1;
                                }
                                $episode_num = $3 - 1 if $3;
                                $processed = 1;
                    }
                }
                
                # after processing see if $title contains "season" text that should
                # probably be removed
                if ($opt->{debug} && $title =~ /season/i) {
                    t("    Title text contains \"Season\":  $title");
                    $title_text_to_remove{$title} = $title;
                }
                
                # after processing see if $episode contains "series" text
                if ($opt->{debug} && $episode =~ /series/i) {
                    t("    Possible series numbering still seen:  $episode");
                    $possible_series_nums{$episode} = $episode;
                }
                
                # check for potential episode numbering left unprocessed
                if ( $opt->{debug} && ($episode =~ /^\d{1,2}\D/ || $episode =~ /\D\d{1,2}$/) 
                        && $episode !~ /(Part|Pt(\.)?)(\d+|\s+\w+)/) {
                    t("    Possible episode numbering still seen: $episode");
                    $possible_episode_nums{$episode} = $episode;
                }

                # Set $episode to undefined if empty/whitespace
                if ($episode =~ /^\s*$/) {
                    $episode = undef;
                }

                # output updated title/episode information after processing
                if ($processed) {
                    my $ep_out = "<UNDEF>";
                    if (defined $episode) {
                        $episode =~ s/\s+/ /g;  # tidy whitespace
                        $ep_out = $episode;
                    }
                    t("  Post-processing title/episode details '" . $title . " / " . $ep_out . "'");
                }
            }

            # Create the hash to store the programme's details
            my %p = (channel => $ch, title => [ [ $title ] ]);
            
            # For non-films which may contain subtitle/episode information in
            # the title, add the title to the list of programme titles for
            # later debugging
            if (!$film) {
                $prog_titles{$title} = $title;
            }

            # Occasionally film listings contain the title duplicated in the 
            # $episode field, so we remove it here
            if ($film && uc $title eq uc $episode) {
                $episode = undef;
            }

            # Write out the programme's episode title ($episode) if present
            if (defined $episode && $episode !~ /^\s*$/) {
                $p{'sub-title'} = [ [ $episode ] ];
            }

            # Check episode description if present
            if (defined $desc) {
                $desc =~ s/\s+/ /g;
                # Remove any last-minute scheduling info inserted into description
                if ($desc =~ s/\s?UPDATED LISTING(?:\s?:\s?|\s?-\s?|\s?)(.*)$//) {
                    $updated_listing_info = $1;
                    t("  Removed updated listing information from description:\n"
                      . "    '$updated_listing_info'");
                }

                # Extract series information if seen at start or end of description
                # "(Series one) ..."
                # "... (Series one)"
                if ($desc =~
                    s{
                        \s*             # leading whitespace
                        \(Series        # (Series 
                        \s*             # whitespace
                        (\d+)           # CAPTURE the series number ($series_num)
                        \)              # )
                        \s*             # trailing whitespace
                    }
                    {}ix ) {
                        $series_num = $1 - 1;
                        t("  Series number found: series $1 (desc)");
                }

                # Extract episode numbering if seen at end of description
                # "... (3/8)"
                if ($desc =~
                    s{
                        \s*             # leading whitespace
                        \(              # (
                        (\d+)           # CAPTURE the first number(s) found ($episode_num)
                        \/              # forward slash
                        (\d+)           # CAPTURE the second number(s) found ($num_episodes)
                        \)              # )
                        $               # end of desc
                    }
                    {}ix ) {
                        $episode_num = $1 - 1;
                        $num_episodes = $2;
                        t("  Episode number found: episode $1 of $2 (desc)");
                }
                
                # Extract episode and series info from start of description
                # "1/6, series 1; ...."
                if ($desc =~
                    s{
                      ^                    # start at beginning of episode details
                      (\d+)                # CAPTURE the first number(s) found ($episode_num)
                      \s*                  # ignore any whitespace
                      (?:&\d+)?            # check for "&2" details relating to following episode
                      \s*                  # ignore any whitespace
                      \/                   # forward slash
                      \s*                  # ignore any whitespace
                      (\d+)                # CAPTURE the second number(s) found ($num_episodes)
                      \s*                  # ignore any whitespace
                      (?:\.|\/|,|;|:)?     # check for punctuation characters
                      \s*                  # ignore any whitespace
                      -?                   # check for hyphen
                      \s*                  # ignore any whitespace
                      (?:series\s*(\w+)\s*(?:;|\.)?)? 
                                           # check for series number information ($series_num)
                      \s*                  # ignore any whitespace
                    }
                    {}ix ) {
                        $episode_num = $1 - 1;
                        $num_episodes = $2;
                        t("    Episode number found: episode $1 of $2 (desc)");
                        if ( defined $3 ) {
                            my $series_digits = word_to_digit($3);
                            if ( defined $series_digits and $series_digits > 0 ) {
                                t("    Series number found: series $series_digits (parsed as $3 from desc)");
                                $series_num = $series_digits - 1;
                            }
                        }
                }

                # Finally write out the non-empty description
                if ($desc !~ /^\s*$/) {
                    $p{desc} = [ [ $desc, 'en' ] ];
                }
            }

            # Write out episode numbering information extracted from $episode 
            # and $desc fields
            if (!defined $series_num) {
                $series_num = '';
            }
            if (!defined $episode_num) {
                $episode_num = '';
            }
            if (defined $num_episodes) {
                $num_episodes = "/$num_episodes";
            }
            else {
                $num_episodes = '';
            }
            # Write out the details if we have the series and/or episode numbers
            if ($series_num ne '' || $episode_num ne '') {
                $p{'episode-num'} = [ [ "${series_num}.${episode_num}${num_episodes}.", "xmltv_ns" ] ];
            }

            if (defined $director && $director !~ /^\s*$/) {
                $p{credits}{director} = [ $director ];
            }

            # The Radio Times data includes cast information in 2 formats:
            #
            # a) pairings of 'character*actor' with subsequent pairings 
            #    separated by '|' - '*' does not appear in any text
            # b) a comma separated list of actors with no character details
            #
            # If 'Director' appears in the character entry, this is to be used 
            # as a regular cast member, not the programme's director
            if (defined $cast) {
                my @cast;
                $cast =~ s/\s+/ /g; # remove extra spaces
                $cast =~ s/\|\|/\|/g;  # remove empty pipe-separated fields
                $cast =~ s/,,/,/g;  # remove empty comma-separated fields
                
                # First we check for 'character*actor' entries
                if ($cast =~ tr/*//) {
                    # Multiple 'character*actor'entries
                    if ($cast =~ tr/|//) {
                        @cast = split /\|/, $cast;
                    }
                    # Single 'character*actor' entry
                    else {
                        push @cast, $cast;
                    }
                    
                    # Process the list of cast entries
                    foreach my $cast (@cast) {
                        # Replace any actor given as Himself/Herself with the
                        # character name given
                        if ($cast =~ /^(.*)[*](Himself|Herself)$/) {
                            $cast = "$1*$1";
                        }
                        # Remove the 'character*' portion of the entry
                        if ($cast !~ s/^.*[*]//) {
                            t("  Bad cast entry for '$title': $cast");
                        }
                    }
                }
                # Next we check for CSV-style actor entries
                elsif ($cast =~ tr/,//) {
                    @cast = split /,/, $cast;
                }
                # Finally we assume a single actor's name that contains neither 
                # '*' nor ','
                else {
                    push @cast, $cast;
                }
                # Trim whitespace from beginning/end of actor names
                foreach my $cast (@cast) {
                    $cast =~ s/^\s+//;
                    $cast =~ s/\s+$//;
                }
                $p{credits}{actor} = \@cast;
            }

            if (defined $year) {
                $p{date} = $year;
            }
            if ($film) {
                push @{$p{category}}, [ 'Film', 'en' ];
            }
            if (defined $genre && !$film) {
                push @{$p{category}}, [ $genre, 'en' ];
                # check for unhelpful/duplicated categories
                if ($opt->{debug}) {
                    $categories{$genre} = $genre;
                    if ($genre =~ /(No Genre|TBA)/ && $title !~ /To Be Announced/) {
                        $uncategorised_progs{$title} = $title;
                    }
                    $cats_per_prog{$title}{$genre}++;
                }
            }
            if ($opt->{debug} && !defined $genre && $title !~ /To Be Announced/) {
                $uncategorised_progs{$title} = $title;
            }
            if ($widescreen) {
                $p{video}{aspect} = '16:9';
            }
            if ($black_and_white) {
                $p{video}{colour} = 0;
            }
            # override aspect if channel is flagged as HDTV
            if (defined $video_quality{$xmltv_id}) {
                if ($video_quality{$xmltv_id} =~ /HDTV/) {
                    $p{video}{quality} = 'HDTV';
                    $p{video}{aspect} = '16:9';
                }
                elsif ($video_quality{$xmltv_id} =~ /SDTV/) {
                    $p{video}{quality} = 'SDTV';
                }
            }
            if ($repeat) {
                $p{'previously-shown'} = {};
            }
            if ($premiere) {
                $p{premiere} = [ '' ];
            }
            if ($new_series) {
                $p{new} = 1;
            }
            if ($subtitles) {
                push @{$p{subtitles}},{type=>'teletext'};
            }
            if ($deaf_signed) {
                push @{$p{subtitles}},{type=>'deaf-signed'};
            }
            if (defined $certificate && $certificate !~ /^\s*$/) {
                $p{rating} = [ [ $certificate, 'BBFC' ] ];
            }
            if (defined $star_rating  && $star_rating !~ /^\s*$/ && $film) {
                push @{$p{'star-rating'}}, [ "$star_rating/5", 'Radio Times Film Rating' ];
            }
            if ($choice) {
                push @{$p{'star-rating'}}, [ '1/1', 'Radio Times Recommendation' ];
            }

            # Broadcast date, start/stop times, and timezone adjustments.
            #
            # For each programme entry, the Radio Times data includes the 
            # date at start of broadcast, the start time and the stop time.
            #
            # The Radio Times sometimes explicitly flags a programme's start/stop
            # times as being in a specific timezone (GMT or BST). We parse this
            # information out when processing the programme's title and apply it
            # to the start time of any such programmes ($explicit_tz). Flagged
            # programmes are usually seen in the data in March and October, when
            # British Summer Times begins and ends.
            #
            # For the majority of programmes where the timezone is not flagged 
            # explicitly, we determine the UTC offset of the programme's start time
            # via XMLTV::DST::utc_offset().
            #
            # We then calculate the programme's stop time using the 
            # UTC-offset-corrected start time and its stated length. This allows 
            # us to handle occasions when programmes having mixed GMT/BST 
            # timings are not flagged.
            #
            # On the day of the  GMT->BST transition, any unflagged programme 
            # starting before 0100 +0000 generally has both start/stop times 
            # given in GMT (+0000) in the RT data (although this is not always 
            # the case).
            #
            # The Summer Time Order of 2002 defines British Summer Time as 
            # "...the period beginning at one o'clock, Greenwich mean time, in 
            # the morning of the last Sunday in March and ending at one o'clock, 
            # Greenwich mean time, in the morning of the last Sunday in October."
            #
            # utc_offset() will only provide the correct stop time if we ensure 
            # dates have the correct UTC offset applied. DateCalc will always use
            # TZ=+0000 when processing/displaying dates ( Date_Init('TZ=+0000') ) 
            # so we must also allow for this when adjusting dates and using this 
            # output with utc_offset (we employ UnixDate() to help).
            #
            my ($yyyy, $mm, $dd);
            my ($implicit_tz, $tz);
            
            # Check for valid date format
            if ($date !~ m{(\d\d)/(\d\d)/(\d{4})$}) {
                t("  A bad date '$date' was seen for '$title', skipping");
                next PROGRAMME;
            }
            ($dd, $mm, $yyyy) = ($1, $2, $3);
            t("  Start time given as '$yyyy/$mm/$dd $start', duration $duration_mins mins");

            # Use BST information found in title, otherwise calculate it ourselves
            if (defined $explicit_tz) {
                t("  Explicit timezone '$explicit_tz' detected in title");
                $tz = $explicit_tz;
            }
            else {
                $p{start} = utc_offset( "$yyyy$mm$dd$start", $base_tz );
                if ($p{start} !~ /([+-]\d{4})$/) {
                    t("  Bad UTC offset '$1' detected for '$title', skipping");
                    next PROGRAMME;
                }
                $implicit_tz = $1;
                t("  Implicit timezone calculated to be '$implicit_tz'");
                $tz = $implicit_tz;
            }

            # Calculate start time with correct UTC offset
            $p{start} = utc_offset("$yyyy$mm$dd$start $tz", $base_tz);

            # Calculate stop time by adding length of programme to start time
            my $datecalc_stop
                = DateCalc(ParseDate("$yyyy$mm$dd$start $tz"),
                           ParseDateDelta($duration_mins . "minutes")
                           );
            #t("  Stop time calculated as '$datecalc_stop' via DateCalc()");
            my $unixdate_stop
                = UnixDate($datecalc_stop, "%Y%m%d%H%M %z");
            #t("  Stop time formatted as  '$unixdate_stop' via UnixDate()");

            $p{stop}  = utc_offset($unixdate_stop, $base_tz);

            t("  $p{start} - Start time");
            t("  $p{stop} - Stop time");

            # Now we have determined the correct start/stop times for the programme
            # add any required timeshift defined in channel_ids and preserve the
            # correct timezone information
            #
            if (defined $channel_offset{$xmltv_id}) {
                my $timeshift = $channel_offset{$xmltv_id};
                my $start_ts = DateCalc( ParseDateString( $p{start} ), $timeshift );
                my $stop_ts = DateCalc( ParseDateString( $p{stop} ), $timeshift );
                $p{start} = utc_offset( UnixDate( $start_ts, "%Y%m%d%H%M %z" ), $base_tz );
                $p{stop} = utc_offset( UnixDate( $stop_ts, "%Y%m%d%H%M %z" ), $base_tz );
                t("  $p{start} - Start time after applying '$timeshift' timeshift");
                t("  $p{stop} - Stop time after applying '$timeshift' timeshift");
            }

            # Now check to see whether the channel broadcasting the programme is a
            # part-time channel, and if so, see whether this programme's timeslot 
            # times fall within the broadcast window. If a channel broadcasts
            # through the night, we also need to test against the next day's
            # broadcast times.
            #
            if (defined $broadcast_hours{$xmltv_id}) {
                $broadcast_hours{$xmltv_id} =~ /(\d{4})-(\d{4})/;
                my ($chan_start, $chan_stop) = ($1, $2);
                $chan_start = utc_offset( "$yyyy$mm$dd$chan_start", $base_tz );
                $chan_stop  = utc_offset( "$yyyy$mm$dd$chan_stop", $base_tz );
                # Correct the stop time if it is earlier than the start time
                my $chan_stop_next_day = 0;
                if (Date_Cmp( $chan_start, $chan_stop ) > 0) {
                    $chan_stop_next_day = 1;
                    $chan_stop =  utc_offset( 
                                      UnixDate( 
                                          DateCalc( 
                                              ParseDateString($chan_stop), 
                                              ParseDateDelta("+ 1 day")
                                          ), 
                                          "%Y%m%d%H%M %z"),
                                      $base_tz
                                  );
                }

                # Include the current programme if its timeslot lies inside the
                # channel's broadcast window
                if (Date_Cmp($p{start}, $chan_start) >= 0
                  && Date_Cmp($p{stop}, $chan_stop) <= 0) {
                    t("  $chan_start - Start time of channel");
                    t("  $chan_stop - Stop time of channel");
                    t("  '$title' shown whilst channel is on-air, adding");
                }
                # If the channel starts and stops broadcasting on the same 
                # calendar day and the programme's timeslot is outside the 
                # broadcast window, skip it
                elsif (( Date_Cmp( $p{start}, $chan_start) < 0 
                                  || Date_Cmp($p{stop}, $chan_stop) > 0 ) 
                            && $chan_stop_next_day == 0 ) {
                    t("  $chan_start - Start time of channel");
                    t("  $chan_stop - Stop time of channel");
                    t("  '$title' shown whilst channel is off-air, skipping\n");
                    next PROGRAMME;
                }
                else {
                    # If the channel broadcasts through the night, and the channel
                    # start time is later than the stop time, it is possible for a
                    # program shown at or after midnight to result in the generation
                    # of incorrect channel start/stop times (shifted +1day forward).
                    # We therefore generate another pair of channel start/stop 
                    # times for the previous day to match against
                    #
                    $chan_start = utc_offset(
                                      UnixDate(
                                          DateCalc(
                                              ParseDateString($chan_start),
                                              ParseDateDelta("- 1 day")
                                          ),
                                          "%Y%m%d%H%M %z"),
                                      $base_tz
                                  );

                    $chan_stop  = utc_offset(
                                      UnixDate(
                                          DateCalc(
                                              ParseDateString($chan_stop),
                                              ParseDateDelta("- 1 day")
                                          ),
                                          "%Y%m%d%H%M %z"),
                                      $base_tz
                                  );

                    t("  $chan_start - Start time of channel");
                    t("  $chan_stop - Stop time of channel");

                    # Test again to see if the programme falls between the adjusted
                    # channel broadcast times
                    if (Date_Cmp($p{start}, $chan_start) >= 0
                      && Date_Cmp($p{stop}, $chan_stop) <= 0 ) {
                        t("  '$title' shown whilst channel is on-air, adding");
                    } else {
                        t("  '$title' shown whilst channel is off-air, skipping\n");
                        next PROGRAMME;
                    }
                }
            }

            # Compare the stated and calculated durations of the programme. Since
            # we use the given programme length to determine the stop time, any
            # problems here need investigating
            my $rt_prog_length = ParseDateDelta( $duration_mins . " minutes" );
            my $real_prog_length = DateCalc( ParseDate( $p{start} ),
                                             ParseDate( $p{stop} ) );
            if ($rt_prog_length ne $real_prog_length) {
                t("  Calculated/stated programme durations do not agree for '$title':");
                t("    Start time: '$p{start}'\t\tCalculated:  '$real_prog_length'");
                t("    Stop time:  '$p{stop}'\t\tRadio Times: '$rt_prog_length'");
            }

            # Check for a programme having zero length and skip it
            my $zero_prog_length = ParseDateDelta( "0 minutes" );
            if (($rt_prog_length eq $zero_prog_length) ||
                            ($real_prog_length eq $zero_prog_length)) {
                t("  Programme '$title' has zero length, skipping");
                t("");
                next PROGRAMME;
            }

            # Check that this programme's start time is not earlier than the
            # previous programme's stop time. This is seen very infrequently
            # in the data, and might be due to last minute schedule changes
            # where the overlapping timings have not been noticed/corrected.
            #
            # If an overlap is detected, move the start time of the second 
            # programme to be the same as the stop time for the first programme. 
            # This has been chosen to try to ensure that the first programme 
            # does not lose its ending, which is almost always more important 
            # than the start of a programme.
            #
            # FIXME
            # However, examination of a recent occurence of an overlap showed 
            # that the start time of the second programme was correct, and that
            # the end time and duration of the previous programme was incorrect
            # so this behaviour may need to be reversed/altered if start times
            # are found to be consistently correct
            #
            if ( defined $previous_programme && Date_Cmp($p{start}, $$previous_programme{stop}) < 0 ) {
                # ignore programmes that are listed as starting before the 
                # previous programe has finished *and* finishing before or 
                # at the same time the previous programme has finished 
                # (seen with news/weather bulletins)
                if ( Date_Cmp($p{stop}, $$previous_programme{stop}) <= 0 ) {
                    t("  Programme '$title' starts/stops during previous programme, ignoring");
                    t("");
                    push @overlapping_progs,
                            $p{start} . " (" . $rt_name . "): " . $p{title}->[0]->[0]
                            . " starts/stops during previous programme, IGNORING";
                    next PROGRAMME;
                }
                # determine overlap
                my $delta = DateCalc( ParseDate( $p{start} ),
                                      ParseDate( $$previous_programme{stop} ) );
                # get the overlap in minutes
                my $overlap = Delta_Format($delta, 1, "%mv");
                t("  Scheduling overlap detected (" . $overlap . " mins) with " 
                        . "previous programme '" . $$previous_programme{title}->[0]->[0] . "'");
                # Record the overlap for debug output
                push @overlapping_progs, 
                        $p{start} . " (" . $rt_name . "): Start of '" . $p{title}->[0]->[0] 
                        . "' overlaps end of '" . $$previous_programme{title}->[0]->[0] 
                        . "' by " . $overlap . " minutes, ADJUSTING";
                $p{start} = $$previous_programme{stop};
                t("  Adjusting start time of '" . $p{title}->[0]->[0] . "' to remove overlap");
                t("  $p{start} - Start time after overlap adjustment");
                t("  $p{stop} - Stop time after overlap adjustment");
            }

            # Finally, write the programme's XML data to programme list
            push @programmes, \%p;
            # Update reference to previous programme
            $previous_programme = \%p;
            
            $num_titles++;
            t("");
        }

        if ($num_titles < 1) {
            $empty_listings{$rt_name} = $rt_listings_uri;
            t("  No programmes found for '$rt_name' - check source file");
        }
        else {
            # Write the channel's programme elements to output
            foreach my $prog (@programmes) {
                $writer->write_programme($prog);
            }
            t("  Finished processing $num_titles <programme> elements for '$rt_name'");
        }
        
        t("Finished processing listings for '$rt_name' ($xmltv_id)\n");
        
        # Update the progres bar by one increment
        if (defined $listings_bar) {
            $listings_bar->update();
        }
    }

    if (defined $listings_bar) {
        # Only update the progress bar to 100% if we need to
        if ($need_final_update) {
            $listings_bar->update($num_req_chans);
        }
        $listings_bar->finish();
        if (!$opt->{quiet}) {
            say("\n");
        }
    }
}

sub write_xmltv_footer {
    t("\nWriting XMLTV footer\n");
    $writer->end;
}

sub print_titles_with_colons {
    if (%prog_titles) {
        say("\nStart of list of titles containing colons");
        foreach my $title (sort keys %prog_titles) {
            if ($title =~ /:/) {
                say("  $title");
            }
        }
        say("End of list of titles containing colons");
    }
}

sub print_titles_with_hyphens {
    if (%prog_titles) {
        say("\nStart of list of titles containing hyphens");
        foreach my $title (sort keys %prog_titles) {
            if ($title =~ /\s+-\s+/) {
                say("  $title");
            }
        }
        say("End of list of titles containing hyphens");
    }
}

sub print_misencoded_utf8_data {
    if (%hasC27FBFchars && scalar keys %hasC27FBFchars >= 1) {
        say("\nStart of list of channels containing bad bytes in range [C2][7F-BF]");
        foreach my $chan (sort keys %hasC27FBFchars) {
            say("  $chan ($hasC27FBFchars{$chan})");
        }
        say("End of list of channels");
    }
    if (%hadEFBFBD && scalar keys %hadEFBFBD >= 1) {
        say("\nStart of list of channels containing Unicode Replacement Character");
        foreach my $chan (sort keys %hadEFBFBD) {
            say("  $chan ($hadEFBFBD{$chan})");
        }
        say("End of list of channels");
    }
    if (%hadC3AFC2BFC2BD && scalar keys %hadC3AFC2BFC2BD >= 1) {
        say("\nStart of list of channels containing double-encoded Unicode Replacement Character");
        foreach my $chan (sort keys %hadC3AFC2BFC2BD) {
            say("  $chan ($hadC3AFC2BFC2BD{$chan})");
        }
        say("End of list of channels");
    }
}

sub print_uncategorised_progs {
    if (%uncategorised_progs && scalar keys %uncategorised_progs > 0) {
        say("\nStart of list of uncategorised programmes");
        foreach my $title (sort keys %uncategorised_progs) {
            say("  $title");
        }
        say("End of list of uncategorised programmes");
    }
}

sub print_categories {
    if (%categories && scalar keys %categories > 0) {
        say("\nStart of list of programme categories seen");
        foreach my $category (sort keys %categories) {
            say("  $category");
        }
        say("End of list of programme categories seen");
    }
}

sub print_possible_prog_numbering {
    if (%possible_series_nums && scalar keys %possible_series_nums > 0) {
        say("\nStart of list of possible series numbering seen in listings");
        foreach my $poss (sort keys %possible_series_nums) {
            say("  $poss");
        }
        say("End of list of possible series numbering seen in listings");
    }
    if (%possible_episode_nums && scalar keys %possible_episode_nums > 0) {
        say("\nStart of list of possible episode numbering seen in listings");
        foreach my $poss (sort keys %possible_episode_nums) {
            say("  $poss");
        }
        say("End of list of possible episode numbering seen in listings");
    }
    if (%possible_part_nums && scalar keys %possible_part_nums > 0) {
        say("\nStart of list of possible part numbering seen in listings");
        foreach my $poss (sort keys %possible_part_nums) {
            say("  $poss");
        }
        say("End of list of possible part numbering seen in listings");
    }
    if (%title_text_to_remove && scalar keys %title_text_to_remove > 0) {
        say("\nStart of list of titles containing \"Season\"");
        foreach my $t (sort keys %title_text_to_remove) {
            say("  $t");
        }
        say("End of list of titles containing \"Season\"");
    }
}

sub print_overlapping_progs {
    if (@overlapping_progs) {
        say("\nStart of list of overlapping programmes seen in listings");
        foreach my $overlap (@overlapping_progs) {
            say("  " . $overlap);
        }
        say("End of list of overlapping programmes seen in listings");
    }
}

sub print_empty_listings {
    if (%empty_listings && scalar keys %empty_listings >= 1) {
        say("\nStart of list of channels providing no listings");
        foreach my $chan (sort keys %empty_listings) {
            say("  $chan ($empty_listings{$chan})");
        }
        say("End of list of channels providing no listings");
    }
}

sub print_flagged_title_eps {
    if (%flagged_title_eps && scalar keys %flagged_title_eps >= 1) {
        my %titles_to_output; # temp hash to store matches
        foreach my $flagged_title (sort keys %flagged_title_eps) {
            foreach my $title (sort keys %prog_titles) {
                if (lc $flagged_title eq lc $title) {
                    $titles_to_output{$flagged_title} = $flagged_title;
                }
            }
        }
        # only output something if at least 1 matching title
        if (%titles_to_output && scalar keys %titles_to_output >= 1) {
            say("\nStart of list of titles that may need fixing individually");
            foreach my $title (sort keys %titles_to_output) {
                say("  $title");
            }
            say("End of list of titles that may need fixing individually");
        }
    }
}

sub print_uc_titles_pre {
    if (%uc_prog_titles) {
        say("\nStart of list of uppercase titles before processing");
        foreach my $title (sort keys %uc_prog_titles) {
            say("  $title");
        }
        say("End of list of uppercase titles before processing");
    }
}

sub print_uc_titles_post {
    if (%prog_titles) {
        say("\nStart of list of uppercase titles after processing");
        foreach my $title (sort keys %prog_titles) {
            if ($title eq uc($title) && $title !~ /^\d+$/) {
                say("  $title");
            }
        }
        say("End of list of uppercase titles after processing");
    }
}

sub print_title_variants {
    if (%prog_titles) {
        say("\nStart of possible title variations");
        # hash to store titles after processing to remove punctuation/case
        my %unique_titles;
        foreach my $title (keys %prog_titles) {
            # lowercase and strip all likely punctuation chars
            my $title_nopunc = lc $title;
            $title_nopunc =~ tr/ ,.?;:'!&()-//d;
            # count number of each title variant
            $unique_titles{$title_nopunc}{$title}++;
        }

        foreach my $unique_title (sort keys %unique_titles) {
            if (scalar keys %{$unique_titles{$unique_title}} > 1) {
                foreach my $title (sort keys %{$unique_titles{$unique_title}}) {
                    say("  $title (" . $unique_titles{$unique_title}{$title} . " occurences)");
                }
                say("");
            }
        }
        say("\nEnd of possible title variations");
    }
}

sub print_cats_per_prog {
    if (%cats_per_prog) {
        say("\nStart of programmes with multiple categories");
        foreach my $title (sort keys %cats_per_prog) {
            if (scalar keys %{$cats_per_prog{$title}} > 1) {
                say("  '" . $title . "' is categorised as:");
                foreach my $cat (sort keys %{$cats_per_prog{$title}}) {
                    say("    $cat (" . $cats_per_prog{$title}{$cat} . " occurences)");
                }
                say("");
            }
        }
        say("\nEnd of programmes with multiple categories");
    }
}

#FIXME
#
# The following subs are subject to change and are currently proof of concept

sub list_lineups {
    say("test.freeview.co.uk|DVB-T|FreeView");
    say("test.freesat.co.uk|DVB-S|Freesat");
}

sub get_lineup {

    my $opt = shift;

    # get the lineup_id we're passed
    # my $lineup_id = shift;
    # say("Lineup provided was: " . $lineup_id);
 
    # Configure output and write XMLTV lineup data - header, name, channels 
    # and footer
    #
    #my $lineup_writer;
    setup_xmltv_lineup_writer($opt);
    write_xmltv_lineup_header();

    write_lineup_name();
    write_lineup_icon();
    write_lineup_channels();

    write_xmltv_lineup_footer();
}

# Determine options for, and create XMLTV::Lineup::Writer object
sub setup_xmltv_lineup_writer {

    my $opt = shift;

    # output options
    my %g_args = ();
    if (defined $opt->{output}) {
#        t("\nOpening XML output file '$opt->{output}'\n");
        my $fh = new IO::File ">$opt->{output}";
        die "Error: Cannot write to '$opt->{output}', exiting" if (!$fh);
        %g_args = (OUTPUT => $fh);
    }

#    t("Started writing XMLTV output using " . $xml_encoding . " encoding");
    $lineup_writer = new XMLTV::Lineup::Writer(%g_args, encoding => $xml_encoding);
}

sub write_xmltv_lineup_header {

#    t("Writing XMLTV lineup header");
    $lineup_writer->start(\%xmltv_lineup_attributes);
}

sub write_lineup_name {

    my @values = ( 'Freeview (UK)', 'en' );
    $lineup_writer->write_display_name(\@values);
}

sub write_lineup_icon {

    my %icon_url = ( 'src' => 'http://www.freeview.co.uk/logo.jpg' );
    $lineup_writer->write_lineup_icon(\%icon_url);
}

sub write_lineup_channels {
    
    my %ch = ( 'id' => 'bbc1.bbc.co.uk',
               'old-id' => [ 'old.bbc1.bbc.co.uk', 'evenolder.bbc1.bbc.co.uk' ],
               'display-name' => [ [ 'BBC ONE', 'en' ] ],
               'preferred-preset' => '1',
               'service-id' => '4165',
               'transport-id' => '4101',
               'network-id' => '9018',
               'icon' => [ { 'src' => 'http://www.bbc.co.uk/bbcone.jpg' } ],
               'homepage-url' => "http://www.bbc.co.uk/bbcone", );

    $lineup_writer->write_channel(\%ch);
}

sub write_xmltv_lineup_footer {
#    t("Writing XMLTV lineup footer\n");
    $lineup_writer->end;
}

__END__

=head1 NAME

tv_grab_uk_rt - Grab TV listings for United Kingdom/Republic of Ireland

=head1 SYNOPSIS

tv_grab_uk_rt --help
	
tv_grab_uk_rt --version

tv_grab_uk_rt --capabilities

tv_grab_uk_rt --description

tv_grab_uk_rt [--config-file FILE]
              [--days N] [--offset N]
              [--output FILE] [--quiet] [--debug]

tv_grab_uk_rt --configure [--config-file FILE]

tv_grab_uk_rt --configure-api [--stage NAME]
              [--config-file FILE] [--output FILE]

tv_grab_uk_rt --list-channels [--config-file FILE]
              [--output FILE] [--quiet] [--debug]

tv_grab_uk_rt --list-lineups

tv_grab_uk_rt --lineup LINEUP_ID

=head1 DESCRIPTION

Output TV listings in XMLTV format for many channels available in the 
United Kingdom and Republic of Ireland.  Source data comes from 
machine-readable files made available from the Radio Times website.

=head1 USAGE

First run B<tv_grab_uk_rt --configure> to choose which channels you want to 
receive listings for.  Then run B<tv_grab_uk_rt> with no arguments to get
around 14 days of listings for your configured channels.

=head1 OPTIONS

B<--help> Print a help message and exit.

B<--version> Show the versions of the XMLTV libraries, the grabber and of
key modules used for processing listings.

B<--capabilities> Show which capabilities the grabber supports. For more
information, see L<http://xmltv.org/wiki/xmltvcapabilities.html>

B<--description> Show a brief description of the grabber.

B<--config-file FILE> Specify the name of the configuration file to use. 
If not specified, a default of B<~/.xmltv/tv_grab_uk_rt.conf> is used.  This 
is the file written by B<--configure> and read when grabbing.

B<--output FILE> When grabbing, write output to FILE rather than to standard
output.

B<--days N> When grabbing, grab N days of data instead of all available.
Supported values are 1-15.

B<--offset N> Start grabbing at today + N days. Supported values are 0-14.

Note that due to the format of the source data, tv_grab_uk_rt always downloads 
data for all days and then filters for days specified with --days and 
--offset. Specifying --days and/or --offset in order to speed up downloads or
reduce data transfer will therefore not work.

B<--quiet> Suppress all progress messages normally written to standard error.

B<--debug> Provide detailed progress messages to standard error. Due to the
volume of debug information produced, it is not advised to use this option 
during normal grabber use.

B<--gui OPTION> Use this option to enable a graphical interface to be used.
OPTION may be 'Tk', or left blank for the best available choice.
Additional allowed values of OPTION are 'Term' for normal terminal output
(default) and 'TermNoProgressBar' to disable the use of Term::ProgressBar.

B<--configure> Prompt for which channels to download listings for, where to 
store the cache directory for retrieved listings, what character encoding
to use for output, and also specify regional and TV platform information.

B<--list-channels> Outputs a list of every channel available to the grabber
in XMLTV format.

B<--list-lineups> Outputs a list of every channel lineup available to the 
grabber in XMLTV format.

B<--lineup LINEUP_ID> Outputs the given channel lineup in XMLTV format for
the given LINEUP_ID.

=head1 SOURCE DATA TERMS OF USE

All data is the copyright of the Radio Times and the use of this data is 
restricted to personal use only. L<http://www.radiotimes.com/>

In accessing this XML feed, you agree that you will only access its contents 
for your own personal and non-commercial use and not for any commercial 
or other purposes, including advertising or selling any goods or services, 
including any third-party software applications available to the general public.

=head1 CHARACTER ENCODING

During configuration, the software asks the user to choose the character
encoding to be used for output. Current supported encodings are UTF-8 and 
ISO-8859-1. If any users would like support for other encodings, please contact
the maintainer.

=head1 TITLE PROCESSING

Over time, the listings may contain inconsistent programme details, such as
the programme title combined with episode details for some showings of a 
programme, but separate for others; or the episode title being given as the 
programme title, and the programme title given as the episode title. Some
programme titles may also change slightly over time, or across channels. 
Enabling title processing during configuration enables this software to 
process programme titles against a list of flagged titles. The 
software will correct such programme titles, which in turn should result in 
better performance of PVR software which rely on consistent programme data. 
Please be aware that enabling title processing will result in the grabber 
taking slightly longer to complete its operation due to the extra 
processing overhead.

=head1 REGIONAL CHANNELS

During configuration, the software asks for UK postcode information. The first
half of a user's postcode is used to determine which regional channels
are likely available in their locality, and only these matching regional 
channels are included in the list of available channels. A user can enter 
'none' during configuration to disable regional channel filtering. Users in 
the Republic of Ireland should use the pseudo-postcode 'EIRE' to enable 
filtering of regional channels. Errors are quite possible, so please report 
any incorrect channel/postcode information.

=head1 TV PLATFORMS

After specifying a postcode, the software will ask the user to select their
TV service platform (Freeview, analogue, Sky, etc) from a list. Selecting one 
of these entries will filter the channel selection list (shown at the end of 
the configuration phase) to show only those channels available on the chosen 
platform. If the user has entered a valid postcode, the channel list will 
also only include those regional channels available in the user's locality.

=head1 ERROR HANDLING

tv_grab_uk_rt will only terminate early if it is impossible to continue with grabbing
data. This can be due to a lack of channel configuration data, a bad/missing
configuration file, or filesystem permission problems. Running the grabber in
non-quiet mode should report why the grabber failed.

There have been several occasions when the Radio Times channel index has been
missing from the Radio Times website. This file is essential to being able to
run the grabber, as it contains the list of channels having available listings
data. If this file is missing or empty, and there is no locally-cached copy of
the file, it will not be possible to run the grabber. The file usually
regenerates automatically over the course of the next day, at which point it
will be possible to run the grabber.

Non-fatal errors are reported during a grabber run, and can result in listings
for a channel being skipped either in part, or entirely. Progress messages
will state why data is missing when it is possible to do so. A non-zero exit 
status will normally be given when the grabber has encountered problems 
during listings retrieval.

=head1 ENVIRONMENT VARIABLES

The environment variable HOME can be set to change where the configuration
file is stored. All configuration is stored in $HOME/.xmltv/ by default. On
Windows it might be necessary to set HOME to a pathname containing no spaces.

The environment variable XMLTV_SUPPLEMENT can be set to change where the 
supplemental XMLTV files are retrieved from. By default, the file is 
retrieved from the XMLTV supplement server. See L<XMLTV::Supplement> for 
more information.

If you want the grabber to use customised local copies of the supplemental
files, you should set XMLTV_SUPPLEMENT to the path of the directory containing
a tv_grab_uk_rt/ directory containing the supplement files. For example, if
your local supplement files are stored in /usr/local/share/xmltv/tv_grab_uk_rt/
you should 'export XMLTV_SUPPLEMENT=/usr/local/share/xmltv/' before running the
grabber.

=head1 RADIO LISTINGS

Ironically, the Radio Times does not offer listings for radio. They 
have been asked about the possibility of adding radio listings, but stated 
that this would require significant development effort. It has not been
ruled out entirely, but is unlikely to be added soon.

Users who would like to obtain BBC radio listings in XMLTV format are advised 
to investigate a new grabber that obtains listings from the BBC Backstage 
service. See L<http://wiki.xmltv.org/index.php/BBC_Backstage> for more 
information.

=head1 LINEUPS

** Work in progress ** In order to allow more straightforward configuration
of programs that make use of XMLTV data, lineups contain a list of channels
and their names, EPG numbers and identifiers, to allow such programs to quickly
match XMLTV channel data against channels configured in the program. The
intention is to allow programs to be made aware of changes to a lineup and to
allow automated configuration of XMLTV listings.

=head1 MAILING LIST

You can subscribe to and read the XMLTV users mailing list by visiting 
L<http://lists.sourceforge.net/lists/listinfo/xmltv-users>. This is a source
of help and advice for new users. A searchable archive of the list is
available at L<http://news.gmane.org/gmane.comp.tv.xmltv.general>.

=head1 SEE ALSO

L<xmltv(5)>, L<http://wiki.xmltv.org>, L<http://www.radiotimes.com/>

=head1 BUGS

If you encounter a reproducible bug, please report it on the XMLTV bug 
tracker at L<http://sourceforge.net/tracker/?group_id=39046&atid=424135>, 
making sure you assign the bug to the tv_grab_uk_rt category. Please check
that the bug has not already been reported.

There is an ongoing issue with the Radio Times source data containing
mis-encoded UTF-8 characters, which we handle automatically in the grabber
and on a case-by-case basis when required. The issue has been reported to 
the Radio Times.

The source data on the Radio Times website is generated nightly, usually
between 5-6am. Occasionally the source data may not get recreated, leaving 
the source files for some (or all) channels empty. Users are encouraged 
to wait at least 1 day before reporting an issue with missing listings, 
as they frequently reappear in the next update. If listings continue to 
be missing from the Radio Times website, please report the fact on the XMLTV users 
mailing list.

There are no other reported ongoing issues.

=head1 AUTHOR

Since 2007 the maintainer has been Nick Morrott (knowledgejunkie at gmail dot com).

The original author was Ed Avis (ed at membled dot com). Parts of this code
were copied from tv_grab_se_swedb by Mattias Holmlund, and from the XMLTV
wiki L<http://www.xmltv.org/wiki/>. Regional postcode information was kindly
made available from L<http://www.ukfree.tv>.

=cut

