#!/usr/bin/perl -W

use strict;
use warnings;

# Downloads the title of each post listed at https://www.doctorofcredit.com/best-bank-account-bonuses/
# and searches the first $length characters for 'expired'

my $length = 20;		# length of title to search

use feature 'say';
require LWP::UserAgent;
require LWP::ConnCache;

my $ua = LWP::UserAgent->new;
$ua->default_header('Accept-Encoding' => scalar HTTP::Message::decodable());
$ua->conn_cache(LWP::ConnCache->new);
$ua->agent('Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36');

my $response = $ua->get('https://www.doctorofcredit.com/best-bank-account-bonuses/');
die 'Error pulling list: ' . $response->status_line unless $response->is_success;
my $list = $response->decoded_content;
my $title = index $list, '<title>';
#die "Title location error: $title" unless $title > 100 && $title < 2000;	# arbitrary safeties
$length += 7;

say 'This will take a few minutes';
while ($list =~ m#<li><a href="(https://www.doctorofcredit.com/[^>]+/)">Read our#g) {
    sleep 1;
    my $url = $1;
    $response = $ua->get($url, 'Range' => "bytes=$title-" . ($title + $length));
    if ( $response->status_line eq '206 Partial Content') {
        $response = $response->decoded_content;
    } elsif ($response->status_line eq '200 OK') {
        $response = substr $response->decoded_content, $title, $length;
    } else {
        say $url;
        say $response->status_line;
        next;
    }
    if ($response =~ s/^<title>//) {
        say $url if $response =~ /expired/i;
    } else {
        say $url;
        say 'Title read error';
        next;
    }
}