#!/usr/bin/perl
##############################################################################
#
# 	Script to get details from FSA
#
##############################################################################

require 5.002;

use LWP::UserAgent;
use HTTP::Cookies;

$ua = new LWP::UserAgent;
$ua->agent("Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:15.0) Gecko/20100101 Firefox/15.0.1");
$cookies=new HTTP::Cookies(file=>'cookies.dat',autosave=>1);
$ua->cookie_jar($cookies);

$mainurl = "https://www.parliament.uk/mps-lords-and-offices/mps/";
$outputfile  = 'mps2018.csv';
open OHFILE, '>'.$outputfile or die "failed to open file ($!) \n";

$| = 1;

# read in the page
$mainpage = fetch_url($mainurl);
@mpage = split(/\n/, $mainpage);

$count = 0;
foreach $line (@mpage) {
  chomp($line);
  if ($line =~ /biographies/) {
    $line =~ m/href="(.*?biographies.*?)"/;
    $link = $1;

    print "fetching: $link\n";
    $page = fetch_url($link);

##print "$page\n";
##exit(0);

    @bpage = split(/\n/, $page);

    $name  = '';
    $party = '';
    $const = '';
    $email = '';

    $idx_name  = -1;
    $idx_party = -1;
    $idx_const = -1;
    foreach $bline (@bpage) {
      if ($idx_name > 0)  { $idx_name--;  } else { $idx_name = -1;  }
      if ($idx_party > 0) { $idx_party--; } else { $idx_party = -1; }
      if ($idx_const > 0) { $idx_const--; } else { $idx_const = -1; }

      if ($bline =~ /commons-biography-header/) {
        $idx_name = 2
      }
      if ($idx_name == 0) {
        $name = $bline;
        $name =~s/^\s+//;
        $name =~s/\s+$//;
      }

      if ($bline =~ /commons-party"/) {
        $idx_party = 1
      }
      if ($idx_party == 0) {
        $party = $bline;
        $party =~s/^\s+//;
        $party =~s/\s+$//;
      }

      if ($bline =~ /commons-constituency/) {
        $idx_const = 1
      }
      if ($idx_const == 0) {
        $const = $bline;
        $const =~s/^\s+//;
        $const =~s/\s+$//;
      }

      if ($bline =~ /hypEmailAddress/) {
        $bline =~ m/href="(.*?)"/;
        $email = $1;
        $email =~ s/mailto://;
      }

    }

    print OHFILE $name.'|'.$party.'|'.$const.'|'.$email."\n";

  }
}


close OHFILE;

sub do_post {

  my $url = shift(@_);
  my $pc  = shift(@_);

  my $res = $ua->post( $url,
    { pageNumber=> '0',
      firmName=> '',
      postcodeOut => $pc,
      searchType => '2',
      currAuthorisedInd => 'on',
    ddd => 'Submit' }  );

  # Return the result

  if ($res->is_success) { return $res->content; }
  else                  { return ''; }

}


sub fetch_url {

  my $url = shift(@_);

  # Create a request

  my $req = new HTTP::Request GET => $url;

  # Pass request to the user agent and get a response back

  my $res = $ua->request($req);

  # Return the result

  if ($res->is_success) { return $res->content; }
  else                  { return ''; }

}