perl script to send download file from website - integration with Jenkins- using Machanize

#!/usr/bin/perl

#use strict;
#use warnings;
use WWW::Mechanize;
use LWP;
use JSON;
use Data::Dumper;
use Getopt::Long;
use POSIX qw/ strftime /;
$| = 1;
my $username = 'something@something.com';
my $password = 'xxxxxx';
my $loginurl= 'https://loginurl';
my $insighturl= 'https://loginurl';
my %region_Squeryurl= ( 'apac' => '/cloud/paths/download/prod/slow-query/9538?s=1767176',
        'na' => '/cloud/paths/download/prod/slow-query/8972?s=1767151',
        'emea' => '/cloud/paths/download/prod/slow-query/3664?s=670131');
my $region;
GetOptions(
        'region|r=s' => \$region  );
die 'usage: $0 -r [apac|na|emea]'  unless( $region_Squeryurl{$region} );


my $date = my $time = strftime("%Y%m%d", localtime());
#my $log_dir = "$ENV{JENKINS_HOME}/docroot/getlog/";
my $log_dir = "/mnt/getlog/";

my $fullpath= "$log_dir"."$region/"."$date/";


unless (-d $fullpath ){
print "creating $fullpath ";
    system( "sudo mkdir -p  $fullpath" )  ;
    system( "sudo chown -R www-data:www-data  $fullpath" )  ;

}




my $mech = WWW::Mechanize->new();
$mech -> cookie_jar(HTTP::Cookies->new());

download_file ($username,$password,$loginurl,$insighturl,$region_Squeryurl{$region},$date,$fullpath);

sub download_file {

    my ($username,$password,$loginurl,$insighturl,$region_Squeryurl,$date,$fullpath)=@_;
    $mech -> get("$loginurl");
    $mech -> form_id('user-login');
    $mech -> field ('name' => $username);
    $mech -> field ('pass' => $password);
    $mech -> click_button (value => 'Sign in');
    $mech -> get ("$insighturl$region_Squeryurl");
    my $jdata_1 =   decode_json($mech ->content());

    $jdata_1 = $jdata_1->[1]->{'url'};
    $jdata_1 =~ s/'//g;
    print "$insighturl$jdata\n";
    while (1){
    sleep(2);
     $mech -> get ("$insighturl$jdata_1");
    $jdata = decode_json($mech ->content());
     print "get url $insighturl$jdata_1\n";
        if ( $jdata->[3]->{'delay'} =~ /^0$/ ) {
         print "mnt url --> ".$jdata->[3]->{'url'}."\n";
         print "delay --->". $jdata->[3]->{'delay'}."\n";
        last;
        }

    }
    my $D_url =  $jdata->[3]->{'url'}."\n";

    $mech -> get ("$D_url");
    $mech->save_content( "/tmp/mysql-slow-log-$date.$$.log" );
    system("sudo mv /tmp/mysql-slow-log-$date.$$.log ${fullpath}mysql-slow-log-$date.$$.log" );
    print $mech->content_type()."\n";
    if ($mech ->status()){
    print "created successfully ".$fullpath."mysql-slow-log-$date.$$.log\n ";
    my $msql_slow_log_q=$fullpath."mysql-slow-log-$date.$$.log";
    system("/bin/bash $ENV{WORKSPACE}/scripts/pt-query-digest.sh $region $msql_slow_log_q ");
    }

}
#print $mech ->status();
#print $mech ->content();

#print $mech-> content();


logstash - ELK

input {
        file {
         path => [  "/var/log/apache2/example.com_access_log",
                    "/var/log/apache2/domain.com_access_log",
                    "/var/log/apache2/somesite.com_access_log",
                    "/var/log/apache2/access.log"
                 ]
         type => "apache-access-log"
            }


         }


filter {
  if [type] == "apache-access-log" {
    grok {
      match => { "message" => "%{COMBINEDAPACHELOG}" }
    }
  }


  date {
    match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
  }
}






logstash.conf 

output {
stdout {
    codec => rubydebug
  }
elasticsearch {
         bind_host => "127.0.0.1"
        cluster => "elasticsearch"
         host => "127.0.0.1"
        }
   }


web scraping part 1

This script download all emails of all recruiters. 

=============================================
using simple regex. 
=============================================
#!/usr/bin/perl

use strict;
use warnings;
use Data::Dumper;
use LWP::Simple;




my $html =  get "$url";

my @html = split(/\n/,$html);

for my $i  (@html){
    next unless  $i =~ /<a href="\/recruiter\/(\S.*\/)">/ ;
    print "${url}${1}\n";
    my $indiv_recruiter = get("$url$1");
    my @indiv_recruiter = split(/\n/,$indiv_recruiter);
    for my $j (@indiv_recruiter ){

     next unless $j =~ /<a class="recruiter-email-link" href="mailto:(\S.*?\@CyberCoders\.com)">/ ;
        print "$1\n"

    }
}

=========================================================
Using HTM::treebuilder::Xpath

#!/usr/bin/perl

use strict;
use warnings;
use Data::Dumper;
use LWP::Simple;
use HTML::TreeBuilder;
use HTML::TreeBuilder::XPath;




my $html =  get "$url";



my $tree = HTML::TreeBuilder::XPath->new_from_content();
$tree->parse($html);

my @customers = $tree->findnodes('//div[@class="recruiter-item"');

for my $custm (@customers){
    my @all_a_tag = $custm->look_down('_tag' => 'a'  ) ;
     my $recruiter_pth = $all_a_tag[0]->attr('href') ."\n" ;
      $recruiter_pth =~ s/\/recruiter\///g;
      get_email($recruiter_pth);

}


sub get_email {

my ($r_pth)=@_;
my $html =  get ("$url$r_pth") ;
my    $tree2 = HTML::TreeBuilder->new_from_content($html);
    my @mailto = $tree2->look_down( _tag => 'a' , class => 'recruiter-email-link' );
     print $mailto[0]->attr('href')."\n";
   print "get $url$r_pth";
}

Cloud developer Chef


> The sender of this email is registered with Naukri.com as dynproindia. To respond back directly to the Employer, please click on Reply button, or send an email to savitha@dynproindia.com
> Experience required for the Job: 5 - 15 years
> Annual Salary of the Job: 0.0 - 30.0 Lacs
>
> Apply Now   Reply
>
>
> Dear Candidate,
>>
>> Hi All
>>
>> About Dynpro
>> * DynPro specializes in helping companies build infrastructures that can weather the most fierce storms by focusing on strong fundamentals: appropriate and effective systems, the right people, and proven business processes. Our worldwide force of technology and business experts can identify the right solutions to meet any challenge. Our relationships with the world's best service and product providers ensures that you get the most practical, value-driven solutions available anywhere. Location: USA,UK, Kolkata,Bangalore,Noida
>>
>> Skill sets -Cloud,Chef,AWS
>>
>> Experience: 5+ Yrs
>>
>> Location: Bangalore
>>
>> NP: Who can join Immediate or within 15 to 20 days of time
>> JD
>>
>> Following is the detailed Job description:
>> 1. Hands on experience (preferably 2 years) on Chef and Ruby scripting.
>> 2. Aware of cloud technologies such as AWS, Azure.
>> 3. Experience on working on DevOps methodology.
>> 4. Good to have - shell, python scripting hands on.
>> 5. Technically strong with a min of 5 years of experience.
>> 6. Self-motivated, team player and ability to work in high pressure scenarios.
>> 7. Capable of taking instructions and action them without too much hand holding.
>> 8. Show technical innovation and strive for continuous improvement on the platform.
>>
>> Key skills required for the job are:
>>
>> Devops-Chef(Mandatory)
>> Amazon Web Services - AWS(Mandatory)
>> ServiceNow - SaaS
>>
>>
>>
>> Mode Of Employment: Contract to Hire
>>
>> If your profile is relevant to the JD then kindly revert me back with your updated resume and with below Details
>>
>> Total Exp :
>> Rel Exp :
>> Current CTC :
>> Expected CTC :
>> Notice Period :
>> Work Location :
>> Pan no:
>> Passport no:
>> Educational Background with University name and Year of Passing and with %
>>
>> Thanks & Regards,
>>
>> Savita
>> Dynpro India Pvt Ltd.
>> Bangalore
>> Email: savitha@dynproindia.com
>> URL: www.dynproindia.com ; www.dynpro.com
>
>
> Is this job relevant to you? Yes No
> Your feedback would help us in sending you the most relevant job opportunities
> Disclaimer:
> The sender of this email is registered with naukri.com as dynproindia (venkatesh@dynpro.com, #3793, 7th Main HAL 2nd Stage Indiranagar, 121, Dickenson Road,, BANGALORE, Karnataka - 560038) using Naukri.com services. The responsibility of checking the authenticity of offers/correspondence lies with you. If you consider the content of this email inappropriate or spam, you may: Forward this email to: compliance@naukri.com or you can Block this Company from searching your resume in the database.
> Advisory: Please do not pay any money to anyone who promises to find you a job. This could be in the form of a registration fee, or document processing fee or visa charges or any other pretext. The money could be asked for upfront or it could be asked after trust has been built after some correspondence has been exchanged. Also please note that in case you get a job offer or a letter of intent without having been through an interview process it is probably a scam and you should contact compliance@naukri.com for advise.
> Update Your Resume Now to be on the top of Employer's Resume Search
> Create your own job alert to receive similar jobs directly in your inbox.
> Your Resume has only 30 seconds to impress a Recruiter. Get your resume written by Experts.
> Call 1800-102-5557 now! (Toll-Free)

Other Articles

Enter your email address: