windows版爬取csdn

use  LWP::UserAgent;
use POSIX;
use HTML::TreeBuilder::XPath; 
use Encode; 
use HTML::TreeBuilder;

open DATAFH,">csdn.html" || die "open csdn file failed:$!"; 
my $ua = LWP::UserAgent->new;
$ua->timeout(10);
$ua->env_proxy;
$ua->agent("Mozilla/8.0");
my $response = $ua->get(
my $base_dir="F:\\20161219";


if ($response->is_success) {
 print DATAFH  $response->content
 };
 close DATAFH;

 
use HTML::TreeBuilder::XPath;
  my $tree= HTML::TreeBuilder::XPath->new;
  $tree->parse_file( "csdn.html");
  ##     <a href="/zhaoyangjian724/article/category/1756569" οnclick="_gaq.push(['_trackEvent','function', 'onclick', 'blog_articles_wenzhangfenlei']); ">Oracle dump解析
  ##获取博客分类的URL，根据a标签查找
  @Links = $tree->find_by_tag_name('a');
        foreach (@Links) {
                $href = $_->attr('href');
 ###获取博客每个类别的url
 ####@href 表示所有分类的url
                 if ($href =~/category/){print "\$href is $href\n";
                 push (@href,$href);
                    };
					};
					
 #@href 是所有类别url的汇总
 print "\@href is @href\n";
 #@type 是类别名称汇总,根据ul标签查找/li/a对应的值 每个类别的值
 my @type=$tree->findvalues( '/html/body//ul[@class="panel_body"]/li/a');
 #my @type=encode("gbk", decode("utf8","@type"));
 foreach  (@type){
  my $a=encode("gbk", decode("utf8","$_")) ;
  push (@a, $a);
  };
   #@type 是类别名称汇总,根据ul标签查找/li/a对应的值
 my  @type=@a;
 print "\@type is @type\n";
 my $length=@href;
 my @tmp=();
 ##@type 表示所有分类的名称
 for ($i=0;$i<$length;$i++){
    
     print "$href[$i]===$type[$i]\n";
     push (@tmp,$type[$i])};
	 ##@tmp 所有分类的数组
	 #循环类别开始
	for ($i=0;$i<=@type - 1; $i++){
	print "\$type is $type\n";
                     #next  unless ($type[$i]) ;
                   if (! -d "$type[$i]"){
                    mkdir $type[$i];
                                };
                      chdir "$base_dir/$type[$i]";
                     					 
                 ##进入每个分类版块url
                 
                    my  $pageString;
                   my $response = $ua->get("$href[$i]");
                    ##每个版块首页url
					print "\$href[$i] is $href[$i]\n";
				    ##fh1.html每个版块首页url
                    open fh1,">fh1.html" || die "open csdn file failed:$!";
                    print fh1  $response->content;
                    close fh1;
                     my $tree= HTML::TreeBuilder::XPath->new;
                     $tree->parse_file( "fh1.html");
                   ##获取每个版块的页码数 这个方法有问题,这里是数组$_ is  150条数据 共8页
                   my  @pageString = $tree->findvalues('/html/body//div[@id="papelist"]/span');
                if ($pageString[0]){ if ($pageString[0] =~ /.*\s+.*?(\d+).*/){$pageString=$1}; };
				my @pageString =encode("gbk", decode("utf8","@pageString")) ;
				print "\@pageString is @pageString\n;";
				   ##获取$pageString
				   sleep (1);
                   unless ($pageString){$pageString=1};
                    print "\$pageString is $pageString\n";
                     ##进入每页,处理url
                     for ($j=1;$j<=$pageString + 0; $j++){
					 ##每个类别对应的url
                      my $url="href[$i]/$j";
                      print "\$url is $url\n";
                      my $response = $ua->get("$url");
					  ##fh2 每页url
                      open fh2,">fh2.html" || die "open csdn file failed:$!";
                       print fh2  $response->content;
                      close fh2;
                      #获取每页都多少条标题
                     my @pageTitles="";
                      my $tree= HTML::TreeBuilder::XPath->new;
                     $tree->parse_file( "fh2.html");
                     #获取标题,这里会拿到除了该类别下文章外,
                     my @pageTitles = $tree->findvalues('/html/body//span[@class="link_title"]');
					 my @a=();
					  foreach  (@pageTitles){
                     my $a=encode("gbk", decode("utf8","$_")) ;
                     push (@a, $a);
                        };
				     my @pageTitles=@a;
                      print "\$pageTitles[0] is  $pageTitles[0]\n";
					  print "\@pageTitles is @pageTitles\n";
					  
                     ##获取标题连接url
                       my $tree= HTML::TreeBuilder::XPath->new;
                      $tree->parse_file( "fh2.html");
					  @titleLinks="";
                      @titleLinks=$tree->find_by_tag_name('a');
                      @urlall=(); 
                      @urltmp=();
                      #@urlall除了包含每个类别的文章,还包含阅读排行里的文章
                      foreach (@titleLinks) {
                                               @titleHref = $_->attr('href');
                                               foreach (@titleHref) {
											   ###获取版块中每个页面的url
                                               if ($_ =~/zhaoyangjian724\/article\/details\/(\d+)$/){
                                                 unless ($_ ~~ @urlall) { print "\$_=========$_\n";push (@urlall ,encode("gbk", decode("utf8","$_")));}}
                                                     };
                                           };
										
                                          print "\@urlall is @urlall\n";
										  sleep (1);
                                          for ($k=0;$k<=@pageTitles - 1;$k++){
                                          print "\$urlall[$k] is $urlall[$k]\n";
										  
                                          push (@urltmp,$urlall[$k]);
                                                                           };     
                                            @urlall=@urltmp;
											
											
                                          print "\$---urlall[0] is  $urlall[0]\n";
										  sleep (1);
					for ($m=0;$m<=@urlall - 1; $m++){
					$pageTitles[$m] =~ s/\s+//g;
					print "===========================\n";
					print "$pageTitles[$m]======$urlall[$m]\n";
				    print "===========================\n";
					     $pageTitles[$m] =~ s/\*//g;
						 $pageTitles[$m] =~ s/\s+//g;
						 $pageTitles[$m] =~ s /\\//g;
						 $pageTitles[$m] =~ s /\///g;
						 $pageTitles[$m] =~ s /://g;
						 $pageTitles[$m] =~ s /\|//g;
						 $pageTitles[$m] =~ s /\<//g;
						 $pageTitles[$m] =~ s /\>//g;
						 $pageTitles[$m] =~ s /\?//g;
						  $pageTitles[$m] =~ s /\"//g;
						   $pageTitles[$m] =~ s /\'//g;
		         		 open fh3,">$pageTitles[$m].html" || die "open csdn file failed:$!";
						  my $response = $ua->get("$urlall[$m]");
                                                      
                                                  print "--------------------------------\n";
                                                  print "$urlall[$m]"."\n";
                                                  print fh3  $response->content;
                                                  close fh3;
												

                                                 
										# unlink("$pageTitles[$m].html.tmp");
                           #循环页码结束
                                                    }; 
													
													
							#循环每个分类的url结束
                                     
									 
									 
						#循环单个类别结束
						
                          }
						  
						 chdir "$base_dir";  
                  
				      }