当前位置：Gxlcms > PHP教程 > 利用curl，正则表达式做的一个php蜘蛛抓取器

利用curl，正则表达式做的一个php蜘蛛抓取器

时间：2021-07-01 10:21:17 帮助过：22人阅读

凤网fcms内容管理系统
get.php 抓取框架，对网页内容的分析处理并进行相关替换
std.php 通用正则
news_67_com.php 对http://news.67.com 的抓取分析器
先抓列表，再抓内容页。
还欠缺监控，统计，错误处理功能。个人觉得还是比较好玩。
  1. include_once dirname(__FILE__) . '/std.php';
  3. $site = array(
  4. 'aname' => '中国娱乐网',
  5. 'domain' => 'news.67.com',
  6. 'dirname' => '目录名称，用于匹配基于目录不同的正文',
  7. 'gettype' => 'default',
  8. //获取主文件
  9. 'creg' => '/(?si)(.*?)\<\!--文章 end-->/',
  10. 'code' => 'utf-8',
  11. 'sub' => '获取子目录正则',
  12. 'content' => 'tag1',
  13. 'img_upload'=> array('tag1' => ''),
  14. //下一页
  15. 'reg_next' => '/(?is)下一页\>\><\/a>/',
  16. 'key0' => '/(?is)/',
  17. 'key0_ap' => array(array(',', '|'), ' '),
  18. 'tag0' => '/(?is)

    (https://img.gxlcms.com/[^<^>]*?)<\/h1>/',
  19. 'tag0_arp' => array(
  20. array(
  21. '/(?is)\(组图\)/',
  22. '/(?is)\(图\)/',
  23. '/(?is)\(图\.\./',
  24. '/(?is)\(组图\.\./',
  25. '/(?is)\./',
  26. '/(?is)(《|》)/',
  27. ),
  28. array(
  29. '', '', '', '', '', '',
  30. )
  31. ),
  32. 'tag1' => '/(?is)(.*?)https://img.gxlcms.com/[^/',
  33. 'tag1_brp' => array(
  34. array(
  35. '/(?is)（.*?）/',
  36. '/(?is)\(.*?\)/',
  37. '/(?is)\s*

    .*?(.*?)<\/p>\s*/',

  38. '/(?is)\s*

    \s*/',

  39. '/(?is)\s*

    \s*/',

  40. '/(?is)　/',
  41. '/(?is)
    /',
  42. '/(?is)\s*

    \s*/',

  43. '/(?is)\s*

    \s*/',

  44. '/(?is)\s*
    \s*/',
  45. '/(?is)\s*<\/center>\s*/',
  46. '/(?is)\s*

    \s*/',

  47. ),
  48. array(
  49. '', '', '

    ', '

    ', '

    ', '', '', '

    ', '

    ', '

    ', '

    ', '

    '

  50. ),
  51. ),
  52. 'tag1_arp' => array(
  53. array(
  54. '/(?is)

    <\/p>/',

  55. '/(?is)<\/strong>/'
  56. ),
  57. array(
  58. '', ''
  59. ),
  60. ),
  61. 'strip' => array('tag1' => ''),
  62. 'tag2' => '/(?is)导读：\s*(.*?)\s*<\/div>/',
  63. 'tag2_arp' => array(
  64. array(
  65. '/(?is)　/'
  66. ),
  67. array(
  68. ''
  69. ),
  70. ),
  71. 'tag3' => '/(?is)(中国娱乐网)/',
  72. 'tag4' => '/(?is)日期：(\d+-\d+-\d+ \d+:\d+:\d+).*?<\/div>/',
  73. );
  75. $map = array(
  76. 'tag' => 'key0',
  77. 'title' => 'tag0',
  78. 'content' => 'tag1',
  79. 'summary' => 'tag2',
  80. 'source' => 'tag3',
  81. 'pub_date' => 'tag4',
  82. );
  84. $site_list = array(
  85. 'aname' => '中国娱乐网',
  86. 'domain' => 'www.67.com',
  87. 'gettype' => 'default',
  88. 'creg' => '/(?si)(.*?)/',
  89. 'code' => 'gbk',
  90. 'reg_next' => '/(?si)
  91. //链接
  92. 'tag0' => '/(?is).*?https://img.gxlcms.com/[^<^>]*?<\/a>.*?<\/div>/',
  93. //标题
  94. 'tag1' => '/(?is).*?(https://img.gxlcms.com/[^<^>]*?)<\/a>.*?<\/div>/',
  95. 'tag1_arp' => array(
  96. array(
  97. '/(?is)\(组图\)/',
  98. '/(?is)\(图\)/',
  99. '/(?is)\(图\.\./',
  100. '/(?is)\(组图\.\./',
  101. '/(?is)\./',
  102. '/(?is)(《|》)/',
  103. ),
  104. array(
  105. '', '', '', '', '', '',
  106. )
  107. ),
  108. );
  110. $list_map = array(
  111. 'url' => 'tag0',
  112. 'title' => 'tag1',
  113. );
  115. $site_list_sub = array();

  1. global $std;
  2. $std = array(
  3. 'url' => '[0-9a-zA-Z\.\:\-\/%_#;&]+',
  4. 'img' => '/(?is)/',
  5. );
  1. /**
  2. * test.php
  3. *
  4. * @author xzfred
  5. * @copyright 2009 fengone.com
  6. * @created 2010-12-07 .
  7. * @version $Id: php.php 3 2008-10-10 07:49:21Z fred $
  8. * SVNPath $HeadURL: http://192.168.0.16/svn/vim/skeletons/php.php $
  9. */
  10. /*
  11. include_once "std.php";
  12. include_once "lady_163_com.php";
  13. */
  14. include_once $GLOBALS['g_dir_core'] . "get.php";
  16. //================================================================================
  17. include_once DIR_HOST_TAG . '/tuku_ent_china_com.php';
  18. $obj = new FcHtmlParse($site);
  19. $c = $obj->parse(file_get_contents("http://tuku.ent.china.com/fun/html/2011-08-23/181703.xml"));
  20. echo "\n\n\n ===================\n";
  21. echo $c['field']['tag1'][0];
  22. echo "\n\n\n ===================\n";
  23. var_dump($c);
  25. exit();
  26. //列表测试
  27. $obj = new FcHtmlParse($site_list);
  28. $c = $obj->parse(file_get_contents("http://tuku.ent.china.com/fun/html/3569_1.html"));
  29. var_dump($c);
  30. exit();
  31. /*
  33. $obj = new FcHtmlGet($site);
  34. $c = $obj->getPage('http://star.pclady.com.cn/entertainment/ss/1106/703240.html');
  35. var_dump($c);
  37. $obj = new FcHtmlGet($site);
  38. $c = $obj->getPage('http://star.pclady.com.cn/entertainment/ss/1106/703240.html');
  39. var_dump($c);
  41. $obj = new FcHtmlParse($site);
  42. $img_obj = new FcHtmlImgUpload($site);
  44. $data = file_get_contents("e:/b.html");
  45. $c = $obj->parse($data);
  46. $ic = $img_obj->upload($c['tag']['tag1'][0]);
  47. var_dump($ic);
  49. $data = file_get_contents("e:/a.html");
  50. $c = $obj->parse($data);
  51. $ic = $img_obj->upload($c['tag']['tag1'][0]);
  52. var_dump($ic);
  53. */
  55. //var_dump($c['tag']['tag1']);
  1. include_once $GLOBALS['g_dir_core'] . 'host/std.php';
  3. $site = array(
  4. 'aname' => '中华网娱乐图库',
  5. 'domain' => 'tuku.ent.china.com',
  6. 'dirname' => '目录名称，用于匹配基于目录不同的正文',
  7. 'gettype' => 'default',
  8. //获取主文件
  9. 'creg' => '/(?si)(.*?<\/list>)/',
  10. 'code' => 'utf-8',
  11. 'sub' => '获取子目录正则',
  12. 'content' => 'tag1',
  13. 'img_upload'=> array('tag1' => ''),
  14. //下一页
  15. 'reg_next' => '/(?is)下一页<\/a>/',
  16. 'key0' => '/(?is)/',
  17. 'key0_ap' => array(array(',', '|'), ' '),
  18. 'tag0' => '/(?is)title="(https://img.gxlcms.com/[^"]*?)"/',
  19. 'tag0_arp' => array(
  20. array(
  21. '/(?is)\(图\)/',
  22. '/(?is)\"/',
  23. '/(?is)独家：/',
  24. '/(?is)独家:/',
  25. '/(?is)(《|》)/',
  26. ),
  27. array(
  28. '', '', '', '', '',
  29. )
  30. ),
  31. 'tag1' => '/(?is)(.*?)<\/list>/',
  32. 'tag1_brp' => array(
  33. array(
  34. '/(?is)\s*\s*/'
  35. ),
  36. array(
  37. '

    https://www.gxlcms.com/$1

  38. https://www.gxlcms.com/$1

  39. $3

    '
  40. )
  41. ),
  42. 'tag1_arp' => array(
  43. array(
  44. '/(?is)

    (https://img.gxlcms.com/[^<^>]*?)<\/p>/',

  45. '/(?is)\<br\/\>/',
  46. ),
  47. array(
  48. '

    https://www.gxlcms.com/$1

    ',
  49. '',
  50. )
  51. ),
  52. 'strip' => array('tag1' => ''),
  53. //网名
  54. 'tag3' => '/(?is)(https://img.gxlcms.com/[^<^>]*?)<\/span>/',
  55. 'tag4' => '/(?is)(中华网)/'
  56. );
  58. $map = array(
  59. 'tag' => 'key0',
  60. 'title' => 'tag0',
  61. 'content' => 'tag1',
  62. 'author' => 'tag3',
  63. 'source' => 'tag4'
  64. );
  66. $site_list = array(
  67. 'aname' => '中华网娱乐图库',
  68. 'domain' => 'tuku.ent.china.com',
  69. 'gettype' => 'default',
  70. 'creg' => '/(?si)(.*?)<\/div>/',
  71. 'code' => 'utf-8',
  72. 'reg_next' => '/(?si)
  73. //链接
  74. 'tag0' => '/(?is).*?<\/span>https://img.gxlcms.com/[^<^>]*?<\/a><\/div>/',
  75. 'tag0_brp' => array(
  76. array(
  77. '/(?is)\.htm/',
  78. ),
  79. array(
  80. '.xml'
  81. )
  82. ),
  83. //标题
  84. 'tag1' => '/(?is).*?<\/span>(https://img.gxlcms.com/[^<^>]*?)<\/a><\/div>/',
  85. 'tag1_arp' => array(
  86. array(
  87. '/(?is)\(图\)/',
  88. '/(?is)\"/',
  89. '/(?is)独家：/',
  90. '/(?is)独家:/',
  91. '/(?is)(《|》)/',
  92. ),
  93. array(
  94. '', '', '', '', '',
  95. )
  96. ),
  97. );
  99. $list_map = array(
  100. 'url' => 'tag0',
  101. 'title' => 'tag1',
  102. );
  104. $site_list_sub = array();

人气教程排行

本站所有资源全部来源于网络，若本站发布的内容侵害到您的隐私或者利益，请联系我们删除！