当前位置：Gxlcms > PHP教程 > php正则抓取整个域名下的图片

php正则抓取整个域名下的图片

时间：2021-07-01 10:21:17 帮助过：11人阅读

代码出处：jUnion

适用平台：Windows， Linux(Ubuntu)，php-5.2.5+，Apache

功能：抓取整个站点的图片，暂无借助php的curl插件开发， 后期完善

配置：config目录下
domain_name：域名（默认：bizhibar.com）
request_site：网站网址（默认：http://www.bizhibar.com/）
request_url：从网站的哪个页面开始（默认：http://www.bizhibar.com/）
accept_type: 图片类型（默认：gif, bmp, png, ico, jpg, jpeg）
save_path：图片保存路径（默认：savefiles/）
partition_name：图片保存目录名称前缀（默认：img_）
dir_file_limit: 每个目录容许多少个文件（默认：100）
serialize_img_size: 当读取了多少个图片地址才缓存到cache目录下的accompImg文件当中，下次继续抓取的时候会忽略这些地址。（默认：30）
serialize_url_size：与serialize_url_size一样，已读取多少个链接地址才缓存到cache目录
下的overURL，下次继续抓取的时候忽略这些地址。（默认：10）

说明：欢迎诸君批评指教，有任何新问题或者需要改进的地方，请您反馈给我
  1. set_time_limit(0);
  2. require dirname(__FILE__).DIRECTORY_SEPARATOR.'include'.DIRECTORY_SEPARATOR.'Capture.const.php';
  3. require __Home__.'include'.__Os__.'Capture.class.php';
  5. $_cfg = array(
  6. 'site' => __Home__.'config'.__Os__.'capture.site.php',
  7. 'preg' => __Home__.'config'.__Os__.'capture.preg.php',
  8. 'accompImg' => __Home__.'cache'.__Os__.'accompImg',
  9. 'overURL' => __Home__.'cache'.__Os__.'overURL'
  10. );
  12. $_parse = new Capture( $_cfg );
  13. $_parse->parseQuestUrl();
  15. ?>
  1. /**
  2. * The main class
  3. * @author pankai<530911044@qq.com>
  4. * @date 2013-08-10
  5. */
  6. class Capture {
  7. private static $_Config = array();
  9. private static $_CapSite = NULL;
  10. private static $_CapPreg = NULL;
  12. private static $_overURL = array();
  14. private $_mark = FALSE;
  15. private static $_markTime = 1;
  16. /**
  17. * initialize the main class: Capture
  18. * @param $_cfg array
  19. */
  20. public function __construct( &$_cfg ) {
  21. self::$_Config = &$_cfg;
  23. self::$_CapSite = require $_cfg['site'];
  24. self::$_CapPreg = require $_cfg['preg'];
  26. foreach( self::$_CapPreg as $_key => $_value ) {
  27. self::$_CapPreg[$_key] = str_replace( '_request_site', self::$_CapSite['request_site'], $_value );
  28. }
  30. self::import( 'file.OperateFile' );
  31. if( file_exists( $_cfg['overURL'] ) && filesize( $_cfg['overURL'] ) > 0 ) {
  32. $_contents = OperateFile::readText( $_cfg['overURL'], filesize( $_cfg['overURL'] ) );
  33. self::$_overURL = unserialize( $_contents );
  34. }
  36. self::import('pivotal.Pivotal');
  37. if( file_exists( $_cfg['accompImg'] ) && filesize( $_cfg['accompImg'] ) > 0 ) {
  38. $_contents = OperateFile::readText( $_cfg['accompImg'], filesize( $_cfg['accompImg'] ) );
  39. Pivotal::$_accompImg = unserialize( $_contents );
  40. }
  42. }
  43. /**
  44. * load class, follow Java pragrammer(package): import com.jUnion.Capture
  45. * @param $_class
  46. */
  47. public static function import( $_class ) {
  48. require_once __Home__.'include'.__Os__.str_replace( '.', __Os__, $_class ).'.class.php';
  49. }
  51. /**
  52. * create an instance of Pivotal class
  53. * @param $_source
  54. */
  55. private function getCapInstance( &$_source ) {
  56. $this->_mark = FALSE;
  58. $_Captal = new Pivotal( self::$_Config, $_source );
  59. $_tagA = $_Captal->parseUrl();
  61. $this->_mark = TRUE;
  63. return $_tagA;
  64. }
  66. /**
  67. * go forward one by one
  68. * @param $_tagArr
  69. */
  70. private function roundTagA( &$_tagArr ) {
  71. if( $_tagArr == NULL ) {
  72. return;
  73. }
  74. $_tagArrLength = count( $_tagArr );
  75. for( $i = 0; $i < $_tagArrLength; $i ++ ) {
  76. if( is_array( $_tagArr[ $i ] ) ) {
  77. $this->roundTagA( $_tagArr[ $i ] );
  78. }
  79. else {
  80. if( stripos( $_tagArr[$i], self::$_CapSite['domain_name'] )
  81. === FALSE ) {
  82. continue;
  83. }
  84. if( in_array( $_tagArr[$i], self::$_overURL ) ) {
  85. continue;
  86. }
  87. self::$_overURL[] = $_tagArr[$i];
  88. if( count( self::$_overURL ) % self::$_CapSite['serialize_url_size'] == 0 ) {
  89. OperateFile::setText( self::$_Config['overURL'], serialize( self::$_overURL ) );
  90. }
  91. do {
  92. $_tagA = $this->getCapInstance( Http::get( $_tagArr[$i] ) );
  93. sleep( self::$_CapSite['preform_page_time'] * self::$_markTime );
  94. if( $this->_mark === TRUE ) {
  95. self::$_markTime = self::$_CapSite['preform_page_time'];
  96. break;
  97. }
  98. self::$_markTime *= 2;
  99. } while( true );
  100. /* parse the main page and return next page */
  101. $this->roundTagA( $_tagA );
  102. }
  103. }
  104. }
  106. public function parseQuestUrl() {
  107. self::import('http.Http');
  108. $_round_Arr = $this->getCapInstance( Http::get( self::$_CapSite['request_url'] ) );
  109. $this->roundTagA( $_round_Arr );
  110. }
  111. }
  112. ?>

人气教程排行

本站所有资源全部来源于网络，若本站发布的内容侵害到您的隐私或者利益，请联系我们删除！