Java网络爬虫.docVIP

package com.media999.crawlwebsite.crawl; /** * @name:WebPage bean(页面对象) * * @version:1.0 * * @author:Sun Wei(JerrySun) * * @date:21-Oct-2009 * * @description:Store webpages source-code and visit url(保存页面的本地文件名和该页面的URL地址) * * @modify_person: * * @modify_content: * * @last_modify_date: * */ public class webPageBean { // file name 页面的本地文件名 private String strPageContent; // visit url 页面的访问地址 private String strPageUrl; // 保存页面的编码 private String strEncode; public String getStrEncode() { return strEncode; } public void setStrEncode(String strEncode) { this.strEncode = strEncode; } // Get and Set methods get、set方法 public String getStrPageContent() { return strPageContent; } public void setStrPageContent(String strPageContent) { this.strPageContent = strPageContent; } public String getStrPageUrl() { return strPageUrl; } public void setStrPageUrl(String strPageUrl) { this.strPageUrl = strPageUrl; } } package com.media999.crawlwebsite.crawl; /** * @name:Crawl website and create index for the website (抓取网站内容并创建索引类) * * @version:1.0 * * @author:Unknown (get it from internet) * * @reference：/Jack2007/archive/2008/03/24/188138.html * * @date:20-Oct-2009 * * @description:Crawl website and create index for the website(抓取网站内容并创建索引) * * @modify_person:Sun Wei(JerrySun) * * @modify_content: Add two functions,one is parse page content,the other is create Lucene Index * (添加分析页面内容，创建索引的功能) * * @last_modify_date:20-Oct-2009 * */ import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; import .URL; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.log4j.Logger; import org.apache.log4j.PropertyConfigurator; import com.media999.tools.util.Configration; import com.media999.crawlwebsite.index.makeindex; import c