crawler4j编译错误,类CrawlConfig-VariableDeclaratorId应为



代码将不会编译。我将JRE更改为1.7。编译器不会在Eclipse中突出显示类,并且CrawlConfig在编译器中似乎失败了。该类应该在Linux中从命令行运行。

有什么想法吗?

编译器错误-说明资源路径位置类型令牌"crawlStorageFolder"上的语法错误,此令牌之后应为VariableDeclaratorId zeocrawler.java/zeowebcrawler/src/main/java.com/example line 95 java Problem

import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;
import edu.uci.ics.crawler4j.url.WebURL;
    public class Controller {   
         String crawlStorageFolder = "/data/crawl/root";
        int numberOfCrawlers = 7;
        CrawlConfig config = new CrawlConfig();
        config.setCrawlStorageFolder(crawlStorageFolder);
        PageFetcher pageFetcher = new PageFetcher(config);
        RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
        RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
        CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);
           controller.addSeed("http://www.senym.com");
            controller.addSeed("http://www.merrows.co.uk");
            controller.addSeed("http://www.zeoic.com");

            controller.start(MyCrawler.class, numberOfCrawlers);




    }
    public URLConnection connectURL(String strURL) {
        URLConnection conn =null;
        try {
        URL inputURL = new URL(strURL);
        conn = inputURL.openConnection();
            int test = 0;
        }catch(MalformedURLException e) {
        System.out.println("Please input a valid URL");
        }catch(IOException ioe) {
        System.out.println("Can not connect to the URL");
        }
        return conn;
        }
    public static void updatelongurl()
    {
//      System.out.println("Short URL: "+ shortURL);
//        urlConn =  connectURL(shortURL);
 //       urlConn.getHeaderFields();
 //       System.out.println("Original URL: "+ urlConn.getURL());
/* connectURL - This function will take a valid url and return a 
URL object representing the url address. */

    }



    public class MyCrawler extends WebCrawler {
        private      Pattern FILTERS = Pattern.compile(".*(\.(css|js|bmp|gif|jpe?g" 
                                                          + "|png|tiff?|mid|mp2|mp3|mp4"
                                                          + "|wav|avi|mov|mpeg|ram|m4v|pdf" 
                                                          + "|rm|smil|wmv|swf|wma|zip|rar|gz))$");
        /**
         * You should implement this function to specify whether
         * the given url should be crawled or not (based on your
         * crawling logic).
         */
        @Override
        public boolean shouldVisit(WebURL url) {
                String href = url.getURL().toLowerCase();
                return !FILTERS.matcher(href).matches() && href.startsWith("http://www.ics.uci.edu/");
        }
        /**
         * This function is called when a page is fetched and ready 
         * to be processed by your program.
         */
        @Override
        public void visit(Page page) {          
                String url = page.getWebURL().getURL();
                System.out.println("URL: " + url);
                if (page.getParseData() instanceof HtmlParseData) {
                        HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
                        String text = htmlParseData.getText();
                        String html = htmlParseData.getHtml();
                        List<WebURL> links = htmlParseData.getOutgoingUrls();
                        System.out.println("Text length: " + text.length());
                        System.out.println("Html length: " + html.length());
                        System.out.println("Number of outgoing links: " + links.size());
                }
        }
} 

这是一个非常奇怪的错误,因为代码看起来很干净。尝试在命令行中使用-clean选项启动eclipse。

更改

String crawlStorageFolder = "/data/crawl/root";

String crawlStorageFolder = "./data/crawl/root";

即添加一个前导.

相关内容

  • 没有找到相关文章

最新更新