Constructor Detail

• CrawlFilters

  CrawlFilters(ImmutableConfig conf)

• CrawlFilters

  CrawlFilters(List<CrawlFilter> crawlFilters, ChainedUrlNormalizer urlNormalizers, CrawlUrlFilters urlFilters, String scope, ImmutableConfig conf)

Method Detail

• getReversedKeyRanges

   final Map<String, String> getReversedKeyRanges()

• getMaxReversedKeyRange

   final Array<String> getMaxReversedKeyRange()

• getCrawlFilters

   final List<CrawlFilter> getCrawlFilters()

• getUrlNormalizers

   final ChainedUrlNormalizer getUrlNormalizers()

• getUrlFilters

   final CrawlUrlFilters getUrlFilters()

• getScope

   final String getScope()

• getConf

   final ImmutableConfig getConf()

• isNormalizedValid

   final Boolean isNormalizedValid(HyperlinkPersistable hyperlink)

• isNormalizedValid

   final Boolean isNormalizedValid(String url)

• normalizeToEmpty

   final String normalizeToEmpty(String url, String scope)

• normalizeToEmpty

   final String normalizeToEmpty(String url)

• normalizeToNull

   final String normalizeToNull(String url, String scope)

• testUrlSatisfied

   final Boolean testUrlSatisfied(String url)

• testTextSatisfied

   final Boolean testTextSatisfied(String text)

• testKeyRangeSatisfied

   final Boolean testKeyRangeSatisfied(String reversedUrl)

• isAllowed

   final Boolean isAllowed(Node node)

  TODO : Tricky logic

• isDisallowed

   final Boolean isDisallowed(Node node)

  TODO : Tricky logic

• veryLikelyBeDetailUrl

   final Boolean veryLikelyBeDetailUrl(String url)

• veryLikelyBeIndexUrl

   final Boolean veryLikelyBeIndexUrl(String url)

• veryLikelyBeMediaUrl

   final Boolean veryLikelyBeMediaUrl(String url)

• veryLikelyBeSearchUrl

   final Boolean veryLikelyBeSearchUrl(String url)

  Notice : index url is not a search url even if it contains "search"

• toJson

   final String toJson()

• toString

   String toString()