All Implemented Interfaces:: ai.platon.pulsar.crawl.Crawler , java.lang.AutoCloseable


public class StreamingCrawler<T extends UrlAware>
extends AbstractCrawler

Nested Class Summary

Nested Classes
Modifier and Type	Class	Description
`public class`	`StreamingCrawler.Companion`

Field Summary

Fields
Modifier and Type	Field	Description
`private final Integer`	`numPrivacyContexts`
`private final Integer`	`numMaxActiveTabs`
`private final Integer`	`fetchConcurrency`
`private final Duration`	`idleTimeout`
`private final Duration`	`idleTime`
`private final Boolean`	`isIdle`
`private final String`	`defaultArgs`
`private final Boolean`	`isActive`
`private String`	`jobName`
`private ProxyPool`	`proxyPool`
`private final Integer`	`id`
`private Sequence<T>`	`urls`
`private final LoadOptions`	`defaultOptions`
`private final GlobalCacheFactory`	`globalCacheFactory`
`private final CrawlEventHandler`	`crawlEventHandler`
`private final String`	`name`
`private final AtomicBoolean`	`closed`
`private final PulsarSession`	`session`
`private final Boolean`	`autoClose`

Constructor Summary

Constructors
Constructor	Description
`StreamingCrawler(Sequence<T> urls, LoadOptions defaultOptions, PulsarSession session, GlobalCacheFactory globalCacheFactory, CrawlEventHandler crawlEventHandler, String name, Boolean autoClose)`

Method Summary

Modifier and Type	Method	Description
`final Integer`	`getNumPrivacyContexts()`
`final Integer`	`getNumMaxActiveTabs()`
`final Integer`	`getFetchConcurrency()`
`final Duration`	`getIdleTimeout()`
`final Duration`	`getIdleTime()`
`final Boolean`	`getIsIdle()`
`final String`	`getDefaultArgs()`
`Boolean`	`getIsActive()`
`final String`	`getJobName()`
`final Unit`	`setJobName(String jobName)`
`final ProxyPool`	`getProxyPool()`
`final Unit`	`setProxyPool(ProxyPool proxyPool)`
`final Integer`	`getId()`
`final Sequence<T>`	`getUrls()`	The url sequence
`final Unit`	`setUrls(Sequence<T> urls)`	The url sequence
`final LoadOptions`	`getDefaultOptions()`	The default load options
`final GlobalCacheFactory`	`getGlobalCacheFactory()`	A optional global cache which will hold the retry tasks
`final CrawlEventHandler`	`getCrawlEventHandler()`	The crawl event handler
`final String`	`getName()`	The crawl name
`final AtomicBoolean`	`getClosed()`
`final PulsarSession`	`getSession()`
`final Boolean`	`getAutoClose()`
`Unit`	`run()`
`Unit`	`run(CoroutineScope scope)`
`final Unit`	`quit()`

Methods inherited from class ai.platon.pulsar.crawl.StreamingCrawler
close
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

StreamingCrawler

StreamingCrawler(Sequence<T> urls, LoadOptions defaultOptions, PulsarSession session, GlobalCacheFactory globalCacheFactory, CrawlEventHandler crawlEventHandler, String name, Boolean autoClose)

Method Detail

getNumPrivacyContexts
```
 final Integer getNumPrivacyContexts()
```

getNumMaxActiveTabs
```
 final Integer getNumMaxActiveTabs()
```

getFetchConcurrency
```
 final Integer getFetchConcurrency()
```

getIdleTimeout
```
 final Duration getIdleTimeout()
```

getIdleTime
```
 final Duration getIdleTime()
```

getIsIdle
```
 final Boolean getIsIdle()
```

getDefaultArgs
```
 final String getDefaultArgs()
```

getIsActive
```
 Boolean getIsActive()
```

getJobName
```
 final String getJobName()
```

setJobName
```
 final Unit setJobName(String jobName)
```

getProxyPool
```
 final ProxyPool getProxyPool()
```

setProxyPool

 final Unit setProxyPool(ProxyPool proxyPool)

getId
```
 final Integer getId()
```

getUrls
```
 final Sequence<T> getUrls()
```
The url sequence

setUrls
```
 final Unit setUrls(Sequence<T> urls)
```
The url sequence

getDefaultOptions
```
 final LoadOptions getDefaultOptions()
```
The default load options

getGlobalCacheFactory
```
 final GlobalCacheFactory getGlobalCacheFactory()
```
A optional global cache which will hold the retry tasks

getCrawlEventHandler

 final CrawlEventHandler getCrawlEventHandler()

The crawl event handler

getName
```
 final String getName()
```
The crawl name

getClosed
```
 final AtomicBoolean getClosed()
```

getSession
```
 final PulsarSession getSession()
```

getAutoClose
```
 final Boolean getAutoClose()
```

run
```
 Unit run()
```

run
```
 Unit run(CoroutineScope scope)
```

quit
```
 final Unit quit()
```

Class StreamingCrawler

Nested Class Summary

Field Summary

Constructor Summary

Method Summary

Methods inherited from class ai.platon.pulsar.crawl.StreamingCrawler

Methods inherited from class java.lang.Object

Constructor Detail

StreamingCrawler

Method Detail

getNumPrivacyContexts

getNumMaxActiveTabs

getFetchConcurrency

getIdleTimeout

getIdleTime

getIsIdle

getDefaultArgs

getIsActive

getJobName

setJobName

getProxyPool

setProxyPool

getId

getUrls

setUrls

getDefaultOptions

getGlobalCacheFactory

getCrawlEventHandler

getName

getClosed

getSession

getAutoClose

run

run

quit