Package ai.pipestream.parsed.data.warc.v1

Interface WarcMetadataOrBuilder

All Superinterfaces:
com.google.protobuf.MessageLiteOrBuilder, com.google.protobuf.MessageOrBuilder
All Known Implementing Classes:
WarcMetadata, WarcMetadata.Builder
@Generated public interface WarcMetadataOrBuilder extends com.google.protobuf.MessageOrBuilder

  • Method Details

    • getWarcWarningsList

      List<String> getWarcWarningsList()
       WARC warnings encountered during parsing
 Multiple warnings can be present
 WARC.WARC_WARNING
      repeated string warc_warnings = 1 [json_name = "warcWarnings"];
      Returns:
      A list containing the warcWarnings.

    • getWarcWarningsCount

      int getWarcWarningsCount()
       WARC warnings encountered during parsing
 Multiple warnings can be present
 WARC.WARC_WARNING
      repeated string warc_warnings = 1 [json_name = "warcWarnings"];
      Returns:
      The count of warcWarnings.

    • getWarcWarnings

      String getWarcWarnings(int index)
       WARC warnings encountered during parsing
 Multiple warnings can be present
 WARC.WARC_WARNING
      repeated string warc_warnings = 1 [json_name = "warcWarnings"];
      Parameters:
      index - The index of the element to return.
      Returns:
      The warcWarnings at the given index.

    • getWarcWarningsBytes

      com.google.protobuf.ByteString getWarcWarningsBytes(int index)
       WARC warnings encountered during parsing
 Multiple warnings can be present
 WARC.WARC_WARNING
      repeated string warc_warnings = 1 [json_name = "warcWarnings"];
      Parameters:
      index - The index of the value to return.
      Returns:
      The bytes of the warcWarnings at the given index.

    • hasWarcRecordContentType

      boolean hasWarcRecordContentType()
       Content type of the WARC record itself
 e.g., "application/http; msgtype=response"
 WARC.WARC_RECORD_CONTENT_TYPE
      optional string warc_record_content_type = 2 [json_name = "warcRecordContentType"];
      Returns:
      Whether the warcRecordContentType field is set.

    • getWarcRecordContentType

      String getWarcRecordContentType()
       Content type of the WARC record itself
 e.g., "application/http; msgtype=response"
 WARC.WARC_RECORD_CONTENT_TYPE
      optional string warc_record_content_type = 2 [json_name = "warcRecordContentType"];
      Returns:
      The warcRecordContentType.

    • getWarcRecordContentTypeBytes

      com.google.protobuf.ByteString getWarcRecordContentTypeBytes()
       Content type of the WARC record itself
 e.g., "application/http; msgtype=response"
 WARC.WARC_RECORD_CONTENT_TYPE
      optional string warc_record_content_type = 2 [json_name = "warcRecordContentType"];
      Returns:
      The bytes for warcRecordContentType.

    • hasWarcPayloadContentType

      boolean hasWarcPayloadContentType()
       Content type of the payload within the WARC record
 e.g., "text/html", "image/jpeg"
 WARC.WARC_PAYLOAD_CONTENT_TYPE
      optional string warc_payload_content_type = 3 [json_name = "warcPayloadContentType"];
      Returns:
      Whether the warcPayloadContentType field is set.

    • getWarcPayloadContentType

      String getWarcPayloadContentType()
       Content type of the payload within the WARC record
 e.g., "text/html", "image/jpeg"
 WARC.WARC_PAYLOAD_CONTENT_TYPE
      optional string warc_payload_content_type = 3 [json_name = "warcPayloadContentType"];
      Returns:
      The warcPayloadContentType.

    • getWarcPayloadContentTypeBytes

      com.google.protobuf.ByteString getWarcPayloadContentTypeBytes()
       Content type of the payload within the WARC record
 e.g., "text/html", "image/jpeg"
 WARC.WARC_PAYLOAD_CONTENT_TYPE
      optional string warc_payload_content_type = 3 [json_name = "warcPayloadContentType"];
      Returns:
      The bytes for warcPayloadContentType.

    • hasWarcRecordId

      boolean hasWarcRecordId()
       Unique identifier for this WARC record
 Format: <urn:uuid:...> or similar URI
 WARC.WARC_RECORD_ID
      optional string warc_record_id = 4 [json_name = "warcRecordId"];
      Returns:
      Whether the warcRecordId field is set.

    • getWarcRecordId

      String getWarcRecordId()
       Unique identifier for this WARC record
 Format: <urn:uuid:...> or similar URI
 WARC.WARC_RECORD_ID
      optional string warc_record_id = 4 [json_name = "warcRecordId"];
      Returns:
      The warcRecordId.

    • getWarcRecordIdBytes

      com.google.protobuf.ByteString getWarcRecordIdBytes()
       Unique identifier for this WARC record
 Format: <urn:uuid:...> or similar URI
 WARC.WARC_RECORD_ID
      optional string warc_record_id = 4 [json_name = "warcRecordId"];
      Returns:
      The bytes for warcRecordId.

    • hasWarcType

      boolean hasWarcType()
       Type of WARC record (response, request, metadata, resource, etc.)
 WARC-Type header
      optional string warc_type = 5 [json_name = "warcType"];
      Returns:
      Whether the warcType field is set.

    • getWarcType

      String getWarcType()
       Type of WARC record (response, request, metadata, resource, etc.)
 WARC-Type header
      optional string warc_type = 5 [json_name = "warcType"];
      Returns:
      The warcType.

    • getWarcTypeBytes

      com.google.protobuf.ByteString getWarcTypeBytes()
       Type of WARC record (response, request, metadata, resource, etc.)
 WARC-Type header
      optional string warc_type = 5 [json_name = "warcType"];
      Returns:
      The bytes for warcType.

    • hasWarcTargetUri

      boolean hasWarcTargetUri()
       Target URI that was archived
 The original URL of the web resource
 WARC-Target-URI
      optional string warc_target_uri = 6 [json_name = "warcTargetUri"];
      Returns:
      Whether the warcTargetUri field is set.

    • getWarcTargetUri

      String getWarcTargetUri()
       Target URI that was archived
 The original URL of the web resource
 WARC-Target-URI
      optional string warc_target_uri = 6 [json_name = "warcTargetUri"];
      Returns:
      The warcTargetUri.

    • getWarcTargetUriBytes

      com.google.protobuf.ByteString getWarcTargetUriBytes()
       Target URI that was archived
 The original URL of the web resource
 WARC-Target-URI
      optional string warc_target_uri = 6 [json_name = "warcTargetUri"];
      Returns:
      The bytes for warcTargetUri.

    • hasWarcDate

      boolean hasWarcDate()
       Date and time when the resource was captured
 WARC-Date
      optional .google.protobuf.Timestamp warc_date = 7 [json_name = "warcDate"];
      Returns:
      Whether the warcDate field is set.

    • getWarcDate

      com.google.protobuf.Timestamp getWarcDate()
       Date and time when the resource was captured
 WARC-Date
      optional .google.protobuf.Timestamp warc_date = 7 [json_name = "warcDate"];
      Returns:
      The warcDate.

    • getWarcDateOrBuilder

      com.google.protobuf.TimestampOrBuilder getWarcDateOrBuilder()
       Date and time when the resource was captured
 WARC-Date
      optional .google.protobuf.Timestamp warc_date = 7 [json_name = "warcDate"];

    • hasWarcContentLength

      boolean hasWarcContentLength()
       Length of the record payload in bytes
 Content-Length
      optional int64 warc_content_length = 8 [json_name = "warcContentLength"];
      Returns:
      Whether the warcContentLength field is set.

    • getWarcContentLength

      long getWarcContentLength()
       Length of the record payload in bytes
 Content-Length
      optional int64 warc_content_length = 8 [json_name = "warcContentLength"];
      Returns:
      The warcContentLength.

    • hasWarcFilename

      boolean hasWarcFilename()
       WARC filename where this record is stored
 WARC-Filename
      optional string warc_filename = 9 [json_name = "warcFilename"];
      Returns:
      Whether the warcFilename field is set.

    • getWarcFilename

      String getWarcFilename()
       WARC filename where this record is stored
 WARC-Filename
      optional string warc_filename = 9 [json_name = "warcFilename"];
      Returns:
      The warcFilename.

    • getWarcFilenameBytes

      com.google.protobuf.ByteString getWarcFilenameBytes()
       WARC filename where this record is stored
 WARC-Filename
      optional string warc_filename = 9 [json_name = "warcFilename"];
      Returns:
      The bytes for warcFilename.

    • hasWarcRefersTo

      boolean hasWarcRefersTo()
       WARC record ID that this record refers to
 Used for linking related records
 WARC-Refers-To
      optional string warc_refers_to = 10 [json_name = "warcRefersTo"];
      Returns:
      Whether the warcRefersTo field is set.

    • getWarcRefersTo

      String getWarcRefersTo()
       WARC record ID that this record refers to
 Used for linking related records
 WARC-Refers-To
      optional string warc_refers_to = 10 [json_name = "warcRefersTo"];
      Returns:
      The warcRefersTo.

    • getWarcRefersToBytes

      com.google.protobuf.ByteString getWarcRefersToBytes()
       WARC record ID that this record refers to
 Used for linking related records
 WARC-Refers-To
      optional string warc_refers_to = 10 [json_name = "warcRefersTo"];
      Returns:
      The bytes for warcRefersTo.

    • hasWarcConcurrentTo

      boolean hasWarcConcurrentTo()
       WARC record ID that this record is concurrent with
 WARC-Concurrent-To
      optional string warc_concurrent_to = 11 [json_name = "warcConcurrentTo"];
      Returns:
      Whether the warcConcurrentTo field is set.

    • getWarcConcurrentTo

      String getWarcConcurrentTo()
       WARC record ID that this record is concurrent with
 WARC-Concurrent-To
      optional string warc_concurrent_to = 11 [json_name = "warcConcurrentTo"];
      Returns:
      The warcConcurrentTo.

    • getWarcConcurrentToBytes

      com.google.protobuf.ByteString getWarcConcurrentToBytes()
       WARC record ID that this record is concurrent with
 WARC-Concurrent-To
      optional string warc_concurrent_to = 11 [json_name = "warcConcurrentTo"];
      Returns:
      The bytes for warcConcurrentTo.

    • hasWarcWarcinfoId

      boolean hasWarcWarcinfoId()
       WARC record ID of the warcinfo record for this crawl
 WARC-Warcinfo-ID
      optional string warc_warcinfo_id = 12 [json_name = "warcWarcinfoId"];
      Returns:
      Whether the warcWarcinfoId field is set.

    • getWarcWarcinfoId

      String getWarcWarcinfoId()
       WARC record ID of the warcinfo record for this crawl
 WARC-Warcinfo-ID
      optional string warc_warcinfo_id = 12 [json_name = "warcWarcinfoId"];
      Returns:
      The warcWarcinfoId.

    • getWarcWarcinfoIdBytes

      com.google.protobuf.ByteString getWarcWarcinfoIdBytes()
       WARC record ID of the warcinfo record for this crawl
 WARC-Warcinfo-ID
      optional string warc_warcinfo_id = 12 [json_name = "warcWarcinfoId"];
      Returns:
      The bytes for warcWarcinfoId.

    • hasWarcIpAddress

      boolean hasWarcIpAddress()
       IP address of the server that provided this resource
 WARC-IP-Address
      optional string warc_ip_address = 13 [json_name = "warcIpAddress"];
      Returns:
      Whether the warcIpAddress field is set.

    • getWarcIpAddress

      String getWarcIpAddress()
       IP address of the server that provided this resource
 WARC-IP-Address
      optional string warc_ip_address = 13 [json_name = "warcIpAddress"];
      Returns:
      The warcIpAddress.

    • getWarcIpAddressBytes

      com.google.protobuf.ByteString getWarcIpAddressBytes()
       IP address of the server that provided this resource
 WARC-IP-Address
      optional string warc_ip_address = 13 [json_name = "warcIpAddress"];
      Returns:
      The bytes for warcIpAddress.

    • hasWarcBlockDigest

      boolean hasWarcBlockDigest()
       Block digest (hash) of the payload
 WARC-Block-Digest
      optional string warc_block_digest = 14 [json_name = "warcBlockDigest"];
      Returns:
      Whether the warcBlockDigest field is set.

    • getWarcBlockDigest

      String getWarcBlockDigest()
       Block digest (hash) of the payload
 WARC-Block-Digest
      optional string warc_block_digest = 14 [json_name = "warcBlockDigest"];
      Returns:
      The warcBlockDigest.

    • getWarcBlockDigestBytes

      com.google.protobuf.ByteString getWarcBlockDigestBytes()
       Block digest (hash) of the payload
 WARC-Block-Digest
      optional string warc_block_digest = 14 [json_name = "warcBlockDigest"];
      Returns:
      The bytes for warcBlockDigest.

    • hasWarcPayloadDigest

      boolean hasWarcPayloadDigest()
       Payload digest (hash) of the actual content
 WARC-Payload-Digest
      optional string warc_payload_digest = 15 [json_name = "warcPayloadDigest"];
      Returns:
      Whether the warcPayloadDigest field is set.

    • getWarcPayloadDigest

      String getWarcPayloadDigest()
       Payload digest (hash) of the actual content
 WARC-Payload-Digest
      optional string warc_payload_digest = 15 [json_name = "warcPayloadDigest"];
      Returns:
      The warcPayloadDigest.

    • getWarcPayloadDigestBytes

      com.google.protobuf.ByteString getWarcPayloadDigestBytes()
       Payload digest (hash) of the actual content
 WARC-Payload-Digest
      optional string warc_payload_digest = 15 [json_name = "warcPayloadDigest"];
      Returns:
      The bytes for warcPayloadDigest.

    • hasWarcTruncated

      boolean hasWarcTruncated()
       Truncation reason if record was truncated
 WARC-Truncated
      optional string warc_truncated = 16 [json_name = "warcTruncated"];
      Returns:
      Whether the warcTruncated field is set.

    • getWarcTruncated

      String getWarcTruncated()
       Truncation reason if record was truncated
 WARC-Truncated
      optional string warc_truncated = 16 [json_name = "warcTruncated"];
      Returns:
      The warcTruncated.

    • getWarcTruncatedBytes

      com.google.protobuf.ByteString getWarcTruncatedBytes()
       Truncation reason if record was truncated
 WARC-Truncated
      optional string warc_truncated = 16 [json_name = "warcTruncated"];
      Returns:
      The bytes for warcTruncated.

    • hasWarcIdentifiedPayloadType

      boolean hasWarcIdentifiedPayloadType()
       Identified payload type (may differ from Content-Type)
 WARC-Identified-Payload-Type
      optional string warc_identified_payload_type = 17 [json_name = "warcIdentifiedPayloadType"];
      Returns:
      Whether the warcIdentifiedPayloadType field is set.

    • getWarcIdentifiedPayloadType

      String getWarcIdentifiedPayloadType()
       Identified payload type (may differ from Content-Type)
 WARC-Identified-Payload-Type
      optional string warc_identified_payload_type = 17 [json_name = "warcIdentifiedPayloadType"];
      Returns:
      The warcIdentifiedPayloadType.

    • getWarcIdentifiedPayloadTypeBytes

      com.google.protobuf.ByteString getWarcIdentifiedPayloadTypeBytes()
       Identified payload type (may differ from Content-Type)
 WARC-Identified-Payload-Type
      optional string warc_identified_payload_type = 17 [json_name = "warcIdentifiedPayloadType"];
      Returns:
      The bytes for warcIdentifiedPayloadType.

    • hasHttpStatusCode

      boolean hasHttpStatusCode()
       HTTP status code (200, 404, 500, etc.)
 HTTP status
      optional int32 http_status_code = 18 [json_name = "httpStatusCode"];
      Returns:
      Whether the httpStatusCode field is set.

    • getHttpStatusCode

      int getHttpStatusCode()
       HTTP status code (200, 404, 500, etc.)
 HTTP status
      optional int32 http_status_code = 18 [json_name = "httpStatusCode"];
      Returns:
      The httpStatusCode.

    • hasHttpStatusReason

      boolean hasHttpStatusReason()
       HTTP status reason phrase ("OK", "Not Found", etc.)
 HTTP reason phrase
      optional string http_status_reason = 19 [json_name = "httpStatusReason"];
      Returns:
      Whether the httpStatusReason field is set.

    • getHttpStatusReason

      String getHttpStatusReason()
       HTTP status reason phrase ("OK", "Not Found", etc.)
 HTTP reason phrase
      optional string http_status_reason = 19 [json_name = "httpStatusReason"];
      Returns:
      The httpStatusReason.

    • getHttpStatusReasonBytes

      com.google.protobuf.ByteString getHttpStatusReasonBytes()
       HTTP status reason phrase ("OK", "Not Found", etc.)
 HTTP reason phrase
      optional string http_status_reason = 19 [json_name = "httpStatusReason"];
      Returns:
      The bytes for httpStatusReason.

    • getHttpHeadersList

      List<WarcHttpHeader> getHttpHeadersList()
       HTTP response headers
      repeated .ai.pipestream.parsed.data.warc.v1.WarcHttpHeader http_headers = 20 [json_name = "httpHeaders"];

    • getHttpHeaders

      WarcHttpHeader getHttpHeaders(int index)
       HTTP response headers
      repeated .ai.pipestream.parsed.data.warc.v1.WarcHttpHeader http_headers = 20 [json_name = "httpHeaders"];

    • getHttpHeadersCount

      int getHttpHeadersCount()
       HTTP response headers
      repeated .ai.pipestream.parsed.data.warc.v1.WarcHttpHeader http_headers = 20 [json_name = "httpHeaders"];

    • getHttpHeadersOrBuilderList

      List<? extends WarcHttpHeaderOrBuilder> getHttpHeadersOrBuilderList()
       HTTP response headers
      repeated .ai.pipestream.parsed.data.warc.v1.WarcHttpHeader http_headers = 20 [json_name = "httpHeaders"];

    • getHttpHeadersOrBuilder

      WarcHttpHeaderOrBuilder getHttpHeadersOrBuilder(int index)
       HTTP response headers
      repeated .ai.pipestream.parsed.data.warc.v1.WarcHttpHeader http_headers = 20 [json_name = "httpHeaders"];

    • hasWarcCreatedBy

      boolean hasWarcCreatedBy()
       Software/crawler that created this WARC file
 Software info
      optional string warc_created_by = 21 [json_name = "warcCreatedBy"];
      Returns:
      Whether the warcCreatedBy field is set.

    • getWarcCreatedBy

      String getWarcCreatedBy()
       Software/crawler that created this WARC file
 Software info
      optional string warc_created_by = 21 [json_name = "warcCreatedBy"];
      Returns:
      The warcCreatedBy.

    • getWarcCreatedByBytes

      com.google.protobuf.ByteString getWarcCreatedByBytes()
       Software/crawler that created this WARC file
 Software info
      optional string warc_created_by = 21 [json_name = "warcCreatedBy"];
      Returns:
      The bytes for warcCreatedBy.

    • hasWarcFormatVersion

      boolean hasWarcFormatVersion()
       Version of the WARC format specification used
 WARC format version
      optional string warc_format_version = 22 [json_name = "warcFormatVersion"];
      Returns:
      Whether the warcFormatVersion field is set.

    • getWarcFormatVersion

      String getWarcFormatVersion()
       Version of the WARC format specification used
 WARC format version
      optional string warc_format_version = 22 [json_name = "warcFormatVersion"];
      Returns:
      The warcFormatVersion.

    • getWarcFormatVersionBytes

      com.google.protobuf.ByteString getWarcFormatVersionBytes()
       Version of the WARC format specification used
 WARC format version
      optional string warc_format_version = 22 [json_name = "warcFormatVersion"];
      Returns:
      The bytes for warcFormatVersion.

    • hasWarcCollection

      boolean hasWarcCollection()
       Crawl or collection identifier
 Collection name
      optional string warc_collection = 23 [json_name = "warcCollection"];
      Returns:
      Whether the warcCollection field is set.

    • getWarcCollection

      String getWarcCollection()
       Crawl or collection identifier
 Collection name
      optional string warc_collection = 23 [json_name = "warcCollection"];
      Returns:
      The warcCollection.

    • getWarcCollectionBytes

      com.google.protobuf.ByteString getWarcCollectionBytes()
       Crawl or collection identifier
 Collection name
      optional string warc_collection = 23 [json_name = "warcCollection"];
      Returns:
      The bytes for warcCollection.

    • hasWarcCrawlId

      boolean hasWarcCrawlId()
       Crawl job or session identifier
 Crawl identifier
      optional string warc_crawl_id = 24 [json_name = "warcCrawlId"];
      Returns:
      Whether the warcCrawlId field is set.

    • getWarcCrawlId

      String getWarcCrawlId()
       Crawl job or session identifier
 Crawl identifier
      optional string warc_crawl_id = 24 [json_name = "warcCrawlId"];
      Returns:
      The warcCrawlId.

    • getWarcCrawlIdBytes

      com.google.protobuf.ByteString getWarcCrawlIdBytes()
       Crawl job or session identifier
 Crawl identifier
      optional string warc_crawl_id = 24 [json_name = "warcCrawlId"];
      Returns:
      The bytes for warcCrawlId.

    • hasWarcRobotPolicy

      boolean hasWarcRobotPolicy()
       Robot policy information (robots.txt compliance)
 Robot policy
      optional string warc_robot_policy = 25 [json_name = "warcRobotPolicy"];
      Returns:
      Whether the warcRobotPolicy field is set.

    • getWarcRobotPolicy

      String getWarcRobotPolicy()
       Robot policy information (robots.txt compliance)
 Robot policy
      optional string warc_robot_policy = 25 [json_name = "warcRobotPolicy"];
      Returns:
      The warcRobotPolicy.

    • getWarcRobotPolicyBytes

      com.google.protobuf.ByteString getWarcRobotPolicyBytes()
       Robot policy information (robots.txt compliance)
 Robot policy
      optional string warc_robot_policy = 25 [json_name = "warcRobotPolicy"];
      Returns:
      The bytes for warcRobotPolicy.

    • hasFetchTimestamp

      boolean hasFetchTimestamp()
       Fetch timestamp (may differ from WARC-Date)
      optional .google.protobuf.Timestamp fetch_timestamp = 26 [json_name = "fetchTimestamp"];
      Returns:
      Whether the fetchTimestamp field is set.

    • getFetchTimestamp

      com.google.protobuf.Timestamp getFetchTimestamp()
       Fetch timestamp (may differ from WARC-Date)
      optional .google.protobuf.Timestamp fetch_timestamp = 26 [json_name = "fetchTimestamp"];
      Returns:
      The fetchTimestamp.

    • getFetchTimestampOrBuilder

      com.google.protobuf.TimestampOrBuilder getFetchTimestampOrBuilder()
       Fetch timestamp (may differ from WARC-Date)
      optional .google.protobuf.Timestamp fetch_timestamp = 26 [json_name = "fetchTimestamp"];

    • hasContentLanguage

      boolean hasContentLanguage()
       Detected language of the content
      optional string content_language = 27 [json_name = "contentLanguage"];
      Returns:
      Whether the contentLanguage field is set.

    • getContentLanguage

      String getContentLanguage()
       Detected language of the content
      optional string content_language = 27 [json_name = "contentLanguage"];
      Returns:
      The contentLanguage.

    • getContentLanguageBytes

      com.google.protobuf.ByteString getContentLanguageBytes()
       Detected language of the content
      optional string content_language = 27 [json_name = "contentLanguage"];
      Returns:
      The bytes for contentLanguage.

    • hasContentEncoding

      boolean hasContentEncoding()
       Character encoding of the content
      optional string content_encoding = 28 [json_name = "contentEncoding"];
      Returns:
      Whether the contentEncoding field is set.

    • getContentEncoding

      String getContentEncoding()
       Character encoding of the content
      optional string content_encoding = 28 [json_name = "contentEncoding"];
      Returns:
      The contentEncoding.

    • getContentEncodingBytes

      com.google.protobuf.ByteString getContentEncodingBytes()
       Character encoding of the content
      optional string content_encoding = 28 [json_name = "contentEncoding"];
      Returns:
      The bytes for contentEncoding.

    • hasEstimatedWordCount

      boolean hasEstimatedWordCount()
       Estimated word count of textual content
      optional int64 estimated_word_count = 29 [json_name = "estimatedWordCount"];
      Returns:
      Whether the estimatedWordCount field is set.

    • getEstimatedWordCount

      long getEstimatedWordCount()
       Estimated word count of textual content
      optional int64 estimated_word_count = 29 [json_name = "estimatedWordCount"];
      Returns:
      The estimatedWordCount.

    • hasEstimatedCharacterCount

      boolean hasEstimatedCharacterCount()
       Estimated character count of textual content
      optional int64 estimated_character_count = 30 [json_name = "estimatedCharacterCount"];
      Returns:
      Whether the estimatedCharacterCount field is set.

    • getEstimatedCharacterCount

      long getEstimatedCharacterCount()
       Estimated character count of textual content
      optional int64 estimated_character_count = 30 [json_name = "estimatedCharacterCount"];
      Returns:
      The estimatedCharacterCount.

    • hasIsTruncated

      boolean hasIsTruncated()
       Whether the content appears to be truncated
      optional bool is_truncated = 31 [json_name = "isTruncated"];
      Returns:
      Whether the isTruncated field is set.

    • getIsTruncated

      boolean getIsTruncated()
       Whether the content appears to be truncated
      optional bool is_truncated = 31 [json_name = "isTruncated"];
      Returns:
      The isTruncated.

    • hasCompressionMethod

      boolean hasCompressionMethod()
       Compression method used (if any)
      optional string compression_method = 32 [json_name = "compressionMethod"];
      Returns:
      Whether the compressionMethod field is set.

    • getCompressionMethod

      String getCompressionMethod()
       Compression method used (if any)
      optional string compression_method = 32 [json_name = "compressionMethod"];
      Returns:
      The compressionMethod.

    • getCompressionMethodBytes

      com.google.protobuf.ByteString getCompressionMethodBytes()
       Compression method used (if any)
      optional string compression_method = 32 [json_name = "compressionMethod"];
      Returns:
      The bytes for compressionMethod.

    • hasHttpMethod

      boolean hasHttpMethod()
       HTTP method used for the request (GET, POST, etc.)
      optional string http_method = 33 [json_name = "httpMethod"];
      Returns:
      Whether the httpMethod field is set.

    • getHttpMethod

      String getHttpMethod()
       HTTP method used for the request (GET, POST, etc.)
      optional string http_method = 33 [json_name = "httpMethod"];
      Returns:
      The httpMethod.

    • getHttpMethodBytes

      com.google.protobuf.ByteString getHttpMethodBytes()
       HTTP method used for the request (GET, POST, etc.)
      optional string http_method = 33 [json_name = "httpMethod"];
      Returns:
      The bytes for httpMethod.

    • hasHttpVersion

      boolean hasHttpVersion()
       HTTP version (1.0, 1.1, 2.0)
      optional string http_version = 34 [json_name = "httpVersion"];
      Returns:
      Whether the httpVersion field is set.

    • getHttpVersion

      String getHttpVersion()
       HTTP version (1.0, 1.1, 2.0)
      optional string http_version = 34 [json_name = "httpVersion"];
      Returns:
      The httpVersion.

    • getHttpVersionBytes

      com.google.protobuf.ByteString getHttpVersionBytes()
       HTTP version (1.0, 1.1, 2.0)
      optional string http_version = 34 [json_name = "httpVersion"];
      Returns:
      The bytes for httpVersion.

    • getRedirectChainList

      List<WarcRedirect> getRedirectChainList()
       Redirect chain information
      repeated .ai.pipestream.parsed.data.warc.v1.WarcRedirect redirect_chain = 35 [json_name = "redirectChain"];

    • getRedirectChain

      WarcRedirect getRedirectChain(int index)
       Redirect chain information
      repeated .ai.pipestream.parsed.data.warc.v1.WarcRedirect redirect_chain = 35 [json_name = "redirectChain"];

    • getRedirectChainCount

      int getRedirectChainCount()
       Redirect chain information
      repeated .ai.pipestream.parsed.data.warc.v1.WarcRedirect redirect_chain = 35 [json_name = "redirectChain"];

    • getRedirectChainOrBuilderList

      List<? extends WarcRedirectOrBuilder> getRedirectChainOrBuilderList()
       Redirect chain information
      repeated .ai.pipestream.parsed.data.warc.v1.WarcRedirect redirect_chain = 35 [json_name = "redirectChain"];

    • getRedirectChainOrBuilder

      WarcRedirectOrBuilder getRedirectChainOrBuilder(int index)
       Redirect chain information
      repeated .ai.pipestream.parsed.data.warc.v1.WarcRedirect redirect_chain = 35 [json_name = "redirectChain"];

    • getCookiesList

      List<WarcCookie> getCookiesList()
       Cookies associated with this request/response
      repeated .ai.pipestream.parsed.data.warc.v1.WarcCookie cookies = 36 [json_name = "cookies"];

    • getCookies

      WarcCookie getCookies(int index)
       Cookies associated with this request/response
      repeated .ai.pipestream.parsed.data.warc.v1.WarcCookie cookies = 36 [json_name = "cookies"];

    • getCookiesCount

      int getCookiesCount()
       Cookies associated with this request/response
      repeated .ai.pipestream.parsed.data.warc.v1.WarcCookie cookies = 36 [json_name = "cookies"];

    • getCookiesOrBuilderList

      List<? extends WarcCookieOrBuilder> getCookiesOrBuilderList()
       Cookies associated with this request/response
      repeated .ai.pipestream.parsed.data.warc.v1.WarcCookie cookies = 36 [json_name = "cookies"];

    • getCookiesOrBuilder

      WarcCookieOrBuilder getCookiesOrBuilder(int index)
       Cookies associated with this request/response
      repeated .ai.pipestream.parsed.data.warc.v1.WarcCookie cookies = 36 [json_name = "cookies"];

    • hasReferrerUrl

      boolean hasReferrerUrl()
       Referrer URL (if available)
      optional string referrer_url = 37 [json_name = "referrerUrl"];
      Returns:
      Whether the referrerUrl field is set.

    • getReferrerUrl

      String getReferrerUrl()
       Referrer URL (if available)
      optional string referrer_url = 37 [json_name = "referrerUrl"];
      Returns:
      The referrerUrl.

    • getReferrerUrlBytes

      com.google.protobuf.ByteString getReferrerUrlBytes()
       Referrer URL (if available)
      optional string referrer_url = 37 [json_name = "referrerUrl"];
      Returns:
      The bytes for referrerUrl.

    • hasUserAgent

      boolean hasUserAgent()
       User agent string used for the request
      optional string user_agent = 38 [json_name = "userAgent"];
      Returns:
      Whether the userAgent field is set.

    • getUserAgent

      String getUserAgent()
       User agent string used for the request
      optional string user_agent = 38 [json_name = "userAgent"];
      Returns:
      The userAgent.

    • getUserAgentBytes

      com.google.protobuf.ByteString getUserAgentBytes()
       User agent string used for the request
      optional string user_agent = 38 [json_name = "userAgent"];
      Returns:
      The bytes for userAgent.

    • hasRecordPosition

      boolean hasRecordPosition()
       Position of this record within the WARC file
      optional int64 record_position = 39 [json_name = "recordPosition"];
      Returns:
      Whether the recordPosition field is set.

    • getRecordPosition

      long getRecordPosition()
       Position of this record within the WARC file
      optional int64 record_position = 39 [json_name = "recordPosition"];
      Returns:
      The recordPosition.

    • hasRecordSize

      boolean hasRecordSize()
       Size of the complete WARC record (headers + payload)
      optional int64 record_size = 40 [json_name = "recordSize"];
      Returns:
      Whether the recordSize field is set.

    • getRecordSize

      long getRecordSize()
       Size of the complete WARC record (headers + payload)
      optional int64 record_size = 40 [json_name = "recordSize"];
      Returns:
      The recordSize.

    • hasTotalRecordsInFile

      boolean hasTotalRecordsInFile()
       Number of records in the WARC file
      optional int32 total_records_in_file = 41 [json_name = "totalRecordsInFile"];
      Returns:
      Whether the totalRecordsInFile field is set.

    • getTotalRecordsInFile

      int getTotalRecordsInFile()
       Number of records in the WARC file
      optional int32 total_records_in_file = 41 [json_name = "totalRecordsInFile"];
      Returns:
      The totalRecordsInFile.

    • hasRecordIndex

      boolean hasRecordIndex()
       Index of this record within the file (0-based)
      optional int32 record_index = 42 [json_name = "recordIndex"];
      Returns:
      Whether the recordIndex field is set.

    • getRecordIndex

      int getRecordIndex()
       Index of this record within the file (0-based)
      optional int32 record_index = 42 [json_name = "recordIndex"];
      Returns:
      The recordIndex.

    • hasHasMetadataRecords

      boolean hasHasMetadataRecords()
       Whether this record has associated metadata records
      optional bool has_metadata_records = 43 [json_name = "hasMetadataRecords"];
      Returns:
      Whether the hasMetadataRecords field is set.

    • getHasMetadataRecords

      boolean getHasMetadataRecords()
       Whether this record has associated metadata records
      optional bool has_metadata_records = 43 [json_name = "hasMetadataRecords"];
      Returns:
      The hasMetadataRecords.

    • hasHasRequestRecords

      boolean hasHasRequestRecords()
       Whether this record has associated request records
      optional bool has_request_records = 44 [json_name = "hasRequestRecords"];
      Returns:
      Whether the hasRequestRecords field is set.

    • getHasRequestRecords

      boolean getHasRequestRecords()
       Whether this record has associated request records
      optional bool has_request_records = 44 [json_name = "hasRequestRecords"];
      Returns:
      The hasRequestRecords.

    • getValidationErrorsList

      List<String> getValidationErrorsList()
       Validation errors found in the WARC record
      repeated string validation_errors = 45 [json_name = "validationErrors"];
      Returns:
      A list containing the validationErrors.

    • getValidationErrorsCount

      int getValidationErrorsCount()
       Validation errors found in the WARC record
      repeated string validation_errors = 45 [json_name = "validationErrors"];
      Returns:
      The count of validationErrors.

    • getValidationErrors

      String getValidationErrors(int index)
       Validation errors found in the WARC record
      repeated string validation_errors = 45 [json_name = "validationErrors"];
      Parameters:
      index - The index of the element to return.
      Returns:
      The validationErrors at the given index.

    • getValidationErrorsBytes

      com.google.protobuf.ByteString getValidationErrorsBytes(int index)
       Validation errors found in the WARC record
      repeated string validation_errors = 45 [json_name = "validationErrors"];
      Parameters:
      index - The index of the value to return.
      Returns:
      The bytes of the validationErrors at the given index.

    • getValidationWarningsList

      List<String> getValidationWarningsList()
       Validation warnings found in the WARC record
      repeated string validation_warnings = 46 [json_name = "validationWarnings"];
      Returns:
      A list containing the validationWarnings.

    • getValidationWarningsCount

      int getValidationWarningsCount()
       Validation warnings found in the WARC record
      repeated string validation_warnings = 46 [json_name = "validationWarnings"];
      Returns:
      The count of validationWarnings.

    • getValidationWarnings

      String getValidationWarnings(int index)
       Validation warnings found in the WARC record
      repeated string validation_warnings = 46 [json_name = "validationWarnings"];
      Parameters:
      index - The index of the element to return.
      Returns:
      The validationWarnings at the given index.

    • getValidationWarningsBytes

      com.google.protobuf.ByteString getValidationWarningsBytes(int index)
       Validation warnings found in the WARC record
      repeated string validation_warnings = 46 [json_name = "validationWarnings"];
      Parameters:
      index - The index of the value to return.
      Returns:
      The bytes of the validationWarnings at the given index.

    • hasCompletenessScore

      boolean hasCompletenessScore()
       Completeness score (0.0 to 1.0) indicating how complete the capture is
      optional double completeness_score = 47 [json_name = "completenessScore"];
      Returns:
      Whether the completenessScore field is set.

    • getCompletenessScore

      double getCompletenessScore()
       Completeness score (0.0 to 1.0) indicating how complete the capture is
      optional double completeness_score = 47 [json_name = "completenessScore"];
      Returns:
      The completenessScore.

    • hasChecksumVerified

      boolean hasChecksumVerified()
       Whether the content matches expected checksums
      optional bool checksum_verified = 48 [json_name = "checksumVerified"];
      Returns:
      Whether the checksumVerified field is set.

    • getChecksumVerified

      boolean getChecksumVerified()
       Whether the content matches expected checksums
      optional bool checksum_verified = 48 [json_name = "checksumVerified"];
      Returns:
      The checksumVerified.

    • hasArchiveInstitution

      boolean hasArchiveInstitution()
       Institution or organization that created the archive
      optional string archive_institution = 49 [json_name = "archiveInstitution"];
      Returns:
      Whether the archiveInstitution field is set.

    • getArchiveInstitution

      String getArchiveInstitution()
       Institution or organization that created the archive
      optional string archive_institution = 49 [json_name = "archiveInstitution"];
      Returns:
      The archiveInstitution.

    • getArchiveInstitutionBytes

      com.google.protobuf.ByteString getArchiveInstitutionBytes()
       Institution or organization that created the archive
      optional string archive_institution = 49 [json_name = "archiveInstitution"];
      Returns:
      The bytes for archiveInstitution.

    • hasArchiveCollectionName

      boolean hasArchiveCollectionName()
       Archive collection or series name
      optional string archive_collection_name = 50 [json_name = "archiveCollectionName"];
      Returns:
      Whether the archiveCollectionName field is set.

    • getArchiveCollectionName

      String getArchiveCollectionName()
       Archive collection or series name
      optional string archive_collection_name = 50 [json_name = "archiveCollectionName"];
      Returns:
      The archiveCollectionName.

    • getArchiveCollectionNameBytes

      com.google.protobuf.ByteString getArchiveCollectionNameBytes()
       Archive collection or series name
      optional string archive_collection_name = 50 [json_name = "archiveCollectionName"];
      Returns:
      The bytes for archiveCollectionName.

    • hasAccessRights

      boolean hasAccessRights()
       Access restrictions or rights information
      optional string access_rights = 51 [json_name = "accessRights"];
      Returns:
      Whether the accessRights field is set.

    • getAccessRights

      String getAccessRights()
       Access restrictions or rights information
      optional string access_rights = 51 [json_name = "accessRights"];
      Returns:
      The accessRights.

    • getAccessRightsBytes

      com.google.protobuf.ByteString getAccessRightsBytes()
       Access restrictions or rights information
      optional string access_rights = 51 [json_name = "accessRights"];
      Returns:
      The bytes for accessRights.

    • getPreservationActionsList

      List<String> getPreservationActionsList()
       Preservation actions taken on this record
      repeated string preservation_actions = 52 [json_name = "preservationActions"];
      Returns:
      A list containing the preservationActions.

    • getPreservationActionsCount

      int getPreservationActionsCount()
       Preservation actions taken on this record
      repeated string preservation_actions = 52 [json_name = "preservationActions"];
      Returns:
      The count of preservationActions.

    • getPreservationActions

      String getPreservationActions(int index)
       Preservation actions taken on this record
      repeated string preservation_actions = 52 [json_name = "preservationActions"];
      Parameters:
      index - The index of the element to return.
      Returns:
      The preservationActions at the given index.

    • getPreservationActionsBytes

      com.google.protobuf.ByteString getPreservationActionsBytes(int index)
       Preservation actions taken on this record
      repeated string preservation_actions = 52 [json_name = "preservationActions"];
      Parameters:
      index - The index of the value to return.
      Returns:
      The bytes of the preservationActions at the given index.

    • getMigrationHistoryList

      List<WarcMigration> getMigrationHistoryList()
       Migration history if the record has been migrated
      repeated .ai.pipestream.parsed.data.warc.v1.WarcMigration migration_history = 53 [json_name = "migrationHistory"];

    • getMigrationHistory

      WarcMigration getMigrationHistory(int index)
       Migration history if the record has been migrated
      repeated .ai.pipestream.parsed.data.warc.v1.WarcMigration migration_history = 53 [json_name = "migrationHistory"];

    • getMigrationHistoryCount

      int getMigrationHistoryCount()
       Migration history if the record has been migrated
      repeated .ai.pipestream.parsed.data.warc.v1.WarcMigration migration_history = 53 [json_name = "migrationHistory"];

    • getMigrationHistoryOrBuilderList

      List<? extends WarcMigrationOrBuilder> getMigrationHistoryOrBuilderList()
       Migration history if the record has been migrated
      repeated .ai.pipestream.parsed.data.warc.v1.WarcMigration migration_history = 53 [json_name = "migrationHistory"];

    • getMigrationHistoryOrBuilder

      WarcMigrationOrBuilder getMigrationHistoryOrBuilder(int index)
       Migration history if the record has been migrated
      repeated .ai.pipestream.parsed.data.warc.v1.WarcMigration migration_history = 53 [json_name = "migrationHistory"];

    • hasDetectedMimeType

      boolean hasDetectedMimeType()
       MIME type as detected by content analysis
      optional string detected_mime_type = 54 [json_name = "detectedMimeType"];
      Returns:
      Whether the detectedMimeType field is set.

    • getDetectedMimeType

      String getDetectedMimeType()
       MIME type as detected by content analysis
      optional string detected_mime_type = 54 [json_name = "detectedMimeType"];
      Returns:
      The detectedMimeType.

    • getDetectedMimeTypeBytes

      com.google.protobuf.ByteString getDetectedMimeTypeBytes()
       MIME type as detected by content analysis
      optional string detected_mime_type = 54 [json_name = "detectedMimeType"];
      Returns:
      The bytes for detectedMimeType.

    • hasFormatVersion

      boolean hasFormatVersion()
       File format version (if applicable)
      optional string format_version = 55 [json_name = "formatVersion"];
      Returns:
      Whether the formatVersion field is set.

    • getFormatVersion

      String getFormatVersion()
       File format version (if applicable)
      optional string format_version = 55 [json_name = "formatVersion"];
      Returns:
      The formatVersion.

    • getFormatVersionBytes

      com.google.protobuf.ByteString getFormatVersionBytes()
       File format version (if applicable)
      optional string format_version = 55 [json_name = "formatVersion"];
      Returns:
      The bytes for formatVersion.

    • getTechnicalPropertiesList

      List<WarcTechnicalProperty> getTechnicalPropertiesList()
       Technical properties specific to the content type
      repeated .ai.pipestream.parsed.data.warc.v1.WarcTechnicalProperty technical_properties = 56 [json_name = "technicalProperties"];

    • getTechnicalProperties

      WarcTechnicalProperty getTechnicalProperties(int index)
       Technical properties specific to the content type
      repeated .ai.pipestream.parsed.data.warc.v1.WarcTechnicalProperty technical_properties = 56 [json_name = "technicalProperties"];

    • getTechnicalPropertiesCount

      int getTechnicalPropertiesCount()
       Technical properties specific to the content type
      repeated .ai.pipestream.parsed.data.warc.v1.WarcTechnicalProperty technical_properties = 56 [json_name = "technicalProperties"];

    • getTechnicalPropertiesOrBuilderList

      List<? extends WarcTechnicalPropertyOrBuilder> getTechnicalPropertiesOrBuilderList()
       Technical properties specific to the content type
      repeated .ai.pipestream.parsed.data.warc.v1.WarcTechnicalProperty technical_properties = 56 [json_name = "technicalProperties"];

    • getTechnicalPropertiesOrBuilder

      WarcTechnicalPropertyOrBuilder getTechnicalPropertiesOrBuilder(int index)
       Technical properties specific to the content type
      repeated .ai.pipestream.parsed.data.warc.v1.WarcTechnicalProperty technical_properties = 56 [json_name = "technicalProperties"];

    • getEmbeddedResourcesList

      List<WarcEmbeddedResource> getEmbeddedResourcesList()
       Embedded resources found within this record
      repeated .ai.pipestream.parsed.data.warc.v1.WarcEmbeddedResource embedded_resources = 57 [json_name = "embeddedResources"];

    • getEmbeddedResources

      WarcEmbeddedResource getEmbeddedResources(int index)
       Embedded resources found within this record
      repeated .ai.pipestream.parsed.data.warc.v1.WarcEmbeddedResource embedded_resources = 57 [json_name = "embeddedResources"];

    • getEmbeddedResourcesCount

      int getEmbeddedResourcesCount()
       Embedded resources found within this record
      repeated .ai.pipestream.parsed.data.warc.v1.WarcEmbeddedResource embedded_resources = 57 [json_name = "embeddedResources"];

    • getEmbeddedResourcesOrBuilderList

      List<? extends WarcEmbeddedResourceOrBuilder> getEmbeddedResourcesOrBuilderList()
       Embedded resources found within this record
      repeated .ai.pipestream.parsed.data.warc.v1.WarcEmbeddedResource embedded_resources = 57 [json_name = "embeddedResources"];

    • getEmbeddedResourcesOrBuilder

      WarcEmbeddedResourceOrBuilder getEmbeddedResourcesOrBuilder(int index)
       Embedded resources found within this record
      repeated .ai.pipestream.parsed.data.warc.v1.WarcEmbeddedResource embedded_resources = 57 [json_name = "embeddedResources"];

    • hasAdditionalMetadata

      boolean hasAdditionalMetadata()
       === Catch-all for unmapped WARC-specific metadata ===
      .google.protobuf.Struct additional_metadata = 200 [json_name = "additionalMetadata"];
      Returns:
      Whether the additionalMetadata field is set.

    • getAdditionalMetadata

      com.google.protobuf.Struct getAdditionalMetadata()
       === Catch-all for unmapped WARC-specific metadata ===
      .google.protobuf.Struct additional_metadata = 200 [json_name = "additionalMetadata"];
      Returns:
      The additionalMetadata.

    • getAdditionalMetadataOrBuilder

      com.google.protobuf.StructOrBuilder getAdditionalMetadataOrBuilder()
       === Catch-all for unmapped WARC-specific metadata ===
      .google.protobuf.Struct additional_metadata = 200 [json_name = "additionalMetadata"];

    • hasBaseFields

      boolean hasBaseFields()
       Base Tika fields common to all document types
      .ai.pipestream.parsed.data.tika.base.v1.TikaBaseFields base_fields = 201 [json_name = "baseFields"];
      Returns:
      Whether the baseFields field is set.

    • getBaseFields

      TikaBaseFields getBaseFields()
       Base Tika fields common to all document types
      .ai.pipestream.parsed.data.tika.base.v1.TikaBaseFields base_fields = 201 [json_name = "baseFields"];
      Returns:
      The baseFields.

    • getBaseFieldsOrBuilder

      TikaBaseFieldsOrBuilder getBaseFieldsOrBuilder()
       Base Tika fields common to all document types
      .ai.pipestream.parsed.data.tika.base.v1.TikaBaseFields base_fields = 201 [json_name = "baseFields"];