Struct RobotsGroup 

pub struct RobotsGroup {
    pub user_agents: Vec<String>,
    pub rules: Vec<RobotsRule>,
    pub crawl_delay: Option<Duration>,
}

A user-agent group in a robots.txt document.

Fields

user_agents: Vec<String>

rules: Vec<RobotsRule>

crawl_delay: Option<Duration>

Implementations

impl RobotsGroup

pub fn new(user_agent: impl Into<String>) -> RobotsGroup

Create a group for a single user-agent token.

pub fn with_user_agent(self, user_agent: impl Into<String>) -> RobotsGroup

Add an additional user-agent token to this group.

pub fn set_user_agent( &mut self, user_agent: impl Into<String>, ) -> &mut RobotsGroup

Add an additional user-agent token to this group.

pub fn with_allow(self, path: impl Into<String>) -> RobotsGroup

Add an Allow directive to this group.

pub fn set_allow(&mut self, path: impl Into<String>) -> &mut RobotsGroup

Add an Allow directive to this group.

pub fn with_disallow(self, path: impl Into<String>) -> RobotsGroup

Add a Disallow directive to this group.

pub fn set_disallow(&mut self, path: impl Into<String>) -> &mut RobotsGroup

Add a Disallow directive to this group.

pub fn with_crawl_delay(self, delay: Duration) -> RobotsGroup

Set the crawl delay for this group.

pub fn set_crawl_delay(&mut self, delay: Duration) -> &mut RobotsGroup

Set the crawl delay for this group.

Trait Implementations

impl Clone for RobotsGroup

fn clone(&self) -> RobotsGroup

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for RobotsGroup

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl Default for RobotsGroup

fn default() -> RobotsGroup

Returns the “default value” for a type.

impl Display for RobotsGroup

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl PartialEq for RobotsGroup

fn eq(&self, other: &RobotsGroup) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Eq for RobotsGroup

impl StructuralPartialEq for RobotsGroup