Skip to main content

Known Bots

This policy group functions uniquely in WAAP. Instead of challenging or blocking incoming traffic like other policy groups, it's designed to recognize and allow traffic from legitimate automated services.

These services include:

  • Common web crawlers like Google and Microsoft bots
  • Preview fetch bots like Slack, Facebook, or Twitter
  • SEO and monitoring tools
  • Other tools like Instant Payment Notification (IPN) requests from PayPal or HiPay

All these automation services often access your domain to gather some information and are considered beneficial or necessary for normal web activities. Note that some policies are set to Policy-based mode by default, and you can set them to Allow to fully permit traffic from the relevant services.

In some cases, you might want to restrict automation tools from accessing your domain. For instance, you can do so to enforce specific data protection requirements on a domain, protect it from potentially harmful services, or restrict paid content from unauthorized use. In this case, adjust the mode for particular bots in the Known Bots section.

Info

This policy group is available in the Pro and Enterprise plans. More details on the Security pricing page.

Configure Known Bots

You can review and configure relevant bots in the CDB Technical Web Portal:

  1. Navigate to WAAP > Bot Management.

  2. In the domain dropdown at the top of the page, select the needed domain.

  3. Click the Known Bots tab to view and adjust the list of allowed bots.

Known Bots policies control how WAAP handles traffic from 136 recognized automated services — search engine crawlers, social media preview bots, monitoring tools, and payment processors. Each bot can be set to Allow (bypass WAF inspection) or Policy-based (subject to other active WAF rules).

An API token is required, along with the ID of a WAAP-protected domain and the Python or Go SDK installed for SDK examples. This policy group is available on the Pro and Enterprise WAAP plans.

export GCORE_API_KEY="{YOUR_API_KEY}"
export WAAP_DOMAIN_ID="{YOUR_DOMAIN_ID}"

View bot modes

Retrieve the full list of Known Bots policies and their current modes for a domain.

import gcore
import os

client = gcore.CDB(api_key=os.environ["GCORE_API_KEY"])
domain_id = int(os.environ["WAAP_DOMAIN_ID"])

rule_sets = client.waap.domains.list_rule_sets(domain_id)
bot_set = next(
    rs for rs in rule_sets if rs.resource_slug == "common-automated-services"
)

for bot in bot_set.rules:
    mode = "Allow" if bot.mode else "Policy-based"
    print(f"{bot.name}: {mode} ({bot.id})")

package main

import (
    "context"
    "fmt"
    "os"
    "strconv"

    gcore "github.com/G-Core/gcore-go"
    "github.com/G-Core/gcore-go/option"
    "github.com/G-Core/gcore-go/waap"
)

func main() {
    client := gcore.NewClient(option.WithAPIKey(os.Getenv("GCORE_API_KEY")))
    domainID, _ := strconv.ParseInt(os.Getenv("WAAP_DOMAIN_ID"), 10, 64)

    ruleSets, _ := client.Waap.Domains.ListRuleSets(context.Background(), domainID)
    for _, rs := range *ruleSets {
        if rs.ResourceSlug == "common-automated-services" {
            for _, bot := range rs.Rules {
                mode := "Policy-based"
                if bot.Mode {
                    mode = "Allow"
                }
                fmt.Printf("%s: %s (%s)\n", bot.Name, mode, bot.ID)
            }
        }
    }
}

curl -X GET "https://api.cdb-staging.cdn.orange.com/waap/v1/domains/${WAAP_DOMAIN_ID}/rule-sets" \
  -H "Authorization: APIKey ${GCORE_API_KEY}" \
  | jq '.[] | select(.resource_slug == "common-automated-services") | .rules[] | {name, id, mode}'

The API returns all policies in the group. mode: true means Allow; mode: false means Policy-based.

Toggle a bot mode

Switch a bot between Allow and Policy-based. Each call flips the current mode. Use the View bot modes response to find the bot ID.

import gcore
import os

client = gcore.CDB(api_key=os.environ["GCORE_API_KEY"])
domain_id = int(os.environ["WAAP_DOMAIN_ID"])

# Find the bot ID by name
rule_sets = client.waap.domains.list_rule_sets(domain_id)
bot_set = next(
    rs for rs in rule_sets if rs.resource_slug == "common-automated-services"
)
bot = next(r for r in bot_set.rules if r.name == "Google Bot")

result = client.waap.domains.policies.toggle(bot.id, domain_id=domain_id)
mode = "Allow" if result.mode else "Policy-based"
print(f"Google Bot is now {mode}")

package main

import (
    "context"
    "fmt"
    "os"
    "strconv"

    gcore "github.com/G-Core/gcore-go"
    "github.com/G-Core/gcore-go/option"
    "github.com/G-Core/gcore-go/waap"
)

func main() {
    client := gcore.NewClient(option.WithAPIKey(os.Getenv("GCORE_API_KEY")))
    domainID, _ := strconv.ParseInt(os.Getenv("WAAP_DOMAIN_ID"), 10, 64)

    // Find the bot ID by name
    ruleSets, _ := client.Waap.Domains.ListRuleSets(context.Background(), domainID)
    var botID string
    for _, rs := range *ruleSets {
        if rs.ResourceSlug == "common-automated-services" {
            for _, bot := range rs.Rules {
                if bot.Name == "Google Bot" {
                    botID = bot.ID
                }
            }
        }
    }

    result, _ := client.Waap.Domains.Policies.Toggle(context.Background(), botID, waap.DomainPolicyToggleParams{DomainID: domainID})
    mode := "Policy-based"
    if result.Mode {
        mode = "Allow"
    }
    fmt.Printf("Google Bot is now %s\n", mode)
}

# Set BOT_POLICY_ID to the bot ID from the View bot modes response
export BOT_POLICY_ID="{BOT_POLICY_ID}"

curl -X PATCH "https://api.cdb-staging.cdn.orange.com/waap/v1/domains/${WAAP_DOMAIN_ID}/policies/${BOT_POLICY_ID}/toggle" \
  -H "Authorization: APIKey ${GCORE_API_KEY}"

Response:

{"mode": true}

The API returns the updated mode. mode: true means Allow; mode: false means Policy-based.