# As a condition of accessing this website, you agree to abide by the following
# content signals:
# (a) If a Content-Signal = yes, you may collect content for the corresponding use.
# (b) If a Content-Signal = no, you may not collect content for the corresponding use.
# (c) If the website operator does not include a Content-Signal for a corresponding use, 
#     the website operator neither grants nor restricts permission via Content-Signal.

# search: building a search index and providing search results. 
# ai-input: inputting content into AI models for real-time answers.
# ai-train: training or fine-tuning AI models.

# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790.

User-agent: *
Allow: /

# Jeda 20 detik agar bot tidak membuat RAM server penuh
Crawl-delay: 20

User-agent: *
# Allow Google to see the "look" of the site
Allow: /core/*.css
Allow: /core/*.js
Allow: /core/*.png
Allow: /core/*.jpg
Allow: /core/*.svg
Allow: /profiles/*.css
Allow: /profiles/*.js

# System Blocks
Disallow: /core/
Disallow: /profiles/
Disallow: /install.php
Disallow: /index.php/
Disallow: /composer/

# Standard Drupal Admin/User Blocks
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout

# Targeted Fixes for 5xx Errors (The "Problem" URLs)
Disallow: /pa/index.php/
Disallow: /*product-search*
Disallow: /*/news?page=
Disallow: /*/media/

# Fix 404 Errors (Redundant Brand Path)
Disallow: /*/worldoftest/

# Only keep this if you use Path Aliases for all products/articles
Disallow: /*/node/

Disallow: /?* Disallow: /gtranslate/

User-agent: SemrushBot
Disallow: /cdn-cgi/

# SITEMAPS
Sitemap: https://www.worldoftest.com/sitemap.xml
Sitemap: https://www.worldoftest.com/en/sitemap.xml
Sitemap: https://www.worldoftest.com/id/sitemap.xml
Sitemap: https://www.worldoftest.com/vi/sitemap.xml
Sitemap: https://www.worldoftest.com/it/sitemap.xml
Sitemap: https://www.worldoftest.com/es/sitemap.xml
Sitemap: https://www.worldoftest.com/pt/sitemap.xml
Sitemap: https://www.worldoftest.com/th/sitemap.xml
Sitemap: https://www.worldoftest.com/tr/sitemap.xml
Sitemap: https://www.worldoftest.com/ar/sitemap.xml
Sitemap: https://www.worldoftest.com/fr/sitemap.xml
Sitemap: https://www.worldoftest.com/ja/sitemap.xml
Sitemap: https://www.worldoftest.com/el/sitemap.xml
Sitemap: https://www.worldoftest.com/ko/sitemap.xml
Sitemap: https://www.worldoftest.com/de/sitemap.xml
Sitemap: https://www.worldoftest.com/sitemap.xml