# Robots.txt for Victoria International India
# Leading Global Employment Platform for Indian Professionals
# विक्टोरिया इंटरनेशनल - भारतीय पेशेवरों के लिए वैश्विक रोजगार मंच

User-agent: *
Allow: /

# India-specific pages
Allow: /mumbai.html
Allow: /delhi.html
Allow: /bangalore.html
Allow: /chennai.html
Allow: /kolkata.html
Allow: /pune.html
Allow: /ahmedabad.html
Allow: /hyderabad.html

# Hindi content
Allow: /hi/
Allow: /*hindi*
Allow: /*भारत*

# Disallow sensitive areas
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /.git/
Disallow: /backup/

# Allow all static resources
Allow: /css/
Allow: /js/
Allow: /images/
Allow: /fonts/
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.mp4$
Allow: /*.mov$
Allow: /*.avi$

# Important files
Allow: /sitemap.xml
Allow: /favicon.ico
Allow: /manifest.json

# Sitemap location
Sitemap: https://victoriainternational.pro/sitemap.xml

# Crawl delay (optional)
Crawl-delay: 1

# Google specific rules
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Bing specific rules  
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Social media bots
User-agent: facebookexternalhit/1.1
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# SEO tools
User-agent: SemrushBot
Allow: /

User-agent: AhrefsBot
Allow: /

User-agent: MJ12bot
Allow: /

# Archive bots
User-agent: ia_archiver
Allow: /

# Bad bots (block these)
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot  
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: 360Spider
Disallow: /

User-agent: ZmEu
Disallow: /

User-agent: BlackWidow
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: WebBandit
Disallow: /

# Block aggressive crawlers
User-agent: *
Disallow: /search
Disallow: /*?search=*
Disallow: /*?q=*
Disallow: /*?s=*

# Cache directives
# No caching for dynamic content
User-agent: *
Disallow: /api/
Disallow: /cgi-bin/
Disallow: /*?*&*

# Host directive (preferred domain)
Host: victoriainternational.pro