# FINAL robots.txt for thegoldentreeproperties.com - FIXED 2026-09-21
# Fixes: 76,593 Duplicate without canonical, 5,786 www duplicates, 221 Alternate pages, 13 Blocked by robots.txt
# Last updated: Integrated index.php fix + SA/ND/MA spam fix

# Allow Facebook Bots - Important for property sharing
User-agent: facebookexternalhit
Allow: /
Allow: /uploads/
Allow: /for-sale/
Allow: /property/
Allow: /blog/

User-agent: Facebot
Allow: /
Allow: /uploads/
Allow: /for-sale/
Allow: /property/
Allow: /blog/

# Global rules
User-agent: *

# === PRIVATE FOLDERS - BLOCK ===
Disallow: /admin/
Disallow: /config.php
Disallow: /admin/config.php
Disallow: /db.php
Disallow: /.env
Disallow: /.git/

# === FIX SPAM PARAMS ?SA ?ND ?MA (from Coverage 221 alternates) ===
Disallow: /*?*SA=
Disallow: /*?*ND=
Disallow: /*?*MA=
Disallow: /*?SA
Disallow: /*?ND
Disallow: /*?MA
Disallow: /*&SA
Disallow: /*&ND
Disallow: /*&MA

# === FIX 76,593 DUPLICATES - BLOCK TRACKING PARAMS (keep these) ===
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*utm_source=
Disallow: /*?*utm_medium=
Disallow: /*?*utm_campaign=
Disallow: /*?*utm_content=
Disallow: /*?*utm_term=
Disallow: /*?*fb_action_ids=
Disallow: /*?*fb_action_types=
Disallow: /*?*fb_source=

# === FIX OLD PARAM URLS - BUT ALLOW FILTERED PAGES TO SEE NOINDEX ===
# REMOVED: Disallow: /properties.php?*  <- THIS WAS BLOCKING NOINDEX FROM BEING SEEN (causes 13 blocked)
# Instead, block only specific old ID-based URLs, allow loc/city to be crawled for noindex
Disallow: /*?*id=
Disallow: /property_details.php?*
Disallow: /property.php?*
# Keep search_title blocked only if you want, but we now handle via noindex, so ALLOW it to be crawled:
# Disallow: /*?search_title=  <- REMOVED, handled via noindex in properties.php

Disallow: /*?price_min=
Disallow: /*?price_max=
Disallow: /*?location=
Disallow: /*?type=
Disallow: /*?class=

# === BLOCK SORT/FILTER PARAMS ===
Disallow: /*?*sort=
Disallow: /*?*order=
Disallow: /*?*view=
Disallow: /*?*filter=
Disallow: /*?*page=0
Disallow: /*?*page=1$

# === BLOCK GOOGLE DUPLICATE HTML PARAMS ===
Disallow: /*?*.html=
Disallow: /*?*%3C
Disallow: /*?*%3E
Disallow: /*?*%2Fshop%2F
Disallow: /index.php/shop/

# === ALLOW MAIN CONTENT - INCLUDING BLOG ===
# Allow clean properties.php to be indexed, filtered versions will show noindex
Allow: /property/
Allow: /properties.php$
Allow: /properties.php?loc=
Allow: /properties.php?city=
Allow: /properties.php?province=
Allow: /property_details.php$
Allow: /index.php$
Allow: /for-sale/
Allow: /blog/
Allow: /uploads/
Allow: /assets/
Allow: /

# === SITEMAP ===
Sitemap: https://thegoldentreeproperties.com/sitemap.php
Sitemap: https://thegoldentreeproperties.com/sitemap.xml