# robots.txt for Data AI Hub User-agent: * Allow: / # Disallow admin and private pages Disallow: /admin Disallow: /database-admin Disallow: /api/ Disallow: /playground # Allow important pages Allow: / Allow: /industry Allow: /startups Allow: /ai_ecosystem Allow: /ai_news Allow: /data_platforms Allow: /llm_frameworks Allow: /knowledge_graphs Allow: /articles Allow: /videos Allow: /students Allow: /about Allow: /explore Allow: /daily Allow: /daily/archive # Sitemap location Sitemap: https://www.dataaihub.co/sitemap.xml