Skip to content

Commit 5274efd

Browse files
emir-karabegemir-karabeg
authored
improvement(seo): optimize sitemaps, robots.txt, and core web vitals across sim and docs (#4170)
* improvement(seo): optimize sitemaps and robots.txt across sim and docs - Add missing pages to sim sitemap: blog author pages, academy catalog and course pages - Fix 6x duplicate URL bug in docs sitemap by deduplicating with source.getLanguages() - Convert docs sitemap from route handler to Next.js metadata convention with native hreflang - Add x-default hreflang alternate for docs multi-language pages - Remove changeFrequency and priority fields (Google ignores both) - Fix inaccurate lastModified timestamps — derive from real content dates, omit when unknown - Consolidate 20+ redundant per-bot robots rules into single wildcard entry - Add /form/ and /credential-account/ to sim robots disallow list - Reference image sitemap in sim robots.txt - Remove deprecated host directive from sim robots - Move disallow rules before allow in docs robots for crawler compatibility - Extract hardcoded docs baseUrl to env variable with production fallback * fix(seo): remove homepage new Date(), guard latestModelDate empty array * improvement(seo): consolidate DOCS_BASE_URL, optimize core web vitals Extract hardcoded https://docs.sim.ai into shared DOCS_BASE_URL constant in lib/urls.ts and replace all 20+ instances across layouts, metadata, structured data, LLM manifest, sitemap, and robots files. Remove OneDollarStats analytics script and tighten CSP for improved core web vitals. * fix: removed onedollarstats from bun lock * fix(seo): guard per-provider Math.max, consolidate docs robots to single wildcard
1 parent 0b36c8b commit 5274efd

File tree

17 files changed

+159
-366
lines changed

17 files changed

+159
-366
lines changed

‎apps/docs/app/[lang]/[[...slug]]/page.tsx‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,10 @@ import { ResponseSection } from '@/components/ui/response-section'
1717
import { i18n } from '@/lib/i18n'
1818
import { getApiSpecContent, openapi } from '@/lib/openapi'
1919
import { type PageData, source } from '@/lib/source'
20+
import { DOCS_BASE_URL } from '@/lib/urls'
2021

2122
const SUPPORTED_LANGUAGES: Set<string> = new Set(i18n.languages)
22-
const BASE_URL = 'https://docs.sim.ai'
23+
const BASE_URL = DOCS_BASE_URL
2324

2425
const OG_LOCALE_MAP: Record<string, string> = {
2526
en: 'en_US',

‎apps/docs/app/[lang]/layout.tsx‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ import { defineI18nUI } from 'fumadocs-ui/i18n'
33
import { DocsLayout } from 'fumadocs-ui/layouts/docs'
44
import { RootProvider } from 'fumadocs-ui/provider/next'
55
import { Geist_Mono, Inter } from 'next/font/google'
6-
import Script from 'next/script'
76
import {
87
SidebarFolder,
98
SidebarItem,
@@ -13,6 +12,7 @@ import { Navbar } from '@/components/navbar/navbar'
1312
import { SimLogoFull } from '@/components/ui/sim-logo'
1413
import { i18n } from '@/lib/i18n'
1514
import { source } from '@/lib/source'
15+
import { DOCS_BASE_URL } from '@/lib/urls'
1616
import '../global.css'
1717

1818
const inter = Inter({
@@ -67,22 +67,22 @@ export default async function Layout({ children, params }: LayoutProps) {
6767
name: 'Sim Documentation',
6868
description:
6969
'Documentation for Sim — the open-source AI workspace where teams build, deploy, and manage AI agents. Connect 1,000+ integrations and every major LLM.',
70-
url: 'https://docs.sim.ai',
70+
url: DOCS_BASE_URL,
7171
publisher: {
7272
'@type': 'Organization',
7373
name: 'Sim',
7474
url: 'https://sim.ai',
7575
logo: {
7676
'@type': 'ImageObject',
77-
url: 'https://docs.sim.ai/static/logo.png',
77+
url: `${DOCS_BASE_URL}/static/logo.png`,
7878
},
7979
},
8080
inLanguage: lang,
8181
potentialAction: {
8282
'@type': 'SearchAction',
8383
target: {
8484
'@type': 'EntryPoint',
85-
urlTemplate: 'https://docs.sim.ai/api/search?q={search_term_string}',
85+
urlTemplate: `${DOCS_BASE_URL}/api/search?q={search_term_string}`,
8686
},
8787
'query-input': 'required name=search_term_string',
8888
},
@@ -101,7 +101,6 @@ export default async function Layout({ children, params }: LayoutProps) {
101101
/>
102102
</head>
103103
<body className='flex min-h-screen flex-col font-sans'>
104-
<Script src='https://assets.onedollarstats.com/stonks.js' strategy='lazyOnload' />
105104
<RootProvider i18n={provider(lang)}>
106105
<Navbar />
107106
<DocsLayout

‎apps/docs/app/layout.tsx‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import type { ReactNode } from 'react'
22
import type { Viewport } from 'next'
3+
import { DOCS_BASE_URL } from '@/lib/urls'
34

45
export default function RootLayout({ children }: { children: ReactNode }) {
56
return children
@@ -12,7 +13,7 @@ export const viewport: Viewport = {
1213
}
1314

1415
export const metadata = {
15-
metadataBase: new URL('https://docs.sim.ai'),
16+
metadataBase: new URL(DOCS_BASE_URL),
1617
title: {
1718
default: 'Sim Documentation — The AI Workspace for Teams',
1819
template: '%s | Sim Docs',
@@ -61,14 +62,14 @@ export const metadata = {
6162
type: 'website',
6263
locale: 'en_US',
6364
alternateLocale: ['es_ES', 'fr_FR', 'de_DE', 'ja_JP', 'zh_CN'],
64-
url: 'https://docs.sim.ai',
65+
url: DOCS_BASE_URL,
6566
siteName: 'Sim Documentation',
6667
title: 'Sim Documentation — The AI Workspace for Teams',
6768
description:
6869
'Documentation for Sim — the open-source AI workspace where teams build, deploy, and manage AI agents. Connect 1,000+ integrations and every major LLM.',
6970
images: [
7071
{
71-
url: 'https://docs.sim.ai/api/og?title=Sim%20Documentation',
72+
url: `${DOCS_BASE_URL}/api/og?title=Sim%20Documentation`,
7273
width: 1200,
7374
height: 630,
7475
alt: 'Sim Documentation',
@@ -82,7 +83,7 @@ export const metadata = {
8283
'Documentation for Sim — the open-source AI workspace where teams build, deploy, and manage AI agents. Connect 1,000+ integrations and every major LLM.',
8384
creator: '@simdotai',
8485
site: '@simdotai',
85-
images: ['https://docs.sim.ai/api/og?title=Sim%20Documentation'],
86+
images: [`${DOCS_BASE_URL}/api/og?title=Sim%20Documentation`],
8687
},
8788
robots: {
8889
index: true,
@@ -96,15 +97,15 @@ export const metadata = {
9697
},
9798
},
9899
alternates: {
99-
canonical: 'https://docs.sim.ai',
100+
canonical: DOCS_BASE_URL,
100101
languages: {
101-
'x-default': 'https://docs.sim.ai',
102-
en: 'https://docs.sim.ai',
103-
es: 'https://docs.sim.ai/es',
104-
fr: 'https://docs.sim.ai/fr',
105-
de: 'https://docs.sim.ai/de',
106-
ja: 'https://docs.sim.ai/ja',
107-
zh: 'https://docs.sim.ai/zh',
102+
'x-default': DOCS_BASE_URL,
103+
en: DOCS_BASE_URL,
104+
es: `${DOCS_BASE_URL}/es`,
105+
fr: `${DOCS_BASE_URL}/fr`,
106+
de: `${DOCS_BASE_URL}/de`,
107+
ja: `${DOCS_BASE_URL}/ja`,
108+
zh: `${DOCS_BASE_URL}/zh`,
108109
},
109110
},
110111
}

‎apps/docs/app/llms.txt/route.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
import { source } from '@/lib/source'
2+
import { DOCS_BASE_URL } from '@/lib/urls'
23

34
export const revalidate = false
45

56
export async function GET() {
6-
const baseUrl = 'https://docs.sim.ai'
7+
const baseUrl = DOCS_BASE_URL
78

89
try {
910
const pages = source.getPages().filter((page) => {

‎apps/docs/app/robots.txt/route.ts‎

Lines changed: 5 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -1,70 +1,18 @@
1+
import { DOCS_BASE_URL } from '@/lib/urls'
2+
13
export const revalidate = false
24

35
export async function GET() {
4-
const baseUrl = 'https://docs.sim.ai'
6+
const baseUrl = DOCS_BASE_URL
57

68
const robotsTxt = `# Robots.txt for Sim Documentation
79
810
User-agent: *
9-
Allow: /
10-
11-
# Search engine crawlers
12-
User-agent: Googlebot
13-
Allow: /
14-
15-
User-agent: Bingbot
16-
Allow: /
17-
18-
User-agent: Slurp
19-
Allow: /
20-
21-
User-agent: DuckDuckBot
22-
Allow: /
23-
24-
User-agent: Baiduspider
25-
Allow: /
26-
27-
User-agent: YandexBot
28-
Allow: /
29-
30-
# AI and LLM crawlers - explicitly allowed for documentation indexing
31-
User-agent: GPTBot
32-
Allow: /
33-
34-
User-agent: ChatGPT-User
35-
Allow: /
36-
37-
User-agent: CCBot
38-
Allow: /
39-
40-
User-agent: anthropic-ai
41-
Allow: /
42-
43-
User-agent: Claude-Web
44-
Allow: /
45-
46-
User-agent: Applebot
47-
Allow: /
48-
49-
User-agent: PerplexityBot
50-
Allow: /
51-
52-
User-agent: Diffbot
53-
Allow: /
54-
55-
User-agent: FacebookBot
56-
Allow: /
57-
58-
User-agent: cohere-ai
59-
Allow: /
60-
61-
# Disallow admin and internal paths (if any exist)
6211
Disallow: /.next/
6312
Disallow: /api/internal/
6413
Disallow: /_next/static/
6514
Disallow: /admin/
66-
67-
# Allow but don't prioritize these
15+
Allow: /
6816
Allow: /api/search
6917
Allow: /llms.txt
7018
Allow: /llms-full.txt
@@ -73,23 +21,12 @@ Allow: /llms.mdx/
7321
# Sitemaps
7422
Sitemap: ${baseUrl}/sitemap.xml
7523
76-
# Crawl delay for aggressive bots (optional)
77-
# Crawl-delay: 1
78-
7924
# Additional resources for AI indexing
8025
# See https://github.com/AnswerDotAI/llms-txt for more info
8126
# LLM-friendly content:
8227
# Manifest: ${baseUrl}/llms.txt
8328
# Full content: ${baseUrl}/llms-full.txt
84-
# Individual pages: ${baseUrl}/llms.mdx/[page-path]
85-
86-
# Multi-language documentation available at:
87-
# ${baseUrl}/en - English
88-
# ${baseUrl}/es - Español
89-
# ${baseUrl}/fr - Français
90-
# ${baseUrl}/de - Deutsch
91-
# ${baseUrl}/ja - 日本語
92-
# ${baseUrl}/zh - 简体中文`
29+
# Individual pages: ${baseUrl}/llms.mdx/[page-path]`
9330

9431
return new Response(robotsTxt, {
9532
headers: {

‎apps/docs/app/sitemap.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import type { MetadataRoute } from 'next'
2+
import { i18n } from '@/lib/i18n'
3+
import { source } from '@/lib/source'
4+
import { DOCS_BASE_URL } from '@/lib/urls'
5+
6+
export const revalidate = 3600
7+
8+
export default function sitemap(): MetadataRoute.Sitemap {
9+
const baseUrl = DOCS_BASE_URL
10+
const languages = source.getLanguages()
11+
12+
const pagesBySlug = new Map<string, Map<string, string>>()
13+
for (const { language, pages } of languages) {
14+
for (const page of pages) {
15+
const key = page.slugs.join('/')
16+
if (!pagesBySlug.has(key)) {
17+
pagesBySlug.set(key, new Map())
18+
}
19+
pagesBySlug.get(key)!.set(language, `${baseUrl}${page.url}`)
20+
}
21+
}
22+
23+
const entries: MetadataRoute.Sitemap = []
24+
for (const [, localeMap] of pagesBySlug) {
25+
const defaultUrl = localeMap.get(i18n.defaultLanguage)
26+
if (!defaultUrl) continue
27+
28+
const langAlternates: Record<string, string> = {}
29+
for (const [lang, url] of localeMap) {
30+
langAlternates[lang] = url
31+
}
32+
33+
langAlternates['x-default'] = defaultUrl
34+
35+
entries.push({
36+
url: defaultUrl,
37+
alternates: { languages: langAlternates },
38+
})
39+
}
40+
41+
return entries
42+
}

‎apps/docs/app/sitemap.xml/route.ts‎

Lines changed: 0 additions & 62 deletions
This file was deleted.

‎apps/docs/components/structured-data.tsx‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import { DOCS_BASE_URL } from '@/lib/urls'
2+
13
interface StructuredDataProps {
24
title: string
35
description: string
@@ -15,7 +17,7 @@ export function StructuredData({
1517
dateModified,
1618
breadcrumb,
1719
}: StructuredDataProps) {
18-
const baseUrl = 'https://docs.sim.ai'
20+
const baseUrl = DOCS_BASE_URL
1921

2022
const articleStructuredData = {
2123
'@context': 'https://schema.org',

‎apps/docs/lib/urls.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export const DOCS_BASE_URL = process.env.NEXT_PUBLIC_DOCS_URL ?? 'https://docs.sim.ai'

‎apps/sim/app/layout.tsx‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import { BrandedLayout } from '@/components/branded-layout'
55
import { PostHogProvider } from '@/app/_shell/providers/posthog-provider'
66
import { generateBrandedMetadata, generateThemeCSS } from '@/ee/whitelabeling'
77
import '@/app/_styles/globals.css'
8-
import { OneDollarStats } from '@/components/analytics/onedollarstats'
98
import { isHosted, isReactGrabEnabled, isReactScanEnabled } from '@/lib/core/config/feature-flags'
109
import { HydrationErrorHandler } from '@/app/_shell/hydration-error-handler'
1110
import { QueryProvider } from '@/app/_shell/providers/query-provider'
@@ -207,10 +206,6 @@ export default function RootLayout({ children }: { children: React.ReactNode })
207206
<meta name='format-detection' content='telephone=no' />
208207
<meta httpEquiv='x-ua-compatible' content='ie=edge' />
209208

210-
{/* OneDollarStats Analytics */}
211-
<link rel='dns-prefetch' href='https://assets.onedollarstats.com' />
212-
<script defer src='https://assets.onedollarstats.com/stonks.js' />
213-
214209
{/* Google Tag Manager — hosted only */}
215210
{isHosted && (
216211
<Script
@@ -260,7 +255,6 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
260255
</noscript>
261256
)}
262257
<HydrationErrorHandler />
263-
<OneDollarStats />
264258
<PostHogProvider>
265259
<ThemeProvider>
266260
<QueryProvider>

0 commit comments

Comments
 (0)