🔍️ perf: add sitemap and robot.txt to improve SEO (#1337)

* 🔍️ fix: add sitemap and robots.txt

* 🐛 fix: fix canonical url

* 🐛 fix: fix canonical url

* 🐛 fix: fix canonical url

* 🐛 fix: fix deploy url

* 🐛 fix: fix canonical url

* 🐛 fix: fix the robots config

.gitignore

@@ -56,3 +56,4 @@ next-env.d.ts
 .env
 public/*.js
 bun.lockb
+sitemap*.xml

.npmrc

@@ -1,5 +1,8 @@
 lockfile=false
 resolution-mode=highest
+
+enable-pre-post-scripts=true
+
 public-hoist-pattern[]=*@umijs/lint*
 public-hoist-pattern[]=*changelog*
 public-hoist-pattern[]=*commitlint*

next-sitemap.config.mjs

@@ -0,0 +1,54 @@
+import { glob } from 'glob';
+
+const isVercelPreview = process.env.VERCEL === '1' && process.env.VERCEL_ENV !== 'production';
+
+const prodUrl = process.env.SITE_URL || 'https://chat-preview.lobehub.com';
+
+const vercelPreviewUrl = `https://${process.env.VERCEL_URL}`;
+
+const siteUrl = isVercelPreview ? vercelPreviewUrl : prodUrl;
+
+/** @type {import('next-sitemap').IConfig} */
+const config = {
+  // next-sitemap does not work with app dir inside the /src dir (and have other problems e.g. with route groups)
+  // https://github.com/iamvishnusankar/next-sitemap/issues/700#issuecomment-1759458127
+  // https://github.com/iamvishnusankar/next-sitemap/issues/701
+  // additionalPaths is a workaround for this (once the issues are fixed, we can remove it)
+  additionalPaths: async () => {
+    const routes = await glob('src/app/**/page.{md,mdx,ts,tsx}', {
+      cwd: new URL('.', import.meta.url).pathname,
+    });
+
+    // https://nextjs.org/docs/app/building-your-application/routing/colocation#private-folders
+    const publicRoutes = routes.filter(
+      (page) => !page.split('/').some((folder) => folder.startsWith('_')),
+    );
+
+    // https://nextjs.org/docs/app/building-your-application/routing/colocation#route-groups
+    const publicRoutesWithoutRouteGroups = publicRoutes.map((page) =>
+      page
+        .split('/')
+        .filter((folder) => !folder.startsWith('(') && !folder.endsWith(')'))
+        .join('/'),
+    );
+
+    const locs = publicRoutesWithoutRouteGroups.map((route) => {
+      const path = route.replace(/^src\/app/, '').replace(/\/[^/]+$/, '');
+      const loc = path === '' ? siteUrl : `${siteUrl}/${path}`;
+
+      return loc;
+    });
+
+    const paths = locs.map((loc) => ({
+      changefreq: 'daily',
+      lastmod: new Date().toISOString(),
+      loc,
+      priority: 0.7,
+    }));
+
+    return paths;
+  },
+  siteUrl,
+};
+
+export default config;

package.json

@@ -27,6 +27,7 @@
   "sideEffects": false,
   "scripts": {
     "build": "next build",
+    "postbuild": "next-sitemap --config next-sitemap.config.mjs",
     "build:analyze": "ANALYZE=true next build",
     "build:docker": "DOCKER=true next build",
     "dev": "next dev -p 3010",
@@ -107,6 +108,7 @@
     "nanoid": "^5",
     "next": "^14.1",
     "next-auth": "5.0.0-beta.11",
+    "next-sitemap": "^4.2.3",
     "numeral": "^2.0.6",
     "nuqs": "^1.15.4",
     "openai": "^4.22",

public/robots.txt

@@ -0,0 +1,7 @@
+Sitemap: /sitemap.xml
+
+User-agent: *
+Allow: /$
+Allow: /welcome
+Allow: /market*
+Disallow: /

src/app/market/page.tsx

@@ -1,3 +1,6 @@
+import { Metadata } from 'next';
+
+import { getCanonicalUrl } from '@/const/url';
 import { isMobileDevice } from '@/utils/responsive';
 
 import DesktopPage from './(desktop)';
@@ -10,3 +13,7 @@ export default () => {
 
   return <Page />;
 };
+
+export const metadata: Metadata = {
+  alternates: { canonical: getCanonicalUrl('/market') },
+};

src/app/metadata.ts

@@ -2,13 +2,14 @@ import { Metadata } from 'next';
 
 import { getClientConfig } from '@/config/client';
 import { getServerConfig } from '@/config/server';
+import { OFFICIAL_URL } from '@/const/url';
 
 import pkg from '../../package.json';
 
 const title = 'LobeChat';
 const { description, homepage } = pkg;
 
-const { METADATA_BASE_URL = 'https://chat-preview.lobehub.com/' } = getServerConfig();
+const { SITE_URL = OFFICIAL_URL } = getServerConfig();
 const { BASE_PATH } = getClientConfig();
 
 // if there is a base path, then we don't need the manifest
@@ -28,7 +29,7 @@ const metadata: Metadata = {
       'https://registry.npmmirror.com/@lobehub/assets-favicons/latest/files/assets/favicon.ico',
   },
   manifest: noManifest ? undefined : '/manifest.json',
-  metadataBase: new URL(METADATA_BASE_URL),
+  metadataBase: new URL(SITE_URL),
   openGraph: {
     description: description,
     images: [
@@ -58,11 +59,11 @@ const metadata: Metadata = {
   },
   twitter: {
     card: 'summary_large_image',
-    creator: '@lobehub',
     description,
     images: [
       'https://registry.npmmirror.com/@lobehub/assets-favicons/latest/files/assets/og-960x540.png',
     ],
+    site: '@lobehub',
     title,
   },
 };

src/app/page.tsx

@@ -1,3 +1,7 @@
+import { Metadata } from 'next';
+
+import { getCanonicalUrl } from '@/const/url';
+
 import Page from './home';
 import Redirect from './home/Redirect';
 
@@ -9,3 +13,7 @@ const Index = () => (
 );
 
 export default Index;
+
+export const metadata: Metadata = {
+  alternates: { canonical: getCanonicalUrl('/') },
+};

src/app/welcome/page.tsx

@@ -1,3 +1,6 @@
+import { Metadata } from 'next';
+
+import { getCanonicalUrl } from '@/const/url';
 import { isMobileDevice } from '@/utils/responsive';
 
 import DesktopPage from './(desktop)';
@@ -12,3 +15,7 @@ const Page = () => {
 };
 
 export default Page;
+
+export const metadata: Metadata = {
+  alternates: { canonical: getCanonicalUrl('/welcome') },
+};

src/config/server/app.ts

@@ -8,7 +8,7 @@ declare global {
 
       IMGUR_CLIENT_ID?: string;
 
-      METADATA_BASE_URL?: string;
+      SITE_URL?: string;
 
       AGENTS_INDEX_URL?: string;
 
@@ -38,7 +38,7 @@ export const getAppConfig = () => {
 
     SHOW_ACCESS_CODE_CONFIG: !!ACCESS_CODES.length,
 
-    METADATA_BASE_URL: process.env.METADATA_BASE_URL,
+    SITE_URL: process.env.SITE_URL,
 
     IMGUR_CLIENT_ID: process.env.IMGUR_CLIENT_ID || DEFAULT_IMAGUR_CLIENT_ID,
 

src/const/url.ts

@@ -6,6 +6,10 @@ import { withBasePath } from '@/utils/basePath';
 import pkg from '../../package.json';
 import { INBOX_SESSION_ID } from './session';
 
+export const OFFICIAL_URL = 'https://chat-preview.lobehub.com/';
+
+export const getCanonicalUrl = (path: string) => urlJoin(OFFICIAL_URL, path);
+
 export const GITHUB = pkg.homepage;
 export const CHANGELOG = urlJoin(GITHUB, 'blob/master/CHANGELOG.md');