refactor: restructure sitemap and robots handling

- Removed the old sitemap implementation and replaced it with a new one that generates dynamic routes for categories, posts, and documentation. - Introduced a new robots.txt handler to manage crawling rules and specify the sitemap location. - Updated URL handling functions to improve locale support in callback URLs. - Enhanced the content-collections.ts file by reorganizing documentation comments and reintroducing the extractLocaleAndBase function for better clarity and maintainability.
2025-04-12 08:16:47 +08:00 · 2025-04-12 08:16:47 +08:00 · f6bec8b78c
commit f6bec8b78c
parent b951f92ff1
6 changed files with 151 additions and 94 deletions
--- a/content-collections.ts
+++ b/content-collections.ts
@ -8,14 +8,12 @@ import {
 import path from "path";

 /**
- * Content Collections documentation
- * 1. https://www.content-collections.dev/docs/quickstart/next
- * 2. https://www.content-collections.dev/docs/configuration
- * 3. https://www.content-collections.dev/docs/transform#join-collections
- */
-
-/**
- * Use Content Collections for Fumadocs
+ * 1. Content Collections documentation
+ * https://www.content-collections.dev/docs/quickstart/next
+ * https://www.content-collections.dev/docs/configuration
+ * https://www.content-collections.dev/docs/transform#join-collections
+ * 
+ * 2. Use Content Collections for Fumadocs
 * https://fumadocs.vercel.app/docs/headless/content-collections
 */
 const docs = defineCollection({
@ -38,38 +36,14 @@ const metas = defineCollection({
  schema: createMetaSchema,
 });

-/**
- * Helper function to extract locale and base name from filename
- * Handles filename formats:
- * - name -> locale: DEFAULT_LOCALE, base: name
- * - name.zh -> locale: zh, base: name
- * 
- * @param fileName Filename without extension (already has .mdx removed)
- * @returns Object with locale and base name
- */
-function extractLocaleAndBase(fileName: string): { locale: string; base: string } {
-  // Split filename into parts
-  const parts = fileName.split('.');
-
-  if (parts.length === 1) {
-    // Simple filename without locale: xxx
-    return { locale: DEFAULT_LOCALE, base: parts[0] };
-  } else if (parts.length === 2 && LOCALES.includes(parts[1])) {
-    // Filename with locale: xxx.zh
-    return { locale: parts[1], base: parts[0] };
-  } else {
-    // Unexpected format, use first part as base and default locale
-    console.warn(`Unexpected filename format: ${fileName}`);
-    return { locale: DEFAULT_LOCALE, base: parts[0] };
-  }
-}
-
 /**
 * Blog Author collection
 * 
 * Authors are identified by their slug across all languages
 * New format: content/author/authorname.{locale}.mdx
 * Example: content/author/mksaas.mdx (default locale) and content/author/mksaas.zh.mdx (Chinese)
+ * 
+ * For author, slug is slugAsParams
 */
 export const authors = defineCollection({
  name: 'author',
@ -103,6 +77,8 @@ export const authors = defineCollection({
 * Categories are identified by their slug across all languages
 * New format: content/category/categoryname.{locale}.mdx
 * Example: content/category/tutorial.mdx (default locale) and content/category/tutorial.zh.mdx (Chinese)
+ * 
+ * For category, slug is slugAsParams
 */
 export const categories = defineCollection({
  name: 'category',
@ -135,6 +111,9 @@ export const categories = defineCollection({
 * 
 * New format: content/blog/post-slug.{locale}.mdx
 * 
+ * slug: /blog/first-post, used in URL or sitemap
+ * slugAsParams: first-post, used in route params
+ * 
 * 1. For a blog post at content/blog/first-post.mdx (default locale):
 * locale: en
 * slug: /blog/first-post
@ -185,14 +164,10 @@ export const posts = defineCollection({
      return category;
    }).filter(Boolean); // Remove null values

-    // Get the collection name (e.g., "blog")
-    const pathParts = data._meta.path.split(path.sep);
-    const collectionName = pathParts[pathParts.length - 2];
-
    // Create the slug and slugAsParams
-    const slug = `/${collectionName}/${base}`;
+    const slug = `/blog/${base}`;
    const slugAsParams = base;
-
+    
    // Calculate estimated reading time
    const wordCount = data.content.split(/\s+/).length;
    const wordsPerMinute = 200; // average reading speed: 200 words per minute
@ -249,12 +224,8 @@ export const pages = defineCollection({
    const { locale, base } = extractLocaleAndBase(fileName);
    // console.log(`page processed: ${fileName}, base=${base}, locale=${locale}`);

-    // Get the collection name (e.g., "pages")
-    const pathParts = data._meta.path.split(path.sep);
-    const collectionName = pathParts[pathParts.length - 2];
-
    // Create the slug and slugAsParams
-    const slug = `/${collectionName}/${base}`;
+    const slug = `/pages/${base}`;
    const slugAsParams = base;

    return {
@ -306,12 +277,8 @@ export const releases = defineCollection({
    const { locale, base } = extractLocaleAndBase(fileName);
    // console.log(`release processed: ${fileName}, base=${base}, locale=${locale}`);

-    // Get the collection name (e.g., "release")
-    const pathParts = data._meta.path.split(path.sep);
-    const collectionName = pathParts[pathParts.length - 2];
-
    // Create the slug and slugAsParams
-    const slug = `/${collectionName}/${base}`;
+    const slug = `/release/${base}`;
    const slugAsParams = base;

    return {
@ -325,6 +292,32 @@ export const releases = defineCollection({
  }
 });

+/**
+ * Helper function to extract locale and base name from filename
+ * Handles filename formats:
+ * - name -> locale: DEFAULT_LOCALE, base: name
+ * - name.zh -> locale: zh, base: name
+ * 
+ * @param fileName Filename without extension (already has .mdx removed)
+ * @returns Object with locale and base name
+ */
+function extractLocaleAndBase(fileName: string): { locale: string; base: string } {
+  // Split filename into parts
+  const parts = fileName.split('.');
+
+  if (parts.length === 1) {
+    // Simple filename without locale: xxx
+    return { locale: DEFAULT_LOCALE, base: parts[0] };
+  } else if (parts.length === 2 && LOCALES.includes(parts[1])) {
+    // Filename with locale: xxx.zh
+    return { locale: parts[1], base: parts[0] };
+  } else {
+    // Unexpected format, use first part as base and default locale
+    console.warn(`Unexpected filename format: ${fileName}`);
+    return { locale: DEFAULT_LOCALE, base: parts[0] };
+  }
+}
+
 export default defineConfig({
  collections: [docs, metas, authors, categories, posts, pages, releases]
 });
--- a/src/app/robots.ts
+++ b/src/app/robots.ts
@ -1,5 +1,5 @@
 import { MetadataRoute } from 'next';
-import { getBaseUrl } from './lib/urls/urls';
+import { getBaseUrl } from '../lib/urls/urls';

 export default function robots(): MetadataRoute.Robots {
  return {
--- a/src/app/sitemap.ts
+++ b/src/app/sitemap.ts
@ -0,0 +1,101 @@
+import { getLocalePathname } from '@/i18n/navigation';
+import { routing } from '@/i18n/routing';
+import { source } from '@/lib/docs/source';
+import { allCategories, allPosts } from 'content-collections';
+import { MetadataRoute } from 'next';
+import { Locale } from 'next-intl';
+import { getBaseUrl } from '../lib/urls/urls';
+
+type Href = Parameters<typeof getLocalePathname>[0]['href'];
+
+/**
+ * static routes for sitemap, you may change the routes for your own
+ */
+const staticRoutes = [
+  '/',
+  '/pricing',
+  '/blog',
+  '/docs',
+  '/about',
+  '/contact',
+  '/waitlist',
+  '/changelog',
+  '/privacy',
+  '/terms',
+  '/cookie',
+  '/auth/login',
+  '/auth/register',
+];
+
+/**
+ * Generate a sitemap for the website
+ * 
+ * https://nextjs.org/docs/app/api-reference/functions/generate-sitemaps
+ * https://github.com/javayhu/cnblocks/blob/main/app/sitemap.ts
+ */
+export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
+  const sitemapList: MetadataRoute.Sitemap = []; // final result
+
+  // add static routes
+  sitemapList.push(...staticRoutes.flatMap((route) => {
+    return routing.locales.map((locale) => ({
+      url: getUrl(route, locale),
+      lastModified: new Date(),
+      priority: 1,
+      changeFrequency: 'weekly' as const
+    }));
+  }));
+
+  // add categories
+  sitemapList.push(...allCategories.flatMap((category: { slug: string }) =>
+    routing.locales.map((locale) => ({
+      url: getUrl(`/blog/category/${category.slug}`, locale),
+      lastModified: new Date(),
+      priority: 0.8,
+      changeFrequency: 'weekly' as const
+    }))
+  ));
+
+  // add posts
+  sitemapList.push(...allPosts.flatMap((post: { slugAsParams: string }) =>
+    routing.locales.map((locale) => ({
+      url: getUrl(`/blog/${post.slugAsParams}`, locale),
+      lastModified: new Date(),
+      priority: 0.8,
+      changeFrequency: 'weekly' as const
+    }))
+  ));
+
+  // add docs
+  const docsParams = source.generateParams();
+  sitemapList.push(...docsParams.flatMap(param =>
+    routing.locales.map((locale) => ({
+      url: getUrl(`/docs/${param.slug.join('/')}`, locale),
+      lastModified: new Date(),
+      priority: 0.8,
+      changeFrequency: 'weekly' as const
+    }))
+  ));
+
+  return sitemapList;
+}
+
+function getUrl(href: Href, locale: Locale) {
+  const pathname = getLocalePathname({ locale, href });
+  return getBaseUrl() + pathname;
+}
+
+/**
+ * https://next-intl.dev/docs/environments/actions-metadata-route-handlers#sitemap
+ * https://github.com/amannn/next-intl/blob/main/examples/example-app-router/src/app/sitemap.ts
+ */
+function getEntries(href: Href) {
+  return routing.locales.map((locale) => ({
+    url: getUrl(href, locale),
+    alternates: {
+      languages: Object.fromEntries(
+        routing.locales.map((cur) => [cur, getUrl(href, cur)])
+      ),
+    },
+  }));
+}
--- a/src/lib/auth.ts
+++ b/src/lib/auth.ts
@ -1,15 +1,14 @@
 import db from '@/db/index';
 import { account, session, user, verification } from '@/db/schema';
 import { defaultMessages } from '@/i18n/messages';
+import { LOCALE_COOKIE_NAME, routing } from '@/i18n/routing';
 import { sendEmail } from '@/mail';
 import { betterAuth } from 'better-auth';
 import { drizzleAdapter } from 'better-auth/adapters/drizzle';
 import { admin } from 'better-auth/plugins';
-import { getUrlWithLocale } from './urls/urls';
-import { routing } from '@/i18n/routing';
-import { LOCALE_COOKIE_NAME } from '@/i18n/routing';
-import { Locale } from 'next-intl';
 import { parse as parseCookies } from 'cookie';
+import { Locale } from 'next-intl';
+import { getUrlWithLocaleInCallbackUrl } from './urls/urls';

 /**
 * https://www.better-auth.com/docs/reference/options
@ -50,7 +49,7 @@ export const auth = betterAuth({
    // https://www.better-auth.com/docs/authentication/email-password#forget-password
    async sendResetPassword({ user, url }, request) {
      const locale = getLocaleFromRequest(request);
-      const localizedUrl = getUrlWithLocale(url, locale);
+      const localizedUrl = getUrlWithLocaleInCallbackUrl(url, locale);

      await sendEmail({
        to: user.email,
@ -69,7 +68,7 @@ export const auth = betterAuth({
    // https://www.better-auth.com/docs/authentication/email-password#require-email-verification
    sendVerificationEmail: async ({ user, url, token }, request) => {
      const locale = getLocaleFromRequest(request);
-      const localizedUrl = getUrlWithLocale(url, locale);
+      const localizedUrl = getUrlWithLocaleInCallbackUrl(url, locale);

      await sendEmail({
        to: user.email,
--- a/src/lib/urls/urls.ts
+++ b/src/lib/urls/urls.ts
@ -40,7 +40,7 @@ export function getBaseUrlWithLocale(locale?: Locale | null): string {
 * @param locale - The locale to add to the callbackURL
 * @returns The URL with locale added to callbackURL if necessary
 */
-export function getUrlWithLocale(url: string, locale: Locale): string {
+export function getUrlWithLocaleInCallbackUrl(url: string, locale: Locale): string {
  // If we shouldn't append locale, return original URL
  if (!shouldAppendLocale(locale)) {
    return url;
--- a/src/sitemap.ts
+++ b/src/sitemap.ts
@ -1,36 +0,0 @@
-import { MetadataRoute } from 'next';
-import { routing } from '@/i18n/routing';
-import { getLocalePathname } from '@/i18n/navigation';
-import { getBaseUrl } from './lib/urls/urls';
-import { Locale } from 'next-intl';
-
-/**
- * https://github.com/javayhu/cnblocks/blob/main/app/sitemap.ts
- */
-export default function sitemap(): MetadataRoute.Sitemap {
-  return [...getEntries('/')];
-}
-
-type Href = Parameters<typeof getLocalePathname>[0]['href'];
-
-/**
- * TODO: add more entries for the sitemap
- * 
- * https://next-intl.dev/docs/environments/actions-metadata-route-handlers#sitemap
- * https://github.com/amannn/next-intl/blob/main/examples/example-app-router/src/app/sitemap.ts
- */
-function getEntries(href: Href) {
-  return routing.locales.map((locale) => ({
-    url: getUrl(href, locale),
-    alternates: {
-      languages: Object.fromEntries(
-        routing.locales.map((cur) => [cur, getUrl(href, cur)])
-      ),
-    },
-  }));
-}
-
-function getUrl(href: Href, locale: Locale) {
-  const pathname = getLocalePathname({ locale, href });
-  return getBaseUrl() + pathname;
-}