chore: add robots.tx and update sitemap.ts

src/robots.ts

@@ -0,0 +1,13 @@
+import { MetadataRoute } from "next";
+import { getBaseUrl } from "./lib/urls/get-base-url";
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: {
+      userAgent: '*',
+      allow: '/',
+      disallow: ['/api/'], 
+    },
+    sitemap: `${getBaseUrl()}/sitemap.xml`,
+  }
+}

src/sitemap.ts

@@ -3,6 +3,9 @@ import { routing, Locale } from '@/i18n/routing';
 import { getLocalePathname } from '@/i18n/navigation';
 import { siteConfig } from './config/site';
 
+/**
+ * https://github.com/javayhu/cnblocks/blob/main/app/sitemap.ts
+ */
 export default function sitemap(): MetadataRoute.Sitemap {
     return [...getEntries('/')];
 }