add more SEO support

astro.config.mjs

@@ -11,9 +11,11 @@ import mdx from '@astrojs/mdx';
 import icon from 'astro-icon';
 import pagefind from "astro-pagefind";
 
+import sitemap from '@astrojs/sitemap';
+
 // https://astro.build/config
 export default defineConfig({
-  integrations: [tailwind(), mdx(), icon(), pagefind()],
+  integrations: [tailwind(), mdx(), icon(), pagefind(), sitemap()],
   site: 'https://nfdi4plants.github.io',
   base: '/arc-website/', // will kill pagefind integration without trailing slash
   markdown: {

package.json

@@ -12,6 +12,7 @@
   "dependencies": {
     "@astrojs/check": "^0.9.3",
     "@astrojs/mdx": "^3.1.5",
+    "@astrojs/sitemap": "^3.1.6",
     "@astrojs/tailwind": "^5.1.0",
     "@iconify-json/mdi": "^1.2.0",
     "@tailwindcss/typography": "^0.5.15",

src/pages/robots.txt.ts

@@ -0,0 +1,13 @@
+import type { APIRoute } from 'astro';
+
+const getRobotsTxt = (sitemapURL: URL) => `
+User-agent: *
+Allow: /
+
+Sitemap: ${sitemapURL.href}
+`;
+
+export const GET: APIRoute = ({ site }) => {
+  const sitemapURL = new URL('sitemap-index.xml', site);
+  return new Response(getRobotsTxt(sitemapURL));
+};