Add dynamic robots.txt generation and update sitemap handling

- Remove static robots.txt file.
- Implement dynamic robots.txt generator script (`generate_robots_txt.js`) to allow hostname flexibility.
- Update sitemap generation script to also use dynamic hostname from arguments.
- Modify Vite config to include automated generation of both sitemap and robots.txt during build.
- Add `public/robots.txt` to `.gitignore`.

.gitignore

@@ -3,6 +3,7 @@
 **/.wrangler/
 /.idea/
 public/sitemap.xml
+public/robots.txt
 .dev.vars
 secrets.json
 wrangler.dev.jsonc

public/robots.txt

@@ -1,7 +0,0 @@
-User-agent: *
-Allow: /
-Allow: /connect
-Disallow: /api
-Disallow: /assets
-
-Sitemap: https://geoff.seemueller.io/sitemap.xml

scripts/generate_robots_txt.js

@@ -0,0 +1,36 @@
+#!/usr/bin/env bun
+
+import fs from "fs";
+import {parseArgs} from "util";
+
+
+const {positionals} = parseArgs({
+  args: Bun.argv,
+  options: {},
+  strict: true,
+  allowPositionals: true,
+});
+
+const currentDate = new Date().toISOString().split("T")[0];
+
+const host = positionals[2];
+
+const robotsTxtTemplate = `
+User-agent: *
+Allow: /
+Allow: /connect
+Disallow: /api
+Disallow: /assets
+
+Sitemap: https://${host}/sitemap.xml
+`;
+
+const robotsTxtPath = "./public/robots.txt";
+
+fs.writeFile(robotsTxtPath, robotsTxtTemplate, (err) => {
+  if (err) {
+    console.error("Error writing robots.txt:", err);
+    process.exit(1);
+  }
+  console.log("robots.txt created successfully:", currentDate);
+});

scripts/generate_sitemap.js

@@ -1,19 +1,30 @@
 #!/usr/bin/env bun
 
 import fs from "fs";
+import {parseArgs} from "util";
+
+
+const {positionals} = parseArgs({
+  args: Bun.argv,
+  options: {},
+  strict: true,
+  allowPositionals: true,
+});
 
 const currentDate = new Date().toISOString().split("T")[0];
 
+const host = positionals[2];
+
 const sitemapTemplate = `<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 ">
     <url>
-        <loc>https://geoff.seemueller.io/</loc>
+        <loc>https://${host}/</loc>
         <lastmod>${currentDate}</lastmod>
         <priority>1.0</priority>
     </url>
     <url>
-        <loc>https://geoff.seemueller.io/connect</loc>
+        <loc>https://open-gsio.seemueller.workers.dev/connect</loc>
         <lastmod>${currentDate}</lastmod>
         <priority>0.7</priority>
     </url>

vite.config.ts

@@ -3,14 +3,18 @@ import vike from "vike/plugin";
 import { defineConfig } from "vite";
 import * as child_process from "node:child_process";
 
+const APP_FQDN = "open-gsio.seemueller.workers.dev";
+
 export default defineConfig(({ command }) => {
   const customPlugins = [
     {
       name: "sitemap-generator",
       buildStart(options) {
         if (command === "build") {
-          child_process.execSync("./scripts/generate_sitemap.js");
+          child_process.execSync("./scripts/generate_sitemap.js " + APP_FQDN);
           console.log("Generated Sitemap -> public/sitemap.xml");
+          child_process.execSync("./scripts/generate_robots_txt.js " + APP_FQDN);
+          console.log("Generated robots.txt -> public/robots.txt");
         }
       },
     },
@@ -40,6 +44,7 @@ export default defineConfig(({ command }) => {
     build: {
       emitAssets: false,
       sourcemap: false,
+      minify: true,
       target: ["es2020", "edge88", "firefox78", "chrome87", "safari13"],
       rollupOptions: {
         output: {