From 17063143abc1394f80181f208a268ec604a7184b Mon Sep 17 00:00:00 2001 From: Jim Shepich III Date: Wed, 4 Feb 2026 10:07:58 -0500 Subject: [PATCH] Added robots.txt --- config.yaml | 8 +++++ site/assets/css/theme.css | 7 ++++- site/robots.txt | 50 +++++++++++++++++++++++++++++++ site/templates/pages/default.html | 1 + 4 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 site/robots.txt diff --git a/config.yaml b/config.yaml index c064bbd..ae00c7d 100644 --- a/config.yaml +++ b/config.yaml @@ -12,6 +12,14 @@ sites: - ./pages/*.md template_selections: article: templates.components.simple_article + + robots_txt: + title: robots.txt + base_url: http://localhost:8000/robots.txt + web_root: ./dist + build_cache: ./site + assets: + - robots.txt resume: title: Resume diff --git a/site/assets/css/theme.css b/site/assets/css/theme.css index 36ac776..11e6001 100644 --- a/site/assets/css/theme.css +++ b/site/assets/css/theme.css @@ -230,7 +230,7 @@ img { width: 50vw; } -li::marker { +ul > li::marker { content: '⍟ '; /* TODO: Decide on a default li marker for the theme*/ } @@ -248,6 +248,11 @@ ul:has(> .blog-archive-li){ border-bottom-style: solid; } +.blog-archive-li::marker{ + display: none; + content: none; +} + .blog-archive-li > time { float: right; } \ No newline at end of file diff --git a/site/robots.txt b/site/robots.txt new file mode 100644 index 0000000..35dcfd0 --- /dev/null +++ b/site/robots.txt @@ -0,0 +1,50 @@ +User-agent *: +Crawl-delay: 5 +Content-Signal: ai-train=no, search=yes, ai-input=no +Disallow: /assets/ +Disallow: /dogma-jimfinium/assets/ +Disallow: /*/assets/ +Disallow: /shepich_resume.pdf + + +# Block AI bots. +User-agent: GPTBot +User-agent: OAI-SearchBot +User-agent: ChatGPT-User +User-agent: anthropic-ai +User-agent: ClaudeBot +User-agent: claude-web +User-agent: PerplexityBot +User-agent: Perplexity-User +User-agent: Google-Extended +User-agent: Googlebot +User-agent: Bingbot +User-agent: Amazonbot +User-agent: Applebot +User-agent: Applebot-Extended +User-agent: FacebookBot +User-agent: meta-externalagent +User-agent: LinkedInBot +User-agent: Bytespider +User-agent: DuckAssistBot +User-agent: cohere-ai +User-agent: AI2Bot +User-agent: CCBot +User-agent: Diffbot +User-agent: omgili +User-agent: Timpibot +User-agent: YouBot +User-agent: MistralAI-User +User-agent: GoogleAgent-Mariner +User-agent: Standard Chrome UA +User-agent: No known UA +User-agent: Knowatoa AI Search Console +User-agent: Merkle robots.txt Tester +User-agent: Block GPT model training +User-agent: Allow ChatGPT user traffic +User-agent: ChatGPT Atlas +User-agent: OpenAI Operator +User-agent: Google Project Mariner +User-agent: Anthropic Computer Use +User-agent: xAI Grok +Disallow: / diff --git a/site/templates/pages/default.html b/site/templates/pages/default.html index 2080d69..b8c4fe6 100644 --- a/site/templates/pages/default.html +++ b/site/templates/pages/default.html @@ -3,6 +3,7 @@ + {templates.partials.default_css} {templates.partials.header}