diff --git a/custom/public/robots.txt b/custom/public/robots.txt
new file mode 100644
index 0000000..28c92d8
--- /dev/null
+++ b/custom/public/robots.txt
@@ -0,0 +1,86 @@
+User-agent: MJ12bot
+Disallow: /
+
+
+User-agent: AdsBot-Google
+Disallow: /
+
+
+User-agent: Mediapartners*
+Disallow: /
+
+
+User-agent: SemrushBot*
+Disallow: /
+
+
+User-agent: *
+
+Disallow: /api/*
+Disallow: /avatars
+Disallow: /user/*
+Disallow: /*/*/src/commit/*
+Disallow: /*/*/commit/*
+Disallow: /*/*/*/refs/*
+Disallow: /*/*/*/star
+Disallow: /*/*/*/watch
+Disallow: /*/*/labels
+Disallow: /*/*/activity/*
+Disallow: /vendor/*
+Disallow: /swagger.*.json
+
+Disallow: /*?lang=
+
+Disallow: */pulse
+Disallow: */tree/*
+Disallow: */blob/*
+Disallow: */wiki/
+Disallow: /gist/
+Disallow: /oembed
+Disallow: */forks
+Disallow: */stars
+Disallow: */download
+Disallow: */revisions
+Disallow: */issues/new
+Disallow: */issues/search
+Disallow: */commits/
+Disallow: */commits/*?author
+Disallow: */commits/*?path
+Disallow: */branches
+Disallow: */tags
+Disallow: */contributors
+Disallow: */comments
+Disallow: */tarball/
+Disallow: */zipball/
+Disallow: */raw/*
+Disallow: */followers
+Disallow: */following
+Disallow: */archive/
+Disallow: */blame/
+Disallow: */watchers
+Disallow: */network
+Disallow: */graphs
+Disallow: */raw/
+Disallow: */compare/
+Disallow: */cache/
+Disallow: */blame/
+Disallow: */network
+Disallow: */graphs
+Disallow: */raw/
+Disallow: */compare/
+Disallow: */cache/
+Disallow: /.git/
+Disallow: */.git/
+Disallow: /*.git$
+Disallow: /*/sitemap.xml
+Disallow: /search/advanced
+Disallow: /search
+Disallow: */search
+Disallow: /*q=
+Disallow: /*.atom
+Allow: /*/*/tree/master
+Allow: /*/*/tree/main
+Allow: /*/*/blob/master
+Allow: /*/*/blob/main
+
+Crawl-delay: 1