add "noindex, nofollow" HTML robots meta-tag to documentation pages

 - Customize the "readthedocs" mkdocs theme: https://www.mkdocs.org/user-guide/styling-your-docs/#customizing-a-theme
 - Adds a '<meta name="robots" content="noindex, nofollow">' HTML tag on each page
 - Do not include robots directive on readthedocs.org, only in local builds

doc/custom_theme/main.html

@@ -0,0 +1,25 @@
+{% extends "base.html" %}
+
+{#
+The entry point for the ReadTheDocs Theme.
+ 
+Any theme customisations should override this file to redefine blocks defined in
+the various templates. The custom theme should only need to define a main.html
+which `{% extends "base.html" %}` and defines various blocks which will replace
+the blocks defined in base.html and its included child templates.
+#}
+
+{%- block site_meta %}
+<meta charset="utf-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+{%- if 'media.readthedocs.org' not in config.extra_css[0] %}
+<meta name="robots" content="noindex, nofollow">
+{%- endif %}
+
+{% if page and page.is_homepage %}<meta name="description" content="{{ config.site_description }}">{% endif %}
+{% if config.site_author %}<meta name="author" content="{{ config.site_author }}">{% endif %}
+{% if config.site_favicon %}<link rel="shortcut icon" href="{{ config.site_favicon|url }}">
+{% else %}<link rel="shortcut icon" href="{{ 'img/favicon.ico'|url }}">{% endif %}
+{%- endblock %}

mkdocs.yml

@@ -2,7 +2,9 @@ site_name: Shaarli Documentation
 repo_url: https://github.com/shaarli/Shaarli
 edit_uri: edit/master/doc/md
 site_description: The personal, minimalist, super-fast, database free, bookmarking service
-theme: readthedocs
+theme:
+  name: readthedocs
+  custom_dir: doc/custom_theme/
 docs_dir: doc/md
 site_dir: doc/html
 # Disable strict mode until ReadTheDocs provides up-to-date MkDocs settings: