Inital Commit

Function Source Files/botocore/docs/__init__.py

@@ -0,0 +1,54 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import os
+
+from botocore.docs.service import ServiceDocumenter
+
+DEPRECATED_SERVICE_NAMES = {'sms-voice'}
+
+
+def generate_docs(root_dir, session):
+    """Generates the reference documentation for botocore
+
+    This will go through every available AWS service and output ReSTructured
+    text files documenting each service.
+
+    :param root_dir: The directory to write the reference files to. Each
+        service's reference documentation is loacated at
+        root_dir/reference/services/service-name.rst
+    """
+    # Create the root directory where all service docs live.
+    services_dir_path = os.path.join(root_dir, 'reference', 'services')
+    if not os.path.exists(services_dir_path):
+        os.makedirs(services_dir_path)
+
+    # Prevents deprecated service names from being generated in docs.
+    available_services = [
+        service
+        for service in session.get_available_services()
+        if service not in DEPRECATED_SERVICE_NAMES
+    ]
+
+    # Generate reference docs and write them out.
+    for service_name in available_services:
+        docs = ServiceDocumenter(
+            service_name, session, services_dir_path
+        ).document_service()
+
+        # Write the main service documentation page.
+        # Path: <root>/reference/services/<service>/index.rst
+        service_file_path = os.path.join(
+            services_dir_path, f'{service_name}.rst'
+        )
+        with open(service_file_path, 'wb') as f:
+            f.write(docs)

Function Source Files/botocore/docs/bcdoc/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+__version__ = '0.16.0'

Function Source Files/botocore/docs/bcdoc/docstringparser.py

@@ -0,0 +1,315 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from html.parser import HTMLParser
+from itertools import zip_longest
+
+PRIORITY_PARENT_TAGS = ('code', 'a')
+OMIT_NESTED_TAGS = ('span', 'i', 'code', 'a')
+OMIT_SELF_TAGS = ('i', 'b')
+HTML_BLOCK_DISPLAY_TAGS = ('p', 'note', 'ul', 'li')
+
+
+class DocStringParser(HTMLParser):
+    """
+    A simple HTML parser.  Focused on converting the subset of HTML
+    that appears in the documentation strings of the JSON models into
+    simple ReST format.
+    """
+
+    def __init__(self, doc):
+        self.tree = None
+        self.doc = doc
+        super().__init__()
+
+    def reset(self):
+        HTMLParser.reset(self)
+        self.tree = HTMLTree(self.doc)
+
+    def feed(self, data):
+        super().feed(data)
+        self.tree.write()
+        self.tree = HTMLTree(self.doc)
+
+    def close(self):
+        super().close()
+        # Write if there is anything remaining.
+        self.tree.write()
+        self.tree = HTMLTree(self.doc)
+
+    def handle_starttag(self, tag, attrs):
+        self.tree.add_tag(tag, attrs=attrs)
+
+    def handle_endtag(self, tag):
+        self.tree.add_tag(tag, is_start=False)
+
+    def handle_data(self, data):
+        self.tree.add_data(data)
+
+
+class HTMLTree:
+    """
+    A tree which handles HTML nodes. Designed to work with a python HTML parser,
+    meaning that the current_node will be the most recently opened tag. When
+    a tag is closed, the current_node moves up to the parent node.
+    """
+
+    def __init__(self, doc):
+        self.doc = doc
+        self.head = StemNode()
+        self.current_node = self.head
+        self.unhandled_tags = []
+
+    def add_tag(self, tag, attrs=None, is_start=True):
+        if not self._doc_has_handler(tag, is_start):
+            self.unhandled_tags.append(tag)
+            return
+
+        if is_start:
+            node = TagNode(tag, attrs)
+            self.current_node.add_child(node)
+            self.current_node = node
+        else:
+            self.current_node = self.current_node.parent
+
+    def _doc_has_handler(self, tag, is_start):
+        if is_start:
+            handler_name = 'start_%s' % tag
+        else:
+            handler_name = 'end_%s' % tag
+
+        return hasattr(self.doc.style, handler_name)
+
+    def add_data(self, data):
+        self.current_node.add_child(DataNode(data))
+
+    def write(self):
+        self.head.write(self.doc)
+
+
+class Node:
+    def __init__(self, parent=None):
+        self.parent = parent
+
+    def write(self, doc):
+        raise NotImplementedError
+
+
+class StemNode(Node):
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self.children = []
+
+    def add_child(self, child):
+        child.parent = self
+        self.children.append(child)
+
+    def write(self, doc):
+        self.collapse_whitespace()
+        self._write_children(doc)
+
+    def _write_children(self, doc):
+        for child, next_child in zip_longest(self.children, self.children[1:]):
+            if isinstance(child, TagNode) and next_child is not None:
+                child.write(doc, next_child)
+            else:
+                child.write(doc)
+
+    def is_whitespace(self):
+        return all(child.is_whitespace() for child in self.children)
+
+    def startswith_whitespace(self):
+        return self.children and self.children[0].startswith_whitespace()
+
+    def endswith_whitespace(self):
+        return self.children and self.children[-1].endswith_whitespace()
+
+    def lstrip(self):
+        while self.children and self.children[0].is_whitespace():
+            self.children = self.children[1:]
+        if self.children:
+            self.children[0].lstrip()
+
+    def rstrip(self):
+        while self.children and self.children[-1].is_whitespace():
+            self.children = self.children[:-1]
+        if self.children:
+            self.children[-1].rstrip()
+
+    def collapse_whitespace(self):
+        """Remove collapsible white-space from HTML.
+
+        HTML in docstrings often contains extraneous white-space around tags,
+        for readability. Browsers would collapse this white-space before
+        rendering. If not removed before conversion to RST where white-space is
+        part of the syntax, for example for indentation, it can result in
+        incorrect output.
+        """
+        self.lstrip()
+        self.rstrip()
+        for child in self.children:
+            child.collapse_whitespace()
+
+
+class TagNode(StemNode):
+    """
+    A generic Tag node. It will verify that handlers exist before writing.
+    """
+
+    def __init__(self, tag, attrs=None, parent=None):
+        super().__init__(parent)
+        self.attrs = attrs
+        self.tag = tag
+
+    def _has_nested_tags(self):
+        # Returns True if any children are TagNodes and False otherwise.
+        return any(isinstance(child, TagNode) for child in self.children)
+
+    def write(self, doc, next_child=None):
+        prioritize_nested_tags = (
+            self.tag in OMIT_SELF_TAGS and self._has_nested_tags()
+        )
+        prioritize_parent_tag = (
+            isinstance(self.parent, TagNode)
+            and self.parent.tag in PRIORITY_PARENT_TAGS
+            and self.tag in OMIT_NESTED_TAGS
+        )
+        if prioritize_nested_tags or prioritize_parent_tag:
+            self._write_children(doc)
+            return
+
+        self._write_start(doc)
+        self._write_children(doc)
+        self._write_end(doc, next_child)
+
+    def collapse_whitespace(self):
+        """Remove collapsible white-space.
+
+        All tags collapse internal whitespace. Block-display HTML tags also
+        strip all leading and trailing whitespace.
+
+        Approximately follows the specification used in browsers:
+        https://www.w3.org/TR/css-text-3/#white-space-rules
+        https://developer.mozilla.org/en-US/docs/Web/API/Document_Object_Model/Whitespace
+        """
+        if self.tag in HTML_BLOCK_DISPLAY_TAGS:
+            self.lstrip()
+            self.rstrip()
+        # Collapse whitespace in situations like ``</b> <i> foo</i>`` into
+        # ``</b><i> foo</i>``.
+        for prev, cur in zip(self.children[:-1], self.children[1:]):
+            if (
+                isinstance(prev, DataNode)
+                and prev.endswith_whitespace()
+                and cur.startswith_whitespace()
+            ):
+                cur.lstrip()
+        # Same logic, but for situations like ``<b>bar </b> <i>``:
+        for cur, nxt in zip(self.children[:-1], self.children[1:]):
+            if (
+                isinstance(nxt, DataNode)
+                and cur.endswith_whitespace()
+                and nxt.startswith_whitespace()
+            ):
+                cur.rstrip()
+        # Recurse into children
+        for child in self.children:
+            child.collapse_whitespace()
+
+    def _write_start(self, doc):
+        handler_name = 'start_%s' % self.tag
+        if hasattr(doc.style, handler_name):
+            getattr(doc.style, handler_name)(self.attrs)
+
+    def _write_end(self, doc, next_child):
+        handler_name = 'end_%s' % self.tag
+        if hasattr(doc.style, handler_name):
+            if handler_name == 'end_a':
+                # We use lookahead to determine if a space is needed after a link node
+                getattr(doc.style, handler_name)(next_child)
+            else:
+                getattr(doc.style, handler_name)()
+
+
+class DataNode(Node):
+    """
+    A Node that contains only string data.
+    """
+
+    def __init__(self, data, parent=None):
+        super().__init__(parent)
+        if not isinstance(data, str):
+            raise ValueError("Expecting string type, %s given." % type(data))
+        self._leading_whitespace = ''
+        self._trailing_whitespace = ''
+        self._stripped_data = ''
+        if data == '':
+            return
+        if data.isspace():
+            self._trailing_whitespace = data
+            return
+        first_non_space = next(
+            idx for idx, ch in enumerate(data) if not ch.isspace()
+        )
+        last_non_space = len(data) - next(
+            idx for idx, ch in enumerate(reversed(data)) if not ch.isspace()
+        )
+        self._leading_whitespace = data[:first_non_space]
+        self._trailing_whitespace = data[last_non_space:]
+        self._stripped_data = data[first_non_space:last_non_space]
+
+    @property
+    def data(self):
+        return (
+            f'{self._leading_whitespace}{self._stripped_data}'
+            f'{self._trailing_whitespace}'
+        )
+
+    def is_whitespace(self):
+        return self._stripped_data == '' and (
+            self._leading_whitespace != '' or self._trailing_whitespace != ''
+        )
+
+    def startswith_whitespace(self):
+        return self._leading_whitespace != '' or (
+            self._stripped_data == '' and self._trailing_whitespace != ''
+        )
+
+    def endswith_whitespace(self):
+        return self._trailing_whitespace != '' or (
+            self._stripped_data == '' and self._leading_whitespace != ''
+        )
+
+    def lstrip(self):
+        if self._leading_whitespace != '':
+            self._leading_whitespace = ''
+        elif self._stripped_data == '':
+            self.rstrip()
+
+    def rstrip(self):
+        if self._trailing_whitespace != '':
+            self._trailing_whitespace = ''
+        elif self._stripped_data == '':
+            self.lstrip()
+
+    def collapse_whitespace(self):
+        """Noop, ``DataNode.write`` always collapses whitespace"""
+        return
+
+    def write(self, doc):
+        words = doc.translate_words(self._stripped_data.split())
+        str_data = (
+            f'{self._leading_whitespace}{" ".join(words)}'
+            f'{self._trailing_whitespace}'
+        )
+        if str_data != '':
+            doc.handle_data(str_data)

Function Source Files/botocore/docs/bcdoc/restdoc.py

@@ -0,0 +1,282 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import logging
+import os
+import re
+
+from botocore.compat import OrderedDict
+from botocore.docs.bcdoc.docstringparser import DocStringParser
+from botocore.docs.bcdoc.style import ReSTStyle
+
+DEFAULT_AWS_DOCS_LINK = 'https://docs.aws.amazon.com/index.html'
+DOCUMENTATION_LINK_REGEX = re.compile(
+    r'`AWS API Documentation '
+    r'<https://docs.aws.amazon.com/goto/WebAPI/[a-z0-9-.]*/[a-zA-Z]*>`_'
+)
+LARGE_SECTION_MESSAGE = """
+
+    **{}**
+    ::
+
+        # This section is too large to render.
+        # Please see the AWS API Documentation linked below.
+
+    {}
+    """
+LOG = logging.getLogger('bcdocs')
+SECTION_LINE_LIMIT_CONFIG = {
+    'response-example': {'name': 'Response Syntax', 'line_limit': 1500},
+    'description': {'name': 'Response Structure', 'line_limit': 5000},
+    'request-example': {'name': 'Request Syntax', 'line_limit': 1500},
+    'request-params': {'name': 'Parameters', 'line_limit': 5000},
+}
+SECTION_METHOD_PATH_DEPTH = {
+    'client-api': 4,
+    'paginator-api': 3,
+    'waiter-api': 3,
+}
+
+
+class ReSTDocument:
+    def __init__(self, target='man'):
+        self.style = ReSTStyle(self)
+        self.target = target
+        self.parser = DocStringParser(self)
+        self.keep_data = True
+        self.do_translation = False
+        self.translation_map = {}
+        self.hrefs = {}
+        self._writes = []
+        self._last_doc_string = None
+
+    def _write(self, s):
+        if self.keep_data and s is not None:
+            self._writes.append(s)
+
+    def write(self, content):
+        """
+        Write content into the document.
+        """
+        self._write(content)
+
+    def writeln(self, content):
+        """
+        Write content on a newline.
+        """
+        self._write(f'{self.style.spaces()}{content}\n')
+
+    def peek_write(self):
+        """
+        Returns the last content written to the document without
+        removing it from the stack.
+        """
+        return self._writes[-1]
+
+    def pop_write(self):
+        """
+        Removes and returns the last content written to the stack.
+        """
+        return self._writes.pop() if len(self._writes) > 0 else None
+
+    def push_write(self, s):
+        """
+        Places new content on the stack.
+        """
+        self._writes.append(s)
+
+    def getvalue(self):
+        """
+        Returns the current content of the document as a string.
+        """
+        if self.hrefs:
+            self.style.new_paragraph()
+            for refname, link in self.hrefs.items():
+                self.style.link_target_definition(refname, link)
+        return ''.join(self._writes).encode('utf-8')
+
+    def translate_words(self, words):
+        return [self.translation_map.get(w, w) for w in words]
+
+    def handle_data(self, data):
+        if data and self.keep_data:
+            self._write(data)
+
+    def include_doc_string(self, doc_string):
+        if doc_string:
+            try:
+                start = len(self._writes)
+                self.parser.feed(doc_string)
+                self.parser.close()
+                end = len(self._writes)
+                self._last_doc_string = (start, end)
+            except Exception:
+                LOG.debug('Error parsing doc string', exc_info=True)
+                LOG.debug(doc_string)
+
+    def remove_last_doc_string(self):
+        # Removes all writes inserted by last doc string
+        if self._last_doc_string is not None:
+            start, end = self._last_doc_string
+            del self._writes[start:end]
+
+
+class DocumentStructure(ReSTDocument):
+    def __init__(self, name, section_names=None, target='man', context=None):
+        """Provides a Hierarichial structure to a ReSTDocument
+
+        You can write to it similiar to as you can to a ReSTDocument but
+        has an innate structure for more orginaztion and abstraction.
+
+        :param name: The name of the document
+        :param section_names: A list of sections to be included
+            in the document.
+        :param target: The target documentation of the Document structure
+        :param context: A dictionary of data to store with the strucuture. These
+            are only stored per section not the entire structure.
+        """
+        super().__init__(target=target)
+        self._name = name
+        self._structure = OrderedDict()
+        self._path = [self._name]
+        self._context = {}
+        if context is not None:
+            self._context = context
+        if section_names is not None:
+            self._generate_structure(section_names)
+
+    @property
+    def name(self):
+        """The name of the document structure"""
+        return self._name
+
+    @property
+    def path(self):
+        """
+        A list of where to find a particular document structure in the
+        overlying document structure.
+        """
+        return self._path
+
+    @path.setter
+    def path(self, value):
+        self._path = value
+
+    @property
+    def available_sections(self):
+        return list(self._structure)
+
+    @property
+    def context(self):
+        return self._context
+
+    def _generate_structure(self, section_names):
+        for section_name in section_names:
+            self.add_new_section(section_name)
+
+    def add_new_section(self, name, context=None):
+        """Adds a new section to the current document structure
+
+        This document structure will be considered a section to the
+        current document structure but will in itself be an entirely
+        new document structure that can be written to and have sections
+        as well
+
+        :param name: The name of the section.
+        :param context: A dictionary of data to store with the strucuture. These
+            are only stored per section not the entire structure.
+        :rtype: DocumentStructure
+        :returns: A new document structure to add to but lives as a section
+            to the document structure it was instantiated from.
+        """
+        # Add a new section
+        section = self.__class__(
+            name=name, target=self.target, context=context
+        )
+        section.path = self.path + [name]
+        # Indent the section apporpriately as well
+        section.style.indentation = self.style.indentation
+        section.translation_map = self.translation_map
+        section.hrefs = self.hrefs
+        self._structure[name] = section
+        return section
+
+    def get_section(self, name):
+        """Retrieve a section"""
+        return self._structure[name]
+
+    def delete_section(self, name):
+        """Delete a section"""
+        del self._structure[name]
+
+    def flush_structure(self, docs_link=None):
+        """Flushes a doc structure to a ReSTructed string
+
+        The document is flushed out in a DFS style where sections and their
+        subsections' values are added to the string as they are visited.
+        """
+        # We are at the root flush the links at the beginning of the
+        # document
+        path_length = len(self.path)
+        if path_length == 1:
+            if self.hrefs:
+                self.style.new_paragraph()
+                for refname, link in self.hrefs.items():
+                    self.style.link_target_definition(refname, link)
+        # Clear docs_link at the correct depth to prevent passing a non-related link.
+        elif path_length == SECTION_METHOD_PATH_DEPTH.get(self.path[1]):
+            docs_link = None
+        value = self.getvalue()
+        for name, section in self._structure.items():
+            # Checks is the AWS API Documentation link has been generated.
+            # If it has been generated, it gets passed as a the doc_link parameter.
+            match = DOCUMENTATION_LINK_REGEX.search(value.decode())
+            docs_link = (
+                f'{match.group(0)}\n\n'.encode() if match else docs_link
+            )
+            value += section.flush_structure(docs_link)
+
+        # Replace response/request sections if the line number exceeds our limit.
+        # The section is replaced with a message linking to AWS API Documentation.
+        line_count = len(value.splitlines())
+        section_config = SECTION_LINE_LIMIT_CONFIG.get(self.name)
+        aws_docs_link = (
+            docs_link.decode()
+            if docs_link is not None
+            else DEFAULT_AWS_DOCS_LINK
+        )
+        if section_config and line_count > section_config['line_limit']:
+            value = LARGE_SECTION_MESSAGE.format(
+                section_config['name'], aws_docs_link
+            ).encode()
+        return value
+
+    def getvalue(self):
+        return ''.join(self._writes).encode('utf-8')
+
+    def remove_all_sections(self):
+        self._structure = OrderedDict()
+
+    def clear_text(self):
+        self._writes = []
+
+    def add_title_section(self, title):
+        title_section = self.add_new_section('title')
+        title_section.style.h1(title)
+        return title_section
+
+    def write_to_file(self, full_path, file_name):
+        if not os.path.exists(full_path):
+            os.makedirs(full_path)
+        sub_resource_file_path = os.path.join(full_path, f'{file_name}.rst')
+        with open(sub_resource_file_path, 'wb') as f:
+            f.write(self.flush_structure())

Function Source Files/botocore/docs/bcdoc/style.py

@@ -0,0 +1,447 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+import logging
+
+logger = logging.getLogger('bcdocs')
+# Terminal punctuation where a space is not needed before.
+PUNCTUATION_CHARACTERS = ('.', ',', '?', '!', ':', ';')
+
+
+class BaseStyle:
+    def __init__(self, doc, indent_width=2):
+        self.doc = doc
+        self.indent_width = indent_width
+        self._indent = 0
+        self.keep_data = True
+
+    @property
+    def indentation(self):
+        return self._indent
+
+    @indentation.setter
+    def indentation(self, value):
+        self._indent = value
+
+    def new_paragraph(self):
+        return '\n%s' % self.spaces()
+
+    def indent(self):
+        self._indent += 1
+
+    def dedent(self):
+        if self._indent > 0:
+            self._indent -= 1
+
+    def spaces(self):
+        return ' ' * (self._indent * self.indent_width)
+
+    def bold(self, s):
+        return s
+
+    def ref(self, link, title=None):
+        return link
+
+    def h2(self, s):
+        return s
+
+    def h3(self, s):
+        return s
+
+    def underline(self, s):
+        return s
+
+    def italics(self, s):
+        return s
+
+    def add_trailing_space_to_previous_write(self):
+        # Adds a trailing space if none exists. This is mainly used for
+        # ensuring inline code and links are separated from surrounding text.
+        last_write = self.doc.pop_write()
+        if last_write is None:
+            last_write = ''
+        if last_write != '' and last_write[-1] != ' ':
+            last_write += ' '
+        self.doc.push_write(last_write)
+
+
+class ReSTStyle(BaseStyle):
+    def __init__(self, doc, indent_width=2):
+        BaseStyle.__init__(self, doc, indent_width)
+        self.do_p = True
+        self.a_href = None
+        self.list_depth = 0
+
+    def new_paragraph(self):
+        self.doc.write('\n\n%s' % self.spaces())
+
+    def new_line(self):
+        self.doc.write('\n%s' % self.spaces())
+
+    def _start_inline(self, markup):
+        # Insert space between any directly adjacent bold and italic inlines to
+        # avoid situations like ``**abc***def*``.
+        try:
+            last_write = self.doc.peek_write()
+        except IndexError:
+            pass
+        else:
+            if last_write in ('*', '**') and markup in ('*', '**'):
+                self.doc.write(' ')
+        self.doc.write(markup)
+
+    def _end_inline(self, markup):
+        # Remove empty and self-closing tags like ``<b></b>`` and ``<b/>``.
+        # If we simply translate that directly then we end up with something
+        # like ****, which rst will assume is a heading instead of an empty
+        # bold.
+        last_write = self.doc.pop_write()
+        if last_write == markup:
+            return
+        self.doc.push_write(last_write)
+        self.doc.write(markup)
+
+    def start_bold(self, attrs=None):
+        self._start_inline('**')
+
+    def end_bold(self):
+        self._end_inline('**')
+
+    def start_b(self, attrs=None):
+        self.doc.do_translation = True
+        self.start_bold(attrs)
+
+    def end_b(self):
+        self.doc.do_translation = False
+        self.end_bold()
+
+    def bold(self, s):
+        if s:
+            self.start_bold()
+            self.doc.write(s)
+            self.end_bold()
+
+    def ref(self, title, link=None):
+        if link is None:
+            link = title
+        self.doc.write(f':doc:`{title} <{link}>`')
+
+    def _heading(self, s, border_char):
+        border = border_char * len(s)
+        self.new_paragraph()
+        self.doc.write(f'{border}\n{s}\n{border}')
+        self.new_paragraph()
+
+    def h1(self, s):
+        self._heading(s, '*')
+
+    def h2(self, s):
+        self._heading(s, '=')
+
+    def h3(self, s):
+        self._heading(s, '-')
+
+    def start_italics(self, attrs=None):
+        self._start_inline('*')
+
+    def end_italics(self):
+        self._end_inline('*')
+
+    def italics(self, s):
+        if s:
+            self.start_italics()
+            self.doc.write(s)
+            self.end_italics()
+
+    def start_p(self, attrs=None):
+        if self.do_p:
+            self.doc.write('\n\n%s' % self.spaces())
+
+    def end_p(self):
+        if self.do_p:
+            self.doc.write('\n\n%s' % self.spaces())
+
+    def start_code(self, attrs=None):
+        self.doc.do_translation = True
+        self.add_trailing_space_to_previous_write()
+        self._start_inline('``')
+
+    def end_code(self):
+        self.doc.do_translation = False
+        self._end_inline('``')
+
+    def code(self, s):
+        if s:
+            self.start_code()
+            self.doc.write(s)
+            self.end_code()
+
+    def start_note(self, attrs=None):
+        self.new_paragraph()
+        self.doc.write('.. note::')
+        self.indent()
+        self.new_paragraph()
+
+    def end_note(self):
+        self.dedent()
+        self.new_paragraph()
+
+    def start_important(self, attrs=None):
+        self.new_paragraph()
+        self.doc.write('.. warning::')
+        self.indent()
+        self.new_paragraph()
+
+    def end_important(self):
+        self.dedent()
+        self.new_paragraph()
+
+    def start_danger(self, attrs=None):
+        self.new_paragraph()
+        self.doc.write('.. danger::')
+        self.indent()
+        self.new_paragraph()
+
+    def end_danger(self):
+        self.dedent()
+        self.new_paragraph()
+
+    def start_a(self, attrs=None):
+        # Write an empty space to guard against zero whitespace
+        # before an "a" tag. Example: hi<a>Example</a>
+        self.add_trailing_space_to_previous_write()
+        if attrs:
+            for attr_key, attr_value in attrs:
+                if attr_key == 'href':
+                    # Removes unnecessary whitespace around the href link.
+                    # Example: <a href=" http://example.com ">Example</a>
+                    self.a_href = attr_value.strip()
+                    self.doc.write('`')
+        else:
+            # There are some model documentation that
+            # looks like this: <a>DescribeInstances</a>.
+            # In this case we just write out an empty
+            # string.
+            self.doc.write(' ')
+        self.doc.do_translation = True
+
+    def link_target_definition(self, refname, link):
+        self.doc.writeln(f'.. _{refname}: {link}')
+
+    def sphinx_reference_label(self, label, text=None):
+        if text is None:
+            text = label
+        if self.doc.target == 'html':
+            self.doc.write(f':ref:`{text} <{label}>`')
+        else:
+            self.doc.write(text)
+
+    def _clean_link_text(self):
+        doc = self.doc
+        # Pop till we reach the link start character to retrieve link text.
+        last_write = doc.pop_write()
+        while not last_write.startswith('`'):
+            last_write = doc.pop_write() + last_write
+        if last_write != '':
+            # Remove whitespace from the start of link text.
+            if last_write.startswith('` '):
+                last_write = f'`{last_write[1:].lstrip(" ")}'
+            doc.push_write(last_write)
+
+    def end_a(self, next_child=None):
+        self.doc.do_translation = False
+        if self.a_href:
+            self._clean_link_text()
+            last_write = self.doc.pop_write()
+            last_write = last_write.rstrip(' ')
+            if last_write and last_write != '`':
+                if ':' in last_write:
+                    last_write = last_write.replace(':', r'\:')
+                self.doc.push_write(last_write)
+                self.doc.push_write(' <%s>`__' % self.a_href)
+            elif last_write == '`':
+                # Look at start_a().  It will do a self.doc.write('`')
+                # which is the start of the link title.  If that is the
+                # case then there was no link text.  We should just
+                # use an inline link.  The syntax of this is
+                # `<http://url>`_
+                self.doc.push_write('`<%s>`__' % self.a_href)
+            else:
+                self.doc.push_write(self.a_href)
+                self.doc.hrefs[self.a_href] = self.a_href
+                self.doc.write('`__')
+            self.a_href = None
+
+    def start_i(self, attrs=None):
+        self.doc.do_translation = True
+        self.start_italics()
+
+    def end_i(self):
+        self.doc.do_translation = False
+        self.end_italics()
+
+    def start_li(self, attrs=None):
+        self.new_line()
+        self.do_p = False
+        self.doc.write('* ')
+
+    def end_li(self):
+        self.do_p = True
+        self.new_line()
+
+    def li(self, s):
+        if s:
+            self.start_li()
+            self.doc.writeln(s)
+            self.end_li()
+
+    def start_ul(self, attrs=None):
+        if self.list_depth != 0:
+            self.indent()
+        self.list_depth += 1
+        self.new_paragraph()
+
+    def end_ul(self):
+        self.list_depth -= 1
+        if self.list_depth != 0:
+            self.dedent()
+        self.new_paragraph()
+
+    def start_ol(self, attrs=None):
+        # TODO: Need to control the bullets used for LI items
+        if self.list_depth != 0:
+            self.indent()
+        self.list_depth += 1
+        self.new_paragraph()
+
+    def end_ol(self):
+        self.list_depth -= 1
+        if self.list_depth != 0:
+            self.dedent()
+        self.new_paragraph()
+
+    def start_examples(self, attrs=None):
+        self.doc.keep_data = False
+
+    def end_examples(self):
+        self.doc.keep_data = True
+
+    def start_fullname(self, attrs=None):
+        self.doc.keep_data = False
+
+    def end_fullname(self):
+        self.doc.keep_data = True
+
+    def start_codeblock(self, attrs=None):
+        self.doc.write('::')
+        self.indent()
+        self.new_paragraph()
+
+    def end_codeblock(self):
+        self.dedent()
+        self.new_paragraph()
+
+    def codeblock(self, code):
+        """
+        Literal code blocks are introduced by ending a paragraph with
+        the special marker ::.  The literal block must be indented
+        (and, like all paragraphs, separated from the surrounding
+        ones by blank lines).
+        """
+        self.start_codeblock()
+        self.doc.writeln(code)
+        self.end_codeblock()
+
+    def toctree(self):
+        if self.doc.target == 'html':
+            self.doc.write('\n.. toctree::\n')
+            self.doc.write('  :maxdepth: 1\n')
+            self.doc.write('  :titlesonly:\n\n')
+        else:
+            self.start_ul()
+
+    def tocitem(self, item, file_name=None):
+        if self.doc.target == 'man':
+            self.li(item)
+        else:
+            if file_name:
+                self.doc.writeln('  %s' % file_name)
+            else:
+                self.doc.writeln('  %s' % item)
+
+    def hidden_toctree(self):
+        if self.doc.target == 'html':
+            self.doc.write('\n.. toctree::\n')
+            self.doc.write('  :maxdepth: 1\n')
+            self.doc.write('  :hidden:\n\n')
+
+    def hidden_tocitem(self, item):
+        if self.doc.target == 'html':
+            self.tocitem(item)
+
+    def table_of_contents(self, title=None, depth=None):
+        self.doc.write('.. contents:: ')
+        if title is not None:
+            self.doc.writeln(title)
+        if depth is not None:
+            self.doc.writeln('   :depth: %s' % depth)
+
+    def start_sphinx_py_class(self, class_name):
+        self.new_paragraph()
+        self.doc.write('.. py:class:: %s' % class_name)
+        self.indent()
+        self.new_paragraph()
+
+    def end_sphinx_py_class(self):
+        self.dedent()
+        self.new_paragraph()
+
+    def start_sphinx_py_method(self, method_name, parameters=None):
+        self.new_paragraph()
+        content = '.. py:method:: %s' % method_name
+        if parameters is not None:
+            content += '(%s)' % parameters
+        self.doc.write(content)
+        self.indent()
+        self.new_paragraph()
+
+    def end_sphinx_py_method(self):
+        self.dedent()
+        self.new_paragraph()
+
+    def start_sphinx_py_attr(self, attr_name):
+        self.new_paragraph()
+        self.doc.write('.. py:attribute:: %s' % attr_name)
+        self.indent()
+        self.new_paragraph()
+
+    def end_sphinx_py_attr(self):
+        self.dedent()
+        self.new_paragraph()
+
+    def write_py_doc_string(self, docstring):
+        docstring_lines = docstring.splitlines()
+        for docstring_line in docstring_lines:
+            self.doc.writeln(docstring_line)
+
+    def external_link(self, title, link):
+        if self.doc.target == 'html':
+            self.doc.write(f'`{title} <{link}>`_')
+        else:
+            self.doc.write(title)
+
+    def internal_link(self, title, page):
+        if self.doc.target == 'html':
+            self.doc.write(f':doc:`{title} <{page}>`')
+        else:
+            self.doc.write(title)

Function Source Files/botocore/docs/client.py

@@ -0,0 +1,455 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import os
+
+from botocore import xform_name
+from botocore.compat import OrderedDict
+from botocore.docs.bcdoc.restdoc import DocumentStructure
+from botocore.docs.example import ResponseExampleDocumenter
+from botocore.docs.method import (
+    document_custom_method,
+    document_model_driven_method,
+    get_instance_public_methods,
+)
+from botocore.docs.params import ResponseParamsDocumenter
+from botocore.docs.sharedexample import document_shared_examples
+from botocore.docs.utils import DocumentedShape, get_official_service_name
+
+
+def _allowlist_generate_presigned_url(method_name, service_name, **kwargs):
+    if method_name != 'generate_presigned_url':
+        return None
+    return service_name in ['s3']
+
+
+class ClientDocumenter:
+    _CLIENT_METHODS_FILTERS = [
+        _allowlist_generate_presigned_url,
+    ]
+
+    def __init__(self, client, root_docs_path, shared_examples=None):
+        self._client = client
+        self._client_class_name = self._client.__class__.__name__
+        self._root_docs_path = root_docs_path
+        self._shared_examples = shared_examples
+        if self._shared_examples is None:
+            self._shared_examples = {}
+        self._service_name = self._client.meta.service_model.service_name
+
+    def document_client(self, section):
+        """Documents a client and its methods
+
+        :param section: The section to write to.
+        """
+        self._add_title(section)
+        self._add_class_signature(section)
+        client_methods = self._get_client_methods()
+        self._add_client_intro(section, client_methods)
+        self._add_client_methods(client_methods)
+
+    def _get_client_methods(self):
+        client_methods = get_instance_public_methods(self._client)
+        return self._filter_client_methods(client_methods)
+
+    def _filter_client_methods(self, client_methods):
+        filtered_methods = {}
+        for method_name, method in client_methods.items():
+            include = self._filter_client_method(
+                method=method,
+                method_name=method_name,
+                service_name=self._service_name,
+            )
+            if include:
+                filtered_methods[method_name] = method
+        return filtered_methods
+
+    def _filter_client_method(self, **kwargs):
+        # Apply each filter to the method
+        for filter in self._CLIENT_METHODS_FILTERS:
+            filter_include = filter(**kwargs)
+            # Use the first non-None value returned by any of the filters
+            if filter_include is not None:
+                return filter_include
+        # Otherwise default to including it
+        return True
+
+    def _add_title(self, section):
+        section.style.h2('Client')
+
+    def _add_client_intro(self, section, client_methods):
+        section = section.add_new_section('intro')
+        # Write out the top level description for the client.
+        official_service_name = get_official_service_name(
+            self._client.meta.service_model
+        )
+        section.write(
+            f"A low-level client representing {official_service_name}"
+        )
+        section.style.new_line()
+        section.include_doc_string(
+            self._client.meta.service_model.documentation
+        )
+
+        # Write out the client example instantiation.
+        self._add_client_creation_example(section)
+
+        # List out all of the possible client methods.
+        section.style.dedent()
+        section.style.new_paragraph()
+        section.writeln('These are the available methods:')
+        section.style.toctree()
+        for method_name in sorted(client_methods):
+            section.style.tocitem(f'{self._service_name}/client/{method_name}')
+
+    def _add_class_signature(self, section):
+        section.style.start_sphinx_py_class(
+            class_name=f'{self._client_class_name}.Client'
+        )
+
+    def _add_client_creation_example(self, section):
+        section.style.start_codeblock()
+        section.style.new_line()
+        section.write(
+            'client = session.create_client(\'{service}\')'.format(
+                service=self._service_name
+            )
+        )
+        section.style.end_codeblock()
+
+    def _add_client_methods(self, client_methods):
+        for method_name in sorted(client_methods):
+            # Create a new DocumentStructure for each client method and add contents.
+            method_doc_structure = DocumentStructure(
+                method_name, target='html'
+            )
+            self._add_client_method(
+                method_doc_structure, method_name, client_methods[method_name]
+            )
+            # Write client methods in individual/nested files.
+            # Path: <root>/reference/services/<service>/client/<method_name>.rst
+            client_dir_path = os.path.join(
+                self._root_docs_path, self._service_name, 'client'
+            )
+            method_doc_structure.write_to_file(client_dir_path, method_name)
+
+    def _add_client_method(self, section, method_name, method):
+        breadcrumb_section = section.add_new_section('breadcrumb')
+        breadcrumb_section.style.ref(
+            self._client_class_name, f'../../{self._service_name}'
+        )
+        breadcrumb_section.write(f' / Client / {method_name}')
+        section.add_title_section(method_name)
+        method_section = section.add_new_section(
+            method_name,
+            context={'qualifier': f'{self._client_class_name}.Client.'},
+        )
+        if self._is_custom_method(method_name):
+            self._add_custom_method(
+                method_section,
+                method_name,
+                method,
+            )
+        else:
+            self._add_model_driven_method(method_section, method_name)
+
+    def _is_custom_method(self, method_name):
+        return method_name not in self._client.meta.method_to_api_mapping
+
+    def _add_custom_method(self, section, method_name, method):
+        document_custom_method(section, method_name, method)
+
+    def _add_method_exceptions_list(self, section, operation_model):
+        error_section = section.add_new_section('exceptions')
+        error_section.style.new_line()
+        error_section.style.bold('Exceptions')
+        error_section.style.new_line()
+        for error in operation_model.error_shapes:
+            class_name = (
+                f'{self._client_class_name}.Client.exceptions.{error.name}'
+            )
+            error_section.style.li(':py:class:`%s`' % class_name)
+
+    def _add_model_driven_method(self, section, method_name):
+        service_model = self._client.meta.service_model
+        operation_name = self._client.meta.method_to_api_mapping[method_name]
+        operation_model = service_model.operation_model(operation_name)
+
+        example_prefix = 'response = client.%s' % method_name
+        full_method_name = (
+            f"{section.context.get('qualifier', '')}{method_name}"
+        )
+        document_model_driven_method(
+            section,
+            full_method_name,
+            operation_model,
+            event_emitter=self._client.meta.events,
+            method_description=operation_model.documentation,
+            example_prefix=example_prefix,
+        )
+
+        # Add any modeled exceptions
+        if operation_model.error_shapes:
+            self._add_method_exceptions_list(section, operation_model)
+
+        # Add the shared examples
+        shared_examples = self._shared_examples.get(operation_name)
+        if shared_examples:
+            document_shared_examples(
+                section, operation_model, example_prefix, shared_examples
+            )
+
+
+class ClientExceptionsDocumenter:
+    _USER_GUIDE_LINK = (
+        'https://boto3.amazonaws.com/'
+        'v1/documentation/api/latest/guide/error-handling.html'
+    )
+    _GENERIC_ERROR_SHAPE = DocumentedShape(
+        name='Error',
+        type_name='structure',
+        documentation=('Normalized access to common exception attributes.'),
+        members=OrderedDict(
+            [
+                (
+                    'Code',
+                    DocumentedShape(
+                        name='Code',
+                        type_name='string',
+                        documentation=(
+                            'An identifier specifying the exception type.'
+                        ),
+                    ),
+                ),
+                (
+                    'Message',
+                    DocumentedShape(
+                        name='Message',
+                        type_name='string',
+                        documentation=(
+                            'A descriptive message explaining why the exception '
+                            'occured.'
+                        ),
+                    ),
+                ),
+            ]
+        ),
+    )
+
+    def __init__(self, client, root_docs_path):
+        self._client = client
+        self._client_class_name = self._client.__class__.__name__
+        self._service_name = self._client.meta.service_model.service_name
+        self._root_docs_path = root_docs_path
+
+    def document_exceptions(self, section):
+        self._add_title(section)
+        self._add_overview(section)
+        self._add_exceptions_list(section)
+        self._add_exception_classes()
+
+    def _add_title(self, section):
+        section.style.h2('Client Exceptions')
+
+    def _add_overview(self, section):
+        section.style.new_line()
+        section.write(
+            'Client exceptions are available on a client instance '
+            'via the ``exceptions`` property. For more detailed instructions '
+            'and examples on the exact usage of client exceptions, see the '
+            'error handling '
+        )
+        section.style.external_link(
+            title='user guide',
+            link=self._USER_GUIDE_LINK,
+        )
+        section.write('.')
+        section.style.new_line()
+
+    def _exception_class_name(self, shape):
+        return f'{self._client_class_name}.Client.exceptions.{shape.name}'
+
+    def _add_exceptions_list(self, section):
+        error_shapes = self._client.meta.service_model.error_shapes
+        if not error_shapes:
+            section.style.new_line()
+            section.write('This client has no modeled exception classes.')
+            section.style.new_line()
+            return
+        section.style.new_line()
+        section.writeln('The available client exceptions are:')
+        section.style.toctree()
+        for shape in error_shapes:
+            section.style.tocitem(
+                f'{self._service_name}/client/exceptions/{shape.name}'
+            )
+
+    def _add_exception_classes(self):
+        for shape in self._client.meta.service_model.error_shapes:
+            # Create a new DocumentStructure for each exception method and add contents.
+            exception_doc_structure = DocumentStructure(
+                shape.name, target='html'
+            )
+            self._add_exception_class(exception_doc_structure, shape)
+            # Write exceptions in individual/nested files.
+            # Path: <root>/reference/services/<service>/client/exceptions/<exception_name>.rst
+            exception_dir_path = os.path.join(
+                self._root_docs_path,
+                self._service_name,
+                'client',
+                'exceptions',
+            )
+            exception_doc_structure.write_to_file(
+                exception_dir_path, shape.name
+            )
+
+    def _add_exception_class(self, section, shape):
+        breadcrumb_section = section.add_new_section('breadcrumb')
+        breadcrumb_section.style.ref(
+            self._client_class_name, f'../../../{self._service_name}'
+        )
+        breadcrumb_section.write(f' / Client / exceptions / {shape.name}')
+        section.add_title_section(shape.name)
+        class_section = section.add_new_section(shape.name)
+        class_name = self._exception_class_name(shape)
+        class_section.style.start_sphinx_py_class(class_name=class_name)
+        self._add_top_level_documentation(class_section, shape)
+        self._add_exception_catch_example(class_section, shape)
+        self._add_response_attr(class_section, shape)
+        class_section.style.end_sphinx_py_class()
+
+    def _add_top_level_documentation(self, section, shape):
+        if shape.documentation:
+            section.style.new_line()
+            section.include_doc_string(shape.documentation)
+            section.style.new_line()
+
+    def _add_exception_catch_example(self, section, shape):
+        section.style.new_line()
+        section.style.bold('Example')
+        section.style.new_paragraph()
+        section.style.start_codeblock()
+        section.write('try:')
+        section.style.indent()
+        section.style.new_line()
+        section.write('...')
+        section.style.dedent()
+        section.style.new_line()
+        section.write('except client.exceptions.%s as e:' % shape.name)
+        section.style.indent()
+        section.style.new_line()
+        section.write('print(e.response)')
+        section.style.dedent()
+        section.style.end_codeblock()
+
+    def _add_response_attr(self, section, shape):
+        response_section = section.add_new_section('response')
+        response_section.style.start_sphinx_py_attr('response')
+        self._add_response_attr_description(response_section)
+        self._add_response_example(response_section, shape)
+        self._add_response_params(response_section, shape)
+        response_section.style.end_sphinx_py_attr()
+
+    def _add_response_attr_description(self, section):
+        section.style.new_line()
+        section.include_doc_string(
+            'The parsed error response. All exceptions have a top level '
+            '``Error`` key that provides normalized access to common '
+            'exception atrributes. All other keys are specific to this '
+            'service or exception class.'
+        )
+        section.style.new_line()
+
+    def _add_response_example(self, section, shape):
+        example_section = section.add_new_section('syntax')
+        example_section.style.new_line()
+        example_section.style.bold('Syntax')
+        example_section.style.new_paragraph()
+        documenter = ResponseExampleDocumenter(
+            service_name=self._service_name,
+            operation_name=None,
+            event_emitter=self._client.meta.events,
+        )
+        documenter.document_example(
+            example_section,
+            shape,
+            include=[self._GENERIC_ERROR_SHAPE],
+        )
+
+    def _add_response_params(self, section, shape):
+        params_section = section.add_new_section('Structure')
+        params_section.style.new_line()
+        params_section.style.bold('Structure')
+        params_section.style.new_paragraph()
+        documenter = ResponseParamsDocumenter(
+            service_name=self._service_name,
+            operation_name=None,
+            event_emitter=self._client.meta.events,
+        )
+        documenter.document_params(
+            params_section,
+            shape,
+            include=[self._GENERIC_ERROR_SHAPE],
+        )
+
+
+class ClientContextParamsDocumenter:
+    _CONFIG_GUIDE_LINK = (
+        'https://boto3.amazonaws.com/'
+        'v1/documentation/api/latest/guide/configuration.html'
+    )
+
+    OMITTED_CONTEXT_PARAMS = {
+        's3': (
+            'Accelerate',
+            'DisableMultiRegionAccessPoints',
+            'ForcePathStyle',
+            'UseArnRegion',
+        ),
+        's3control': ('UseArnRegion',),
+    }
+
+    def __init__(self, service_name, context_params):
+        self._service_name = service_name
+        self._context_params = context_params
+
+    def document_context_params(self, section):
+        self._add_title(section)
+        self._add_overview(section)
+        self._add_context_params_list(section)
+
+    def _add_title(self, section):
+        section.style.h2('Client Context Parameters')
+
+    def _add_overview(self, section):
+        section.style.new_line()
+        section.write(
+            'Client context parameters are configurable on a client '
+            'instance via the ``client_context_params`` parameter in the '
+            '``Config`` object. For more detailed instructions and examples '
+            'on the exact usage of context params see the '
+        )
+        section.style.external_link(
+            title='configuration guide',
+            link=self._CONFIG_GUIDE_LINK,
+        )
+        section.write('.')
+        section.style.new_line()
+
+    def _add_context_params_list(self, section):
+        section.style.new_line()
+        sn = f'``{self._service_name}``'
+        section.writeln(f'The available {sn} client context params are:')
+        for param in self._context_params:
+            section.style.new_line()
+            name = f'``{xform_name(param.name)}``'
+            section.write(f'* {name} ({param.type}) - {param.documentation}')

Function Source Files/botocore/docs/docstring.py

@@ -0,0 +1,97 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from botocore.docs.bcdoc.restdoc import DocumentStructure
+from botocore.docs.method import document_model_driven_method
+from botocore.docs.paginator import document_paginate_method
+from botocore.docs.waiter import document_wait_method
+
+
+class LazyLoadedDocstring(str):
+    """Used for lazily loading docstrings
+
+    You can instantiate this class and assign it to a __doc__ value.
+    The docstring will not be generated till accessed via __doc__ or
+    help(). Note that all docstring classes **must** subclass from
+    this class. It cannot be used directly as a docstring.
+    """
+
+    def __init__(self, *args, **kwargs):
+        """
+        The args and kwargs are the same as the underlying document
+        generation function. These just get proxied to the underlying
+        function.
+        """
+        super().__init__()
+        self._gen_args = args
+        self._gen_kwargs = kwargs
+        self._docstring = None
+
+    def __new__(cls, *args, **kwargs):
+        # Needed in order to sub class from str with args and kwargs
+        return super().__new__(cls)
+
+    def _write_docstring(self, *args, **kwargs):
+        raise NotImplementedError(
+            '_write_docstring is not implemented. Please subclass from '
+            'this class and provide your own _write_docstring method'
+        )
+
+    def expandtabs(self, tabsize=8):
+        """Expands tabs to spaces
+
+        So this is a big hack in order to get lazy loaded docstring work
+        for the ``help()``. In the ``help()`` function, ``pydoc`` and
+        ``inspect`` are used. At some point the ``inspect.cleandoc``
+        method is called. To clean the docs ``expandtabs`` is called
+        and that is where we override the method to generate and return the
+        docstrings.
+        """
+        if self._docstring is None:
+            self._generate()
+        return self._docstring.expandtabs(tabsize)
+
+    def __str__(self):
+        return self._generate()
+
+    # __doc__ of target will use either __repr__ or __str__ of this class.
+    __repr__ = __str__
+
+    def _generate(self):
+        # Generate the docstring if it is not already cached.
+        if self._docstring is None:
+            self._docstring = self._create_docstring()
+        return self._docstring
+
+    def _create_docstring(self):
+        docstring_structure = DocumentStructure('docstring', target='html')
+        # Call the document method function with the args and kwargs
+        # passed to the class.
+        self._write_docstring(
+            docstring_structure, *self._gen_args, **self._gen_kwargs
+        )
+        return docstring_structure.flush_structure().decode('utf-8')
+
+
+class ClientMethodDocstring(LazyLoadedDocstring):
+    def _write_docstring(self, *args, **kwargs):
+        document_model_driven_method(*args, **kwargs)
+
+
+class WaiterDocstring(LazyLoadedDocstring):
+    def _write_docstring(self, *args, **kwargs):
+        document_wait_method(*args, **kwargs)
+
+
+class PaginatorDocstring(LazyLoadedDocstring):
+    def _write_docstring(self, *args, **kwargs):
+        document_paginate_method(*args, **kwargs)

Function Source Files/botocore/docs/example.py

@@ -0,0 +1,236 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from botocore.docs.shape import ShapeDocumenter
+from botocore.docs.utils import py_default
+
+
+class BaseExampleDocumenter(ShapeDocumenter):
+    def document_example(
+        self, section, shape, prefix=None, include=None, exclude=None
+    ):
+        """Generates an example based on a shape
+
+        :param section: The section to write the documentation to.
+
+        :param shape: The shape of the operation.
+
+        :param prefix: Anything to be included before the example
+
+        :type include: Dictionary where keys are parameter names and
+            values are the shapes of the parameter names.
+        :param include: The parameter shapes to include in the documentation.
+
+        :type exclude: List of the names of the parameters to exclude.
+        :param exclude: The names of the parameters to exclude from
+            documentation.
+        """
+        history = []
+        section.style.new_line()
+        section.style.start_codeblock()
+        if prefix is not None:
+            section.write(prefix)
+        self.traverse_and_document_shape(
+            section=section,
+            shape=shape,
+            history=history,
+            include=include,
+            exclude=exclude,
+        )
+        final_blank_line_section = section.add_new_section('final-blank-line')
+        final_blank_line_section.style.new_line()
+
+    def document_recursive_shape(self, section, shape, **kwargs):
+        section.write('{\'... recursive ...\'}')
+
+    def document_shape_default(
+        self, section, shape, history, include=None, exclude=None, **kwargs
+    ):
+        py_type = self._get_special_py_default(shape)
+        if py_type is None:
+            py_type = py_default(shape.type_name)
+
+        if self._context.get('streaming_shape') == shape:
+            py_type = 'StreamingBody()'
+        section.write(py_type)
+
+    def document_shape_type_string(
+        self, section, shape, history, include=None, exclude=None, **kwargs
+    ):
+        if 'enum' in shape.metadata:
+            for i, enum in enumerate(shape.metadata['enum']):
+                section.write('\'%s\'' % enum)
+                if i < len(shape.metadata['enum']) - 1:
+                    section.write('|')
+        else:
+            self.document_shape_default(section, shape, history)
+
+    def document_shape_type_list(
+        self, section, shape, history, include=None, exclude=None, **kwargs
+    ):
+        param_shape = shape.member
+        list_section = section.add_new_section('list-value')
+        self._start_nested_param(list_section, '[')
+        param_section = list_section.add_new_section(
+            'member', context={'shape': param_shape.name}
+        )
+        self.traverse_and_document_shape(
+            section=param_section, shape=param_shape, history=history
+        )
+        ending_comma_section = list_section.add_new_section('ending-comma')
+        ending_comma_section.write(',')
+        ending_bracket_section = list_section.add_new_section('ending-bracket')
+        self._end_nested_param(ending_bracket_section, ']')
+
+    def document_shape_type_structure(
+        self, section, shape, history, include=None, exclude=None, **kwargs
+    ):
+        if not shape.members:
+            section.write('{}')
+            return
+
+        section = section.add_new_section('structure-value')
+        self._start_nested_param(section, '{')
+
+        input_members = self._add_members_to_shape(shape.members, include)
+
+        for i, param in enumerate(input_members):
+            if exclude and param in exclude:
+                continue
+            param_section = section.add_new_section(param)
+            param_section.write('\'%s\': ' % param)
+            param_shape = input_members[param]
+            param_value_section = param_section.add_new_section(
+                'member-value', context={'shape': param_shape.name}
+            )
+            self.traverse_and_document_shape(
+                section=param_value_section,
+                shape=param_shape,
+                history=history,
+                name=param,
+            )
+            if i < len(input_members) - 1:
+                ending_comma_section = param_section.add_new_section(
+                    'ending-comma'
+                )
+                ending_comma_section.write(',')
+                ending_comma_section.style.new_line()
+        self._end_structure(section, '{', '}')
+
+    def document_shape_type_map(
+        self, section, shape, history, include=None, exclude=None, **kwargs
+    ):
+        map_section = section.add_new_section('map-value')
+        self._start_nested_param(map_section, '{')
+        value_shape = shape.value
+        key_section = map_section.add_new_section(
+            'key', context={'shape': shape.key.name}
+        )
+        key_section.write('\'string\': ')
+        value_section = map_section.add_new_section(
+            'value', context={'shape': value_shape.name}
+        )
+        self.traverse_and_document_shape(
+            section=value_section, shape=value_shape, history=history
+        )
+        end_bracket_section = map_section.add_new_section('ending-bracket')
+        self._end_nested_param(end_bracket_section, '}')
+
+    def _add_members_to_shape(self, members, include):
+        if include:
+            members = members.copy()
+            for param in include:
+                members[param.name] = param
+        return members
+
+    def _start_nested_param(self, section, start=None):
+        if start is not None:
+            section.write(start)
+        section.style.indent()
+        section.style.indent()
+        section.style.new_line()
+
+    def _end_nested_param(self, section, end=None):
+        section.style.dedent()
+        section.style.dedent()
+        section.style.new_line()
+        if end is not None:
+            section.write(end)
+
+    def _end_structure(self, section, start, end):
+        # If there are no members in the strucuture, then make sure the
+        # start and the end bracket are on the same line, by removing all
+        # previous text and writing the start and end.
+        if not section.available_sections:
+            section.clear_text()
+            section.write(start + end)
+            self._end_nested_param(section)
+        else:
+            end_bracket_section = section.add_new_section('ending-bracket')
+            self._end_nested_param(end_bracket_section, end)
+
+
+class ResponseExampleDocumenter(BaseExampleDocumenter):
+    EVENT_NAME = 'response-example'
+
+    def document_shape_type_event_stream(
+        self, section, shape, history, **kwargs
+    ):
+        section.write('EventStream(')
+        self.document_shape_type_structure(section, shape, history, **kwargs)
+        end_section = section.add_new_section('event-stream-end')
+        end_section.write(')')
+
+
+class RequestExampleDocumenter(BaseExampleDocumenter):
+    EVENT_NAME = 'request-example'
+
+    def document_shape_type_structure(
+        self, section, shape, history, include=None, exclude=None, **kwargs
+    ):
+        param_format = '\'%s\''
+        operator = ': '
+        start = '{'
+        end = '}'
+
+        if len(history) <= 1:
+            operator = '='
+            start = '('
+            end = ')'
+            param_format = '%s'
+        section = section.add_new_section('structure-value')
+        self._start_nested_param(section, start)
+        input_members = self._add_members_to_shape(shape.members, include)
+
+        for i, param in enumerate(input_members):
+            if exclude and param in exclude:
+                continue
+            param_section = section.add_new_section(param)
+            param_section.write(param_format % param)
+            param_section.write(operator)
+            param_shape = input_members[param]
+            param_value_section = param_section.add_new_section(
+                'member-value', context={'shape': param_shape.name}
+            )
+            self.traverse_and_document_shape(
+                section=param_value_section,
+                shape=param_shape,
+                history=history,
+                name=param,
+            )
+            if i < len(input_members) - 1:
+                ending_comma_section = param_section.add_new_section(
+                    'ending-comma'
+                )
+                ending_comma_section.write(',')
+                ending_comma_section.style.new_line()
+        self._end_structure(section, start, end)

Function Source Files/botocore/docs/method.py

@@ -0,0 +1,328 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import inspect
+import types
+
+from botocore.docs.example import (
+    RequestExampleDocumenter,
+    ResponseExampleDocumenter,
+)
+from botocore.docs.params import (
+    RequestParamsDocumenter,
+    ResponseParamsDocumenter,
+)
+
+AWS_DOC_BASE = 'https://docs.aws.amazon.com/goto/WebAPI'
+
+
+def get_instance_public_methods(instance):
+    """Retrieves an objects public methods
+
+    :param instance: The instance of the class to inspect
+    :rtype: dict
+    :returns: A dictionary that represents an instance's methods where
+        the keys are the name of the methods and the
+        values are the handler to the method.
+    """
+    instance_members = inspect.getmembers(instance)
+    instance_methods = {}
+    for name, member in instance_members:
+        if not name.startswith('_'):
+            if inspect.ismethod(member):
+                instance_methods[name] = member
+    return instance_methods
+
+
+def document_model_driven_signature(
+    section, name, operation_model, include=None, exclude=None
+):
+    """Documents the signature of a model-driven method
+
+    :param section: The section to write the documentation to.
+
+    :param name: The name of the method
+
+    :param operation_model: The operation model for the method
+
+    :type include: Dictionary where keys are parameter names and
+        values are the shapes of the parameter names.
+    :param include: The parameter shapes to include in the documentation.
+
+    :type exclude: List of the names of the parameters to exclude.
+    :param exclude: The names of the parameters to exclude from
+        documentation.
+    """
+    params = {}
+    if operation_model.input_shape:
+        params = operation_model.input_shape.members
+
+    parameter_names = list(params.keys())
+
+    if include is not None:
+        for member in include:
+            parameter_names.append(member.name)
+
+    if exclude is not None:
+        for member in exclude:
+            if member in parameter_names:
+                parameter_names.remove(member)
+
+    signature_params = ''
+    if parameter_names:
+        signature_params = '**kwargs'
+    section.style.start_sphinx_py_method(name, signature_params)
+
+
+def document_custom_signature(
+    section, name, method, include=None, exclude=None
+):
+    """Documents the signature of a custom method
+
+    :param section: The section to write the documentation to.
+
+    :param name: The name of the method
+
+    :param method: The handle to the method being documented
+
+    :type include: Dictionary where keys are parameter names and
+        values are the shapes of the parameter names.
+    :param include: The parameter shapes to include in the documentation.
+
+    :type exclude: List of the names of the parameters to exclude.
+    :param exclude: The names of the parameters to exclude from
+        documentation.
+    """
+    signature = inspect.signature(method)
+    # "raw" class methods are FunctionType and they include "self" param
+    # object methods are MethodType and they skip the "self" param
+    if isinstance(method, types.FunctionType):
+        self_param = next(iter(signature.parameters))
+        self_kind = signature.parameters[self_param].kind
+        # safety check that we got the right parameter
+        assert self_kind == inspect.Parameter.POSITIONAL_OR_KEYWORD
+        new_params = signature.parameters.copy()
+        del new_params[self_param]
+        signature = signature.replace(parameters=new_params.values())
+    signature_params = str(signature).lstrip('(')
+    signature_params = signature_params.rstrip(')')
+    section.style.start_sphinx_py_method(name, signature_params)
+
+
+def document_custom_method(section, method_name, method):
+    """Documents a non-data driven method
+
+    :param section: The section to write the documentation to.
+
+    :param method_name: The name of the method
+
+    :param method: The handle to the method being documented
+    """
+    full_method_name = f"{section.context.get('qualifier', '')}{method_name}"
+    document_custom_signature(section, full_method_name, method)
+    method_intro_section = section.add_new_section('method-intro')
+    method_intro_section.writeln('')
+    doc_string = inspect.getdoc(method)
+    if doc_string is not None:
+        method_intro_section.style.write_py_doc_string(doc_string)
+
+
+def document_model_driven_method(
+    section,
+    method_name,
+    operation_model,
+    event_emitter,
+    method_description=None,
+    example_prefix=None,
+    include_input=None,
+    include_output=None,
+    exclude_input=None,
+    exclude_output=None,
+    document_output=True,
+    include_signature=True,
+):
+    """Documents an individual method
+
+    :param section: The section to write to
+
+    :param method_name: The name of the method
+
+    :param operation_model: The model of the operation
+
+    :param event_emitter: The event emitter to use to emit events
+
+    :param example_prefix: The prefix to use in the method example.
+
+    :type include_input: Dictionary where keys are parameter names and
+        values are the shapes of the parameter names.
+    :param include_input: The parameter shapes to include in the
+        input documentation.
+
+    :type include_output: Dictionary where keys are parameter names and
+        values are the shapes of the parameter names.
+    :param include_input: The parameter shapes to include in the
+        output documentation.
+
+    :type exclude_input: List of the names of the parameters to exclude.
+    :param exclude_input: The names of the parameters to exclude from
+        input documentation.
+
+    :type exclude_output: List of the names of the parameters to exclude.
+    :param exclude_input: The names of the parameters to exclude from
+        output documentation.
+
+    :param document_output: A boolean flag to indicate whether to
+        document the output.
+
+    :param include_signature: Whether or not to include the signature.
+        It is useful for generating docstrings.
+    """
+    # Add the signature if specified.
+    if include_signature:
+        document_model_driven_signature(
+            section,
+            method_name,
+            operation_model,
+            include=include_input,
+            exclude=exclude_input,
+        )
+
+    # Add the description for the method.
+    method_intro_section = section.add_new_section('method-intro')
+    method_intro_section.include_doc_string(method_description)
+    if operation_model.deprecated:
+        method_intro_section.style.start_danger()
+        method_intro_section.writeln(
+            'This operation is deprecated and may not function as '
+            'expected. This operation should not be used going forward '
+            'and is only kept for the purpose of backwards compatiblity.'
+        )
+        method_intro_section.style.end_danger()
+    service_uid = operation_model.service_model.metadata.get('uid')
+    if service_uid is not None:
+        method_intro_section.style.new_paragraph()
+        method_intro_section.write("See also: ")
+        link = f"{AWS_DOC_BASE}/{service_uid}/{operation_model.name}"
+        method_intro_section.style.external_link(
+            title="AWS API Documentation", link=link
+        )
+        method_intro_section.writeln('')
+
+    # Add the example section.
+    example_section = section.add_new_section('request-example')
+    example_section.style.new_paragraph()
+    example_section.style.bold('Request Syntax')
+
+    context = {
+        'special_shape_types': {
+            'streaming_input_shape': operation_model.get_streaming_input(),
+            'streaming_output_shape': operation_model.get_streaming_output(),
+            'eventstream_output_shape': operation_model.get_event_stream_output(),
+        },
+    }
+
+    if operation_model.input_shape:
+        RequestExampleDocumenter(
+            service_name=operation_model.service_model.service_name,
+            operation_name=operation_model.name,
+            event_emitter=event_emitter,
+            context=context,
+        ).document_example(
+            example_section,
+            operation_model.input_shape,
+            prefix=example_prefix,
+            include=include_input,
+            exclude=exclude_input,
+        )
+    else:
+        example_section.style.new_paragraph()
+        example_section.style.start_codeblock()
+        example_section.write(example_prefix + '()')
+
+    # Add the request parameter documentation.
+    request_params_section = section.add_new_section('request-params')
+    if operation_model.input_shape:
+        RequestParamsDocumenter(
+            service_name=operation_model.service_model.service_name,
+            operation_name=operation_model.name,
+            event_emitter=event_emitter,
+            context=context,
+        ).document_params(
+            request_params_section,
+            operation_model.input_shape,
+            include=include_input,
+            exclude=exclude_input,
+        )
+
+    # Add the return value documentation
+    return_section = section.add_new_section('return')
+    return_section.style.new_line()
+    if operation_model.output_shape is not None and document_output:
+        return_section.write(':rtype: dict')
+        return_section.style.new_line()
+        return_section.write(':returns: ')
+        return_section.style.indent()
+        return_section.style.new_line()
+
+        # If the operation is an event stream, describe the tagged union
+        event_stream_output = operation_model.get_event_stream_output()
+        if event_stream_output:
+            event_section = return_section.add_new_section('event-stream')
+            event_section.style.new_paragraph()
+            event_section.write(
+                'The response of this operation contains an '
+                ':class:`.EventStream` member. When iterated the '
+                ':class:`.EventStream` will yield events based on the '
+                'structure below, where only one of the top level keys '
+                'will be present for any given event.'
+            )
+            event_section.style.new_line()
+
+        # Add an example return value
+        return_example_section = return_section.add_new_section(
+            'response-example'
+        )
+        return_example_section.style.new_line()
+        return_example_section.style.bold('Response Syntax')
+        return_example_section.style.new_paragraph()
+        ResponseExampleDocumenter(
+            service_name=operation_model.service_model.service_name,
+            operation_name=operation_model.name,
+            event_emitter=event_emitter,
+            context=context,
+        ).document_example(
+            return_example_section,
+            operation_model.output_shape,
+            include=include_output,
+            exclude=exclude_output,
+        )
+
+        # Add a description for the return value
+        return_description_section = return_section.add_new_section(
+            'description'
+        )
+        return_description_section.style.new_line()
+        return_description_section.style.bold('Response Structure')
+        return_description_section.style.new_paragraph()
+        ResponseParamsDocumenter(
+            service_name=operation_model.service_model.service_name,
+            operation_name=operation_model.name,
+            event_emitter=event_emitter,
+            context=context,
+        ).document_params(
+            return_description_section,
+            operation_model.output_shape,
+            include=include_output,
+            exclude=exclude_output,
+        )
+    else:
+        return_section.write(':returns: None')

Function Source Files/botocore/docs/paginator.py

@@ -0,0 +1,243 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import os
+
+from botocore import xform_name
+from botocore.compat import OrderedDict
+from botocore.docs.bcdoc.restdoc import DocumentStructure
+from botocore.docs.method import document_model_driven_method
+from botocore.docs.utils import DocumentedShape
+from botocore.utils import get_service_module_name
+
+
+class PaginatorDocumenter:
+    def __init__(self, client, service_paginator_model, root_docs_path):
+        self._client = client
+        self._client_class_name = self._client.__class__.__name__
+        self._service_name = self._client.meta.service_model.service_name
+        self._service_paginator_model = service_paginator_model
+        self._root_docs_path = root_docs_path
+        self._USER_GUIDE_LINK = (
+            'https://boto3.amazonaws.com/'
+            'v1/documentation/api/latest/guide/paginators.html'
+        )
+
+    def document_paginators(self, section):
+        """Documents the various paginators for a service
+
+        param section: The section to write to.
+        """
+        section.style.h2('Paginators')
+        self._add_overview(section)
+        section.style.new_line()
+        section.writeln('The available paginators are:')
+        section.style.toctree()
+
+        paginator_names = sorted(
+            self._service_paginator_model._paginator_config
+        )
+
+        # List the available paginators and then document each paginator.
+        for paginator_name in paginator_names:
+            section.style.tocitem(
+                f'{self._service_name}/paginator/{paginator_name}'
+            )
+            # Create a new DocumentStructure for each paginator and add contents.
+            paginator_doc_structure = DocumentStructure(
+                paginator_name, target='html'
+            )
+            self._add_paginator(paginator_doc_structure, paginator_name)
+            # Write paginators in individual/nested files.
+            # Path: <root>/reference/services/<service>/paginator/<paginator_name>.rst
+            paginator_dir_path = os.path.join(
+                self._root_docs_path, self._service_name, 'paginator'
+            )
+            paginator_doc_structure.write_to_file(
+                paginator_dir_path, paginator_name
+            )
+
+    def _add_paginator(self, section, paginator_name):
+        breadcrumb_section = section.add_new_section('breadcrumb')
+        breadcrumb_section.style.ref(
+            self._client_class_name, f'../../{self._service_name}'
+        )
+        breadcrumb_section.write(f' / Paginator / {paginator_name}')
+        section.add_title_section(paginator_name)
+
+        # Docment the paginator class
+        paginator_section = section.add_new_section(paginator_name)
+        paginator_section.style.start_sphinx_py_class(
+            class_name=(
+                f'{self._client_class_name}.Paginator.{paginator_name}'
+            )
+        )
+        paginator_section.style.start_codeblock()
+        paginator_section.style.new_line()
+
+        # Document how to instantiate the paginator.
+        paginator_section.write(
+            f"paginator = client.get_paginator('{xform_name(paginator_name)}')"
+        )
+        paginator_section.style.end_codeblock()
+        paginator_section.style.new_line()
+        # Get the pagination model for the particular paginator.
+        paginator_config = self._service_paginator_model.get_paginator(
+            paginator_name
+        )
+        document_paginate_method(
+            section=paginator_section,
+            paginator_name=paginator_name,
+            event_emitter=self._client.meta.events,
+            service_model=self._client.meta.service_model,
+            paginator_config=paginator_config,
+        )
+
+    def _add_overview(self, section):
+        section.style.new_line()
+        section.write(
+            'Paginators are available on a client instance '
+            'via the ``get_paginator`` method. For more detailed instructions '
+            'and examples on the usage of paginators, see the '
+            'paginators '
+        )
+        section.style.external_link(
+            title='user guide',
+            link=self._USER_GUIDE_LINK,
+        )
+        section.write('.')
+        section.style.new_line()
+
+
+def document_paginate_method(
+    section,
+    paginator_name,
+    event_emitter,
+    service_model,
+    paginator_config,
+    include_signature=True,
+):
+    """Documents the paginate method of a paginator
+
+    :param section: The section to write to
+
+    :param paginator_name: The name of the paginator. It is snake cased.
+
+    :param event_emitter: The event emitter to use to emit events
+
+    :param service_model: The service model
+
+    :param paginator_config: The paginator config associated to a particular
+        paginator.
+
+    :param include_signature: Whether or not to include the signature.
+        It is useful for generating docstrings.
+    """
+    # Retrieve the operation model of the underlying operation.
+    operation_model = service_model.operation_model(paginator_name)
+
+    # Add representations of the request and response parameters
+    # we want to include in the description of the paginate method.
+    # These are parameters we expose via the botocore interface.
+    pagination_config_members = OrderedDict()
+
+    pagination_config_members['MaxItems'] = DocumentedShape(
+        name='MaxItems',
+        type_name='integer',
+        documentation=(
+            '<p>The total number of items to return. If the total '
+            'number of items available is more than the value '
+            'specified in max-items then a <code>NextToken</code> '
+            'will be provided in the output that you can use to '
+            'resume pagination.</p>'
+        ),
+    )
+
+    if paginator_config.get('limit_key', None):
+        pagination_config_members['PageSize'] = DocumentedShape(
+            name='PageSize',
+            type_name='integer',
+            documentation='<p>The size of each page.<p>',
+        )
+
+    pagination_config_members['StartingToken'] = DocumentedShape(
+        name='StartingToken',
+        type_name='string',
+        documentation=(
+            '<p>A token to specify where to start paginating. '
+            'This is the <code>NextToken</code> from a previous '
+            'response.</p>'
+        ),
+    )
+
+    botocore_pagination_params = [
+        DocumentedShape(
+            name='PaginationConfig',
+            type_name='structure',
+            documentation=(
+                '<p>A dictionary that provides parameters to control '
+                'pagination.</p>'
+            ),
+            members=pagination_config_members,
+        )
+    ]
+
+    botocore_pagination_response_params = [
+        DocumentedShape(
+            name='NextToken',
+            type_name='string',
+            documentation=('<p>A token to resume pagination.</p>'),
+        )
+    ]
+
+    service_pagination_params = []
+
+    # Add the normal input token of the method to a list
+    # of input paramters that we wish to hide since we expose our own.
+    if isinstance(paginator_config['input_token'], list):
+        service_pagination_params += paginator_config['input_token']
+    else:
+        service_pagination_params.append(paginator_config['input_token'])
+
+    # Hide the limit key in the documentation.
+    if paginator_config.get('limit_key', None):
+        service_pagination_params.append(paginator_config['limit_key'])
+
+    # Hide the output tokens in the documentation.
+    service_pagination_response_params = []
+    if isinstance(paginator_config['output_token'], list):
+        service_pagination_response_params += paginator_config['output_token']
+    else:
+        service_pagination_response_params.append(
+            paginator_config['output_token']
+        )
+
+    paginate_description = (
+        'Creates an iterator that will paginate through responses '
+        'from :py:meth:`{}.Client.{}`.'.format(
+            get_service_module_name(service_model), xform_name(paginator_name)
+        )
+    )
+
+    document_model_driven_method(
+        section,
+        'paginate',
+        operation_model,
+        event_emitter=event_emitter,
+        method_description=paginate_description,
+        example_prefix='response_iterator = paginator.paginate',
+        include_input=botocore_pagination_params,
+        include_output=botocore_pagination_response_params,
+        exclude_input=service_pagination_params,
+        exclude_output=service_pagination_response_params,
+        include_signature=include_signature,
+    )

Function Source Files/botocore/docs/params.py

@@ -0,0 +1,303 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from botocore.docs.shape import ShapeDocumenter
+from botocore.docs.utils import py_type_name
+
+
+class BaseParamsDocumenter(ShapeDocumenter):
+    def document_params(self, section, shape, include=None, exclude=None):
+        """Fills out the documentation for a section given a model shape.
+
+        :param section: The section to write the documentation to.
+
+        :param shape: The shape of the operation.
+
+        :type include: Dictionary where keys are parameter names and
+            values are the shapes of the parameter names.
+        :param include: The parameter shapes to include in the documentation.
+
+        :type exclude: List of the names of the parameters to exclude.
+        :param exclude: The names of the parameters to exclude from
+            documentation.
+        """
+        history = []
+        self.traverse_and_document_shape(
+            section=section,
+            shape=shape,
+            history=history,
+            name=None,
+            include=include,
+            exclude=exclude,
+        )
+
+    def document_recursive_shape(self, section, shape, **kwargs):
+        self._add_member_documentation(section, shape, **kwargs)
+
+    def document_shape_default(
+        self, section, shape, history, include=None, exclude=None, **kwargs
+    ):
+        self._add_member_documentation(section, shape, **kwargs)
+
+    def document_shape_type_list(
+        self, section, shape, history, include=None, exclude=None, **kwargs
+    ):
+        self._add_member_documentation(section, shape, **kwargs)
+        param_shape = shape.member
+        param_section = section.add_new_section(
+            param_shape.name, context={'shape': shape.member.name}
+        )
+        self._start_nested_param(param_section)
+        self.traverse_and_document_shape(
+            section=param_section,
+            shape=param_shape,
+            history=history,
+            name=None,
+        )
+        section = section.add_new_section('end-list')
+        self._end_nested_param(section)
+
+    def document_shape_type_map(
+        self, section, shape, history, include=None, exclude=None, **kwargs
+    ):
+        self._add_member_documentation(section, shape, **kwargs)
+
+        key_section = section.add_new_section(
+            'key', context={'shape': shape.key.name}
+        )
+        self._start_nested_param(key_section)
+        self._add_member_documentation(key_section, shape.key)
+
+        param_section = section.add_new_section(
+            shape.value.name, context={'shape': shape.value.name}
+        )
+        param_section.style.indent()
+        self._start_nested_param(param_section)
+        self.traverse_and_document_shape(
+            section=param_section,
+            shape=shape.value,
+            history=history,
+            name=None,
+        )
+
+        end_section = section.add_new_section('end-map')
+        self._end_nested_param(end_section)
+        self._end_nested_param(end_section)
+
+    def document_shape_type_structure(
+        self,
+        section,
+        shape,
+        history,
+        include=None,
+        exclude=None,
+        name=None,
+        **kwargs,
+    ):
+        members = self._add_members_to_shape(shape.members, include)
+        self._add_member_documentation(section, shape, name=name)
+        for param in members:
+            if exclude and param in exclude:
+                continue
+            param_shape = members[param]
+            param_section = section.add_new_section(
+                param, context={'shape': param_shape.name}
+            )
+            self._start_nested_param(param_section)
+            self.traverse_and_document_shape(
+                section=param_section,
+                shape=param_shape,
+                history=history,
+                name=param,
+            )
+        section = section.add_new_section('end-structure')
+        self._end_nested_param(section)
+
+    def _add_member_documentation(self, section, shape, **kwargs):
+        pass
+
+    def _add_members_to_shape(self, members, include):
+        if include:
+            members = members.copy()
+            for param in include:
+                members[param.name] = param
+        return members
+
+    def _document_non_top_level_param_type(self, type_section, shape):
+        special_py_type = self._get_special_py_type_name(shape)
+        py_type = py_type_name(shape.type_name)
+
+        type_format = '(%s) --'
+        if special_py_type is not None:
+            # Special type can reference a linked class.
+            # Italicizing it blows away the link.
+            type_section.write(type_format % special_py_type)
+        else:
+            type_section.style.italics(type_format % py_type)
+        type_section.write(' ')
+
+    def _start_nested_param(self, section):
+        section.style.indent()
+        section.style.new_line()
+
+    def _end_nested_param(self, section):
+        section.style.dedent()
+        section.style.new_line()
+
+
+class ResponseParamsDocumenter(BaseParamsDocumenter):
+    """Generates the description for the response parameters"""
+
+    EVENT_NAME = 'response-params'
+
+    def _add_member_documentation(self, section, shape, name=None, **kwargs):
+        name_section = section.add_new_section('param-name')
+        name_section.write('- ')
+        if name is not None:
+            name_section.style.bold('%s' % name)
+            name_section.write(' ')
+        type_section = section.add_new_section('param-type')
+        self._document_non_top_level_param_type(type_section, shape)
+
+        documentation_section = section.add_new_section('param-documentation')
+        if shape.documentation:
+            documentation_section.style.indent()
+            if getattr(shape, 'is_tagged_union', False):
+                tagged_union_docs = section.add_new_section(
+                    'param-tagged-union-docs'
+                )
+                note = (
+                    '.. note::'
+                    '    This is a Tagged Union structure. Only one of the '
+                    '    following top level keys will be set: %s. '
+                    '    If a client receives an unknown member it will '
+                    '    set ``SDK_UNKNOWN_MEMBER`` as the top level key, '
+                    '    which maps to the name or tag of the unknown '
+                    '    member. The structure of ``SDK_UNKNOWN_MEMBER`` is '
+                    '    as follows'
+                )
+                tagged_union_members_str = ', '.join(
+                    ['``%s``' % key for key in shape.members.keys()]
+                )
+                unknown_code_example = (
+                    '\'SDK_UNKNOWN_MEMBER\': '
+                    '{\'name\': \'UnknownMemberName\'}'
+                )
+                tagged_union_docs.write(note % (tagged_union_members_str))
+                example = section.add_new_section('param-unknown-example')
+                example.style.codeblock(unknown_code_example)
+            documentation_section.include_doc_string(shape.documentation)
+        section.style.new_paragraph()
+
+    def document_shape_type_event_stream(
+        self, section, shape, history, **kwargs
+    ):
+        self.document_shape_type_structure(section, shape, history, **kwargs)
+
+
+class RequestParamsDocumenter(BaseParamsDocumenter):
+    """Generates the description for the request parameters"""
+
+    EVENT_NAME = 'request-params'
+
+    def document_shape_type_structure(
+        self, section, shape, history, include=None, exclude=None, **kwargs
+    ):
+        if len(history) > 1:
+            self._add_member_documentation(section, shape, **kwargs)
+            section.style.indent()
+        members = self._add_members_to_shape(shape.members, include)
+        for i, param in enumerate(members):
+            if exclude and param in exclude:
+                continue
+            param_shape = members[param]
+            param_section = section.add_new_section(
+                param, context={'shape': param_shape.name}
+            )
+            param_section.style.new_line()
+            is_required = param in shape.required_members
+            self.traverse_and_document_shape(
+                section=param_section,
+                shape=param_shape,
+                history=history,
+                name=param,
+                is_required=is_required,
+            )
+        section = section.add_new_section('end-structure')
+        if len(history) > 1:
+            section.style.dedent()
+        section.style.new_line()
+
+    def _add_member_documentation(
+        self,
+        section,
+        shape,
+        name=None,
+        is_top_level_param=False,
+        is_required=False,
+        **kwargs,
+    ):
+        py_type = self._get_special_py_type_name(shape)
+        if py_type is None:
+            py_type = py_type_name(shape.type_name)
+        if is_top_level_param:
+            type_section = section.add_new_section('param-type')
+            type_section.write(f':type {name}: {py_type}')
+            end_type_section = type_section.add_new_section('end-param-type')
+            end_type_section.style.new_line()
+            name_section = section.add_new_section('param-name')
+            name_section.write(':param %s: ' % name)
+
+        else:
+            name_section = section.add_new_section('param-name')
+            name_section.write('- ')
+            if name is not None:
+                name_section.style.bold('%s' % name)
+                name_section.write(' ')
+            type_section = section.add_new_section('param-type')
+            self._document_non_top_level_param_type(type_section, shape)
+
+        if is_required:
+            is_required_section = section.add_new_section('is-required')
+            is_required_section.style.indent()
+            is_required_section.style.bold('[REQUIRED]')
+            is_required_section.write(' ')
+        if shape.documentation:
+            documentation_section = section.add_new_section(
+                'param-documentation'
+            )
+            documentation_section.style.indent()
+            if getattr(shape, 'is_tagged_union', False):
+                tagged_union_docs = section.add_new_section(
+                    'param-tagged-union-docs'
+                )
+                note = (
+                    '.. note::'
+                    '    This is a Tagged Union structure. Only one of the '
+                    '    following top level keys can be set: %s. '
+                )
+                tagged_union_members_str = ', '.join(
+                    ['``%s``' % key for key in shape.members.keys()]
+                )
+                tagged_union_docs.write(note % (tagged_union_members_str))
+            documentation_section.include_doc_string(shape.documentation)
+            self._add_special_trait_documentation(documentation_section, shape)
+        end_param_section = section.add_new_section('end-param')
+        end_param_section.style.new_paragraph()
+
+    def _add_special_trait_documentation(self, section, shape):
+        if 'idempotencyToken' in shape.metadata:
+            self._append_idempotency_documentation(section)
+
+    def _append_idempotency_documentation(self, section):
+        docstring = 'This field is autopopulated if not provided.'
+        section.write(docstring)

Function Source Files/botocore/docs/service.py

@@ -0,0 +1,133 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from botocore.docs.bcdoc.restdoc import DocumentStructure
+from botocore.docs.client import (
+    ClientContextParamsDocumenter,
+    ClientDocumenter,
+    ClientExceptionsDocumenter,
+)
+from botocore.docs.paginator import PaginatorDocumenter
+from botocore.docs.waiter import WaiterDocumenter
+from botocore.exceptions import DataNotFoundError
+
+
+class ServiceDocumenter:
+    def __init__(self, service_name, session, root_docs_path):
+        self._session = session
+        self._service_name = service_name
+        self._root_docs_path = root_docs_path
+
+        self._client = self._session.create_client(
+            service_name,
+            region_name='us-east-1',
+            aws_access_key_id='foo',
+            aws_secret_access_key='bar',
+        )
+        self._event_emitter = self._client.meta.events
+
+        self.sections = [
+            'title',
+            'client-api',
+            'client-exceptions',
+            'paginator-api',
+            'waiter-api',
+            'client-context-params',
+        ]
+
+    def document_service(self):
+        """Documents an entire service.
+
+        :returns: The reStructured text of the documented service.
+        """
+        doc_structure = DocumentStructure(
+            self._service_name, section_names=self.sections, target='html'
+        )
+        self.title(doc_structure.get_section('title'))
+        self.client_api(doc_structure.get_section('client-api'))
+        self.client_exceptions(doc_structure.get_section('client-exceptions'))
+        self.paginator_api(doc_structure.get_section('paginator-api'))
+        self.waiter_api(doc_structure.get_section('waiter-api'))
+        context_params_section = doc_structure.get_section(
+            'client-context-params'
+        )
+        self.client_context_params(context_params_section)
+        return doc_structure.flush_structure()
+
+    def title(self, section):
+        section.style.h1(self._client.__class__.__name__)
+        self._event_emitter.emit(
+            f"docs.title.{self._service_name}", section=section
+        )
+
+    def table_of_contents(self, section):
+        section.style.table_of_contents(title='Table of Contents', depth=2)
+
+    def client_api(self, section):
+        examples = None
+        try:
+            examples = self.get_examples(self._service_name)
+        except DataNotFoundError:
+            pass
+
+        ClientDocumenter(
+            self._client, self._root_docs_path, examples
+        ).document_client(section)
+
+    def client_exceptions(self, section):
+        ClientExceptionsDocumenter(
+            self._client, self._root_docs_path
+        ).document_exceptions(section)
+
+    def paginator_api(self, section):
+        try:
+            service_paginator_model = self._session.get_paginator_model(
+                self._service_name
+            )
+        except DataNotFoundError:
+            return
+        if service_paginator_model._paginator_config:
+            paginator_documenter = PaginatorDocumenter(
+                self._client, service_paginator_model, self._root_docs_path
+            )
+            paginator_documenter.document_paginators(section)
+
+    def waiter_api(self, section):
+        if self._client.waiter_names:
+            service_waiter_model = self._session.get_waiter_model(
+                self._service_name
+            )
+            waiter_documenter = WaiterDocumenter(
+                self._client, service_waiter_model, self._root_docs_path
+            )
+            waiter_documenter.document_waiters(section)
+
+    def get_examples(self, service_name, api_version=None):
+        loader = self._session.get_component('data_loader')
+        examples = loader.load_service_model(
+            service_name, 'examples-1', api_version
+        )
+        return examples['examples']
+
+    def client_context_params(self, section):
+        omitted_params = ClientContextParamsDocumenter.OMITTED_CONTEXT_PARAMS
+        params_to_omit = omitted_params.get(self._service_name, [])
+        service_model = self._client.meta.service_model
+        raw_context_params = service_model.client_context_parameters
+        context_params = [
+            p for p in raw_context_params if p.name not in params_to_omit
+        ]
+        if context_params:
+            context_param_documenter = ClientContextParamsDocumenter(
+                self._service_name, context_params
+            )
+            context_param_documenter.document_context_params(section)

Function Source Files/botocore/docs/shape.py

@@ -0,0 +1,135 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+
+# NOTE: This class should not be instantiated and its
+# ``traverse_and_document_shape`` method called directly. It should be
+# inherited from a Documenter class with the appropriate methods
+# and attributes.
+from botocore.utils import is_json_value_header
+
+
+class ShapeDocumenter:
+    EVENT_NAME = ''
+
+    def __init__(
+        self, service_name, operation_name, event_emitter, context=None
+    ):
+        self._service_name = service_name
+        self._operation_name = operation_name
+        self._event_emitter = event_emitter
+        self._context = context
+        if context is None:
+            self._context = {'special_shape_types': {}}
+
+    def traverse_and_document_shape(
+        self,
+        section,
+        shape,
+        history,
+        include=None,
+        exclude=None,
+        name=None,
+        is_required=False,
+    ):
+        """Traverses and documents a shape
+
+        Will take a self class and call its appropriate methods as a shape
+        is traversed.
+
+        :param section: The section to document.
+
+        :param history: A list of the names of the shapes that have been
+            traversed.
+
+        :type include: Dictionary where keys are parameter names and
+            values are the shapes of the parameter names.
+        :param include: The parameter shapes to include in the documentation.
+
+        :type exclude: List of the names of the parameters to exclude.
+        :param exclude: The names of the parameters to exclude from
+            documentation.
+
+        :param name: The name of the shape.
+
+        :param is_required: If the shape is a required member.
+        """
+        param_type = shape.type_name
+        if getattr(shape, 'serialization', {}).get('eventstream'):
+            param_type = 'event_stream'
+        if shape.name in history:
+            self.document_recursive_shape(section, shape, name=name)
+        else:
+            history.append(shape.name)
+            is_top_level_param = len(history) == 2
+            if hasattr(shape, 'is_document_type') and shape.is_document_type:
+                param_type = 'document'
+            getattr(
+                self,
+                f"document_shape_type_{param_type}",
+                self.document_shape_default,
+            )(
+                section,
+                shape,
+                history=history,
+                name=name,
+                include=include,
+                exclude=exclude,
+                is_top_level_param=is_top_level_param,
+                is_required=is_required,
+            )
+            if is_top_level_param:
+                self._event_emitter.emit(
+                    f"docs.{self.EVENT_NAME}.{self._service_name}.{self._operation_name}.{name}",
+                    section=section,
+                )
+            at_overlying_method_section = len(history) == 1
+            if at_overlying_method_section:
+                self._event_emitter.emit(
+                    f"docs.{self.EVENT_NAME}.{self._service_name}.{self._operation_name}.complete-section",
+                    section=section,
+                )
+            history.pop()
+
+    def _get_special_py_default(self, shape):
+        special_defaults = {
+            'document_type': '{...}|[...]|123|123.4|\'string\'|True|None',
+            'jsonvalue_header': '{...}|[...]|123|123.4|\'string\'|True|None',
+            'streaming_input_shape': 'b\'bytes\'|file',
+            'streaming_output_shape': 'StreamingBody()',
+            'eventstream_output_shape': 'EventStream()',
+        }
+        return self._get_value_for_special_type(shape, special_defaults)
+
+    def _get_special_py_type_name(self, shape):
+        special_type_names = {
+            'document_type': ':ref:`document<document>`',
+            'jsonvalue_header': 'JSON serializable',
+            'streaming_input_shape': 'bytes or seekable file-like object',
+            'streaming_output_shape': ':class:`.StreamingBody`',
+            'eventstream_output_shape': ':class:`.EventStream`',
+        }
+        return self._get_value_for_special_type(shape, special_type_names)
+
+    def _get_value_for_special_type(self, shape, special_type_map):
+        if is_json_value_header(shape):
+            return special_type_map['jsonvalue_header']
+        if hasattr(shape, 'is_document_type') and shape.is_document_type:
+            return special_type_map['document_type']
+        for special_type, marked_shape in self._context[
+            'special_shape_types'
+        ].items():
+            if special_type in special_type_map:
+                if shape == marked_shape:
+                    return special_type_map[special_type]
+        return None

Function Source Files/botocore/docs/sharedexample.py

@@ -0,0 +1,227 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import numbers
+import re
+
+from botocore.docs.utils import escape_controls
+from botocore.utils import parse_timestamp
+
+
+class SharedExampleDocumenter:
+    def document_shared_example(
+        self, example, prefix, section, operation_model
+    ):
+        """Documents a single shared example based on its definition.
+
+        :param example: The model of the example
+
+        :param prefix: The prefix to use in the method example.
+
+        :param section: The section to write to.
+
+        :param operation_model: The model of the operation used in the example
+        """
+        section.style.new_paragraph()
+        section.write(example.get('description'))
+        section.style.new_line()
+        self.document_input(
+            section, example, prefix, operation_model.input_shape
+        )
+        self.document_output(section, example, operation_model.output_shape)
+
+    def document_input(self, section, example, prefix, shape):
+        input_section = section.add_new_section('input')
+        input_section.style.start_codeblock()
+        if prefix is not None:
+            input_section.write(prefix)
+        params = example.get('input', {})
+        comments = example.get('comments')
+        if comments:
+            comments = comments.get('input')
+        param_section = input_section.add_new_section('parameters')
+        self._document_params(param_section, params, comments, [], shape)
+        closing_section = input_section.add_new_section('input-close')
+        closing_section.style.new_line()
+        closing_section.style.new_line()
+        closing_section.write('print(response)')
+        closing_section.style.end_codeblock()
+
+    def document_output(self, section, example, shape):
+        output_section = section.add_new_section('output')
+        output_section.style.new_line()
+        output_section.write('Expected Output:')
+        output_section.style.new_line()
+        output_section.style.start_codeblock()
+        params = example.get('output', {})
+
+        # There might not be an output, but we will return metadata anyway
+        params['ResponseMetadata'] = {"...": "..."}
+        comments = example.get('comments')
+        if comments:
+            comments = comments.get('output')
+        self._document_dict(output_section, params, comments, [], shape, True)
+        closing_section = output_section.add_new_section('output-close')
+        closing_section.style.end_codeblock()
+
+    def _document(self, section, value, comments, path, shape):
+        """
+        :param section: The section to add the docs to.
+
+        :param value: The input / output values representing the parameters that
+                      are included in the example.
+
+        :param comments: The dictionary containing all the comments to be
+                         applied to the example.
+
+        :param path: A list describing where the documenter is in traversing the
+                     parameters. This is used to find the equivalent location
+                     in the comments dictionary.
+        """
+        if isinstance(value, dict):
+            self._document_dict(section, value, comments, path, shape)
+        elif isinstance(value, list):
+            self._document_list(section, value, comments, path, shape)
+        elif isinstance(value, numbers.Number):
+            self._document_number(section, value, path)
+        elif shape and shape.type_name == 'timestamp':
+            self._document_datetime(section, value, path)
+        else:
+            self._document_str(section, value, path)
+
+    def _document_dict(
+        self, section, value, comments, path, shape, top_level=False
+    ):
+        dict_section = section.add_new_section('dict-value')
+        self._start_nested_value(dict_section, '{')
+        for key, val in value.items():
+            path.append('.%s' % key)
+            item_section = dict_section.add_new_section(key)
+            item_section.style.new_line()
+            item_comment = self._get_comment(path, comments)
+            if item_comment:
+                item_section.write(item_comment)
+                item_section.style.new_line()
+            item_section.write("'%s': " % key)
+
+            # Shape could be none if there is no output besides ResponseMetadata
+            item_shape = None
+            if shape:
+                if shape.type_name == 'structure':
+                    item_shape = shape.members.get(key)
+                elif shape.type_name == 'map':
+                    item_shape = shape.value
+            self._document(item_section, val, comments, path, item_shape)
+            path.pop()
+        dict_section_end = dict_section.add_new_section('ending-brace')
+        self._end_nested_value(dict_section_end, '}')
+        if not top_level:
+            dict_section_end.write(',')
+
+    def _document_params(self, section, value, comments, path, shape):
+        param_section = section.add_new_section('param-values')
+        self._start_nested_value(param_section, '(')
+        for key, val in value.items():
+            path.append('.%s' % key)
+            item_section = param_section.add_new_section(key)
+            item_section.style.new_line()
+            item_comment = self._get_comment(path, comments)
+            if item_comment:
+                item_section.write(item_comment)
+                item_section.style.new_line()
+            item_section.write(key + '=')
+
+            # Shape could be none if there are no input parameters
+            item_shape = None
+            if shape:
+                item_shape = shape.members.get(key)
+            self._document(item_section, val, comments, path, item_shape)
+            path.pop()
+        param_section_end = param_section.add_new_section('ending-parenthesis')
+        self._end_nested_value(param_section_end, ')')
+
+    def _document_list(self, section, value, comments, path, shape):
+        list_section = section.add_new_section('list-section')
+        self._start_nested_value(list_section, '[')
+        item_shape = shape.member
+        for index, val in enumerate(value):
+            item_section = list_section.add_new_section(index)
+            item_section.style.new_line()
+            path.append('[%s]' % index)
+            item_comment = self._get_comment(path, comments)
+            if item_comment:
+                item_section.write(item_comment)
+                item_section.style.new_line()
+            self._document(item_section, val, comments, path, item_shape)
+            path.pop()
+        list_section_end = list_section.add_new_section('ending-bracket')
+        self._end_nested_value(list_section_end, '],')
+
+    def _document_str(self, section, value, path):
+        # We do the string conversion because this might accept a type that
+        # we don't specifically address.
+        safe_value = escape_controls(value)
+        section.write(f"'{safe_value}',")
+
+    def _document_number(self, section, value, path):
+        section.write("%s," % str(value))
+
+    def _document_datetime(self, section, value, path):
+        datetime_tuple = parse_timestamp(value).timetuple()
+        datetime_str = str(datetime_tuple[0])
+        for i in range(1, len(datetime_tuple)):
+            datetime_str += ", " + str(datetime_tuple[i])
+        section.write("datetime(%s)," % datetime_str)
+
+    def _get_comment(self, path, comments):
+        key = re.sub(r'^\.', '', ''.join(path))
+        if comments and key in comments:
+            return '# ' + comments[key]
+        else:
+            return ''
+
+    def _start_nested_value(self, section, start):
+        section.write(start)
+        section.style.indent()
+        section.style.indent()
+
+    def _end_nested_value(self, section, end):
+        section.style.dedent()
+        section.style.dedent()
+        section.style.new_line()
+        section.write(end)
+
+
+def document_shared_examples(
+    section, operation_model, example_prefix, shared_examples
+):
+    """Documents the shared examples
+
+    :param section: The section to write to.
+
+    :param operation_model: The model of the operation.
+
+    :param example_prefix: The prefix to use in the method example.
+
+    :param shared_examples: The shared JSON examples from the model.
+    """
+    container_section = section.add_new_section('shared-examples')
+    container_section.style.new_paragraph()
+    container_section.style.bold('Examples')
+    documenter = SharedExampleDocumenter()
+    for example in shared_examples:
+        documenter.document_shared_example(
+            example=example,
+            section=container_section.add_new_section(example['id']),
+            prefix=example_prefix,
+            operation_model=operation_model,
+        )

Function Source Files/botocore/docs/translator.py

@@ -0,0 +1,62 @@
+# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from docutils import nodes
+from sphinx.locale import admonitionlabels
+from sphinx.writers.html5 import HTML5Translator as SphinxHTML5Translator
+
+
+class BotoHTML5Translator(SphinxHTML5Translator):
+    """Extension of Sphinx's ``HTML5Translator`` for Botocore documentation."""
+
+    IGNORE_IMPLICIT_HEADINGS = [
+        '[REQUIRED]',
+    ]
+
+    def visit_admonition(self, node, name=""):
+        """Uses the h3 tag for admonition titles instead of the p tag."""
+        self.body.append(
+            self.starttag(node, "div", CLASS=("admonition " + name))
+        )
+        if name:
+            title = (
+                f"<h3 class='admonition-title'> {admonitionlabels[name]}</h3>"
+            )
+            self.body.append(title)
+
+    def is_implicit_heading(self, node):
+        """Determines if a node is an implicit heading.
+
+        An implicit heading is represented by a paragraph node whose only
+        child is a strong node with text that isnt in `IGNORE_IMPLICIT_HEADINGS`.
+        """
+        return (
+            len(node) == 1
+            and isinstance(node[0], nodes.strong)
+            and len(node[0]) == 1
+            and isinstance(node[0][0], nodes.Text)
+            and node[0][0].astext() not in self.IGNORE_IMPLICIT_HEADINGS
+        )
+
+    def visit_paragraph(self, node):
+        """Visit a paragraph HTML element.
+
+        Replaces implicit headings with an h3 tag and defers to default
+        behavior for normal paragraph elements.
+        """
+        if self.is_implicit_heading(node):
+            text = node[0][0]
+            self.body.append(f'<h3>{text}</h3>\n')
+            # Do not visit the current nodes children or call its depart method.
+            raise nodes.SkipNode
+        else:
+            super().visit_paragraph(node)

Function Source Files/botocore/docs/utils.py

@@ -0,0 +1,222 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import re
+from collections import namedtuple
+
+
+def py_type_name(type_name):
+    """Get the Python type name for a given model type.
+
+        >>> py_type_name('list')
+        'list'
+        >>> py_type_name('structure')
+        'dict'
+
+    :rtype: string
+    """
+    return {
+        'blob': 'bytes',
+        'character': 'string',
+        'double': 'float',
+        'long': 'integer',
+        'map': 'dict',
+        'structure': 'dict',
+        'timestamp': 'datetime',
+    }.get(type_name, type_name)
+
+
+def py_default(type_name):
+    """Get the Python default value for a given model type.
+
+        >>> py_default('string')
+        '\'string\''
+        >>> py_default('list')
+        '[...]'
+        >>> py_default('unknown')
+        '...'
+
+    :rtype: string
+    """
+    return {
+        'double': '123.0',
+        'long': '123',
+        'integer': '123',
+        'string': "'string'",
+        'blob': "b'bytes'",
+        'boolean': 'True|False',
+        'list': '[...]',
+        'map': '{...}',
+        'structure': '{...}',
+        'timestamp': 'datetime(2015, 1, 1)',
+    }.get(type_name, '...')
+
+
+def get_official_service_name(service_model):
+    """Generate the official name of an AWS Service
+
+    :param service_model: The service model representing the service
+    """
+    official_name = service_model.metadata.get('serviceFullName')
+    short_name = service_model.metadata.get('serviceAbbreviation', '')
+    if short_name.startswith('Amazon'):
+        short_name = short_name[7:]
+    if short_name.startswith('AWS'):
+        short_name = short_name[4:]
+    if short_name and short_name.lower() not in official_name.lower():
+        official_name += f' ({short_name})'
+    return official_name
+
+
+_DocumentedShape = namedtuple(
+    'DocumentedShape',
+    [
+        'name',
+        'type_name',
+        'documentation',
+        'metadata',
+        'members',
+        'required_members',
+    ],
+)
+
+
+class DocumentedShape(_DocumentedShape):
+    """Use this class to inject new shapes into a model for documentation"""
+
+    def __new__(
+        cls,
+        name,
+        type_name,
+        documentation,
+        metadata=None,
+        members=None,
+        required_members=None,
+    ):
+        if metadata is None:
+            metadata = []
+        if members is None:
+            members = []
+        if required_members is None:
+            required_members = []
+        return super().__new__(
+            cls,
+            name,
+            type_name,
+            documentation,
+            metadata,
+            members,
+            required_members,
+        )
+
+
+class AutoPopulatedParam:
+    def __init__(self, name, param_description=None):
+        self.name = name
+        self.param_description = param_description
+        if param_description is None:
+            self.param_description = (
+                'Please note that this parameter is automatically populated '
+                'if it is not provided. Including this parameter is not '
+                'required\n'
+            )
+
+    def document_auto_populated_param(self, event_name, section, **kwargs):
+        """Documents auto populated parameters
+
+        It will remove any required marks for the parameter, remove the
+        parameter from the example, and add a snippet about the parameter
+        being autopopulated in the description.
+        """
+        if event_name.startswith('docs.request-params'):
+            if self.name in section.available_sections:
+                section = section.get_section(self.name)
+                if 'is-required' in section.available_sections:
+                    section.delete_section('is-required')
+                description_section = section.get_section(
+                    'param-documentation'
+                )
+                description_section.writeln(self.param_description)
+        elif event_name.startswith('docs.request-example'):
+            section = section.get_section('structure-value')
+            if self.name in section.available_sections:
+                section.delete_section(self.name)
+
+
+class HideParamFromOperations:
+    """Hides a single parameter from multiple operations.
+
+    This method will remove a parameter from documentation and from
+    examples. This method is typically used for things that are
+    automatically populated because a user would be unable to provide
+    a value (e.g., a checksum of a serialized XML request body)."""
+
+    def __init__(self, service_name, parameter_name, operation_names):
+        """
+        :type service_name: str
+        :param service_name: Name of the service to modify.
+
+        :type parameter_name: str
+        :param parameter_name: Name of the parameter to modify.
+
+        :type operation_names: list
+        :param operation_names: Operation names to modify.
+        """
+        self._parameter_name = parameter_name
+        self._params_events = set()
+        self._example_events = set()
+        # Build up the sets of relevant event names.
+        param_template = 'docs.request-params.%s.%s.complete-section'
+        example_template = 'docs.request-example.%s.%s.complete-section'
+        for name in operation_names:
+            self._params_events.add(param_template % (service_name, name))
+            self._example_events.add(example_template % (service_name, name))
+
+    def hide_param(self, event_name, section, **kwargs):
+        if event_name in self._example_events:
+            # Modify the structure value for example events.
+            section = section.get_section('structure-value')
+        elif event_name not in self._params_events:
+            return
+        if self._parameter_name in section.available_sections:
+            section.delete_section(self._parameter_name)
+
+
+class AppendParamDocumentation:
+    """Appends documentation to a specific parameter"""
+
+    def __init__(self, parameter_name, doc_string):
+        self._parameter_name = parameter_name
+        self._doc_string = doc_string
+
+    def append_documentation(self, event_name, section, **kwargs):
+        if self._parameter_name in section.available_sections:
+            section = section.get_section(self._parameter_name)
+            description_section = section.get_section('param-documentation')
+            description_section.writeln(self._doc_string)
+
+
+_CONTROLS = {
+    '\n': '\\n',
+    '\r': '\\r',
+    '\t': '\\t',
+    '\b': '\\b',
+    '\f': '\\f',
+}
+# Combines all CONTROLS keys into a big or regular expression
+_ESCAPE_CONTROLS_RE = re.compile('|'.join(map(re.escape, _CONTROLS)))
+# Based on the match get the appropriate replacement from CONTROLS
+_CONTROLS_MATCH_HANDLER = lambda match: _CONTROLS[match.group(0)]
+
+
+def escape_controls(value):
+    return _ESCAPE_CONTROLS_RE.sub(_CONTROLS_MATCH_HANDLER, value)

Function Source Files/botocore/docs/waiter.py

@@ -0,0 +1,184 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import os
+
+from botocore import xform_name
+from botocore.compat import OrderedDict
+from botocore.docs.bcdoc.restdoc import DocumentStructure
+from botocore.docs.method import document_model_driven_method
+from botocore.docs.utils import DocumentedShape
+from botocore.utils import get_service_module_name
+
+
+class WaiterDocumenter:
+    def __init__(self, client, service_waiter_model, root_docs_path):
+        self._client = client
+        self._client_class_name = self._client.__class__.__name__
+        self._service_name = self._client.meta.service_model.service_name
+        self._service_waiter_model = service_waiter_model
+        self._root_docs_path = root_docs_path
+        self._USER_GUIDE_LINK = (
+            'https://boto3.amazonaws.com/'
+            'v1/documentation/api/latest/guide/clients.html#waiters'
+        )
+
+    def document_waiters(self, section):
+        """Documents the various waiters for a service.
+
+        :param section: The section to write to.
+        """
+        section.style.h2('Waiters')
+        self._add_overview(section)
+        section.style.new_line()
+        section.writeln('The available waiters are:')
+        section.style.toctree()
+        for waiter_name in self._service_waiter_model.waiter_names:
+            section.style.tocitem(f'{self._service_name}/waiter/{waiter_name}')
+            # Create a new DocumentStructure for each waiter and add contents.
+            waiter_doc_structure = DocumentStructure(
+                waiter_name, target='html'
+            )
+            self._add_single_waiter(waiter_doc_structure, waiter_name)
+            # Write waiters in individual/nested files.
+            # Path: <root>/reference/services/<service>/waiter/<waiter_name>.rst
+            waiter_dir_path = os.path.join(
+                self._root_docs_path, self._service_name, 'waiter'
+            )
+            waiter_doc_structure.write_to_file(waiter_dir_path, waiter_name)
+
+    def _add_single_waiter(self, section, waiter_name):
+        breadcrumb_section = section.add_new_section('breadcrumb')
+        breadcrumb_section.style.ref(
+            self._client_class_name, f'../../{self._service_name}'
+        )
+        breadcrumb_section.write(f' / Waiter / {waiter_name}')
+        section.add_title_section(waiter_name)
+        waiter_section = section.add_new_section(waiter_name)
+        waiter_section.style.start_sphinx_py_class(
+            class_name=f"{self._client_class_name}.Waiter.{waiter_name}"
+        )
+
+        # Add example on how to instantiate waiter.
+        waiter_section.style.start_codeblock()
+        waiter_section.style.new_line()
+        waiter_section.write(
+            'waiter = client.get_waiter(\'%s\')' % xform_name(waiter_name)
+        )
+        waiter_section.style.end_codeblock()
+
+        # Add information on the wait() method
+        waiter_section.style.new_line()
+        document_wait_method(
+            section=waiter_section,
+            waiter_name=waiter_name,
+            event_emitter=self._client.meta.events,
+            service_model=self._client.meta.service_model,
+            service_waiter_model=self._service_waiter_model,
+        )
+
+    def _add_overview(self, section):
+        section.style.new_line()
+        section.write(
+            'Waiters are available on a client instance '
+            'via the ``get_waiter`` method. For more detailed instructions '
+            'and examples on the usage or waiters, see the '
+            'waiters '
+        )
+        section.style.external_link(
+            title='user guide',
+            link=self._USER_GUIDE_LINK,
+        )
+        section.write('.')
+        section.style.new_line()
+
+
+def document_wait_method(
+    section,
+    waiter_name,
+    event_emitter,
+    service_model,
+    service_waiter_model,
+    include_signature=True,
+):
+    """Documents a the wait method of a waiter
+
+    :param section: The section to write to
+
+    :param waiter_name: The name of the waiter
+
+    :param event_emitter: The event emitter to use to emit events
+
+    :param service_model: The service model
+
+    :param service_waiter_model: The waiter model associated to the service
+
+    :param include_signature: Whether or not to include the signature.
+        It is useful for generating docstrings.
+    """
+    waiter_model = service_waiter_model.get_waiter(waiter_name)
+    operation_model = service_model.operation_model(waiter_model.operation)
+
+    waiter_config_members = OrderedDict()
+
+    waiter_config_members['Delay'] = DocumentedShape(
+        name='Delay',
+        type_name='integer',
+        documentation=(
+            '<p>The amount of time in seconds to wait between '
+            'attempts. Default: {}</p>'.format(waiter_model.delay)
+        ),
+    )
+
+    waiter_config_members['MaxAttempts'] = DocumentedShape(
+        name='MaxAttempts',
+        type_name='integer',
+        documentation=(
+            '<p>The maximum number of attempts to be made. '
+            'Default: {}</p>'.format(waiter_model.max_attempts)
+        ),
+    )
+
+    botocore_waiter_params = [
+        DocumentedShape(
+            name='WaiterConfig',
+            type_name='structure',
+            documentation=(
+                '<p>A dictionary that provides parameters to control '
+                'waiting behavior.</p>'
+            ),
+            members=waiter_config_members,
+        )
+    ]
+
+    wait_description = (
+        'Polls :py:meth:`{}.Client.{}` every {} '
+        'seconds until a successful state is reached. An error is '
+        'returned after {} failed checks.'.format(
+            get_service_module_name(service_model),
+            xform_name(waiter_model.operation),
+            waiter_model.delay,
+            waiter_model.max_attempts,
+        )
+    )
+
+    document_model_driven_method(
+        section,
+        'wait',
+        operation_model,
+        event_emitter=event_emitter,
+        method_description=wait_description,
+        example_prefix='waiter.wait',
+        include_input=botocore_waiter_params,
+        document_output=False,
+        include_signature=include_signature,
+    )