diff --git a/deep_rf/deep_rf_learner.py b/deep_rf/deep_rf_learner.py index 4eccd53..33e8978 100644 --- a/deep_rf/deep_rf_learner.py +++ b/deep_rf/deep_rf_learner.py @@ -14,7 +14,18 @@ import epsilon_method class ExperienceTuple: - """ ExperienceTuple data structure for DeepRFLearner """ + """ ExperienceTuple data structure for DeepRFLearner + + A data structure for keeping track of + tuples. These tuples are used in learning the Q function. + + Args: + state (State): a collection of frames + action (int): an action taken + reward (double): measured reward for taking the selected action + next_state (State): the set of frames after taking the selected action + + """ def __init__(self, state, action, reward, next_state): self.state = state self.action = action @@ -48,21 +59,21 @@ class DeepRFLearner(object): """ DeepRFLearner Class Args: - game: - q_graph: - num_frames: - reward_function: + game (SinglePlayerGame): single player game to learn + q_graph (QGraph): tensorflow Q graph + num_frames (int): number of frames to use as the state + reward_function (func): A function taking a dictionary of parameters and returning a double. Dict args include: 'last_score', 'new_score', 'last_state', 'new_state', 'is_game_over'. - file_save_path: + file_save_path (string): path to save location Methods: - get_next_experience_tuple: - choose_action: - evaluate_q_function: - learn_q_function: - save_tf_weights: + * get_next_experience_tuple + * choose_action + * evaluate_q_function + * learn_q_function + * save_tf_weights """ @@ -108,6 +119,7 @@ def __del__(self): def save_tf_weights(self): + """ Save the Q-network weights """ if self.file_save_path is not None: self._saver.save(self._sess, self.file_save_path) @@ -123,6 +135,21 @@ def _init_training_loss_operation(self): def learn_q_function(self, num_iterations=1000, batch_size=50, num_training_steps=10): + """ Learn deep reinforcement learning Q function. + + Train the Q function by repeatedly playing the single player game. + + Args: + num_iterations (int): number of training iterations + batch_size (int): number of experience tuples to add in each step + num_training_steps (int): + number of optimization steps to take in each iteration. + + Returns: + None: updates the Q function + + """ + # For Training Time # Get next sample -> List of ExperienceTuples # Get a minibatch -> partially Optimize Q for loss @@ -150,10 +177,10 @@ def learn_q_function(self, num_iterations=1000, batch_size=50, def _get_target_values(self, experience_batch): """ Args: - experience_batch: list of ExperienceTuples + experience_batch (list): list of ExperienceTuples Returns: - y_target: np.ndarray of [batch_size, r + max Q(s')] + y_target (ndarray): ndarray of [batch_size, r + max Q(s')] """ rewards = np.array([et.reward for et in experience_batch]) states = [ @@ -173,8 +200,8 @@ def get_next_experience_tuple(self): DeepRFLearner chooses an action based on the Q function and random exploration - yields: - experience_tuple (Experience Tuple) - current state, action, reward, new_state + Yields: + experience_tuple (ExperienceTuple): current state, action, reward, new_state """ while True: self._game.reset() @@ -210,10 +237,10 @@ def choose_action(self, state): """ Return the action with the highest q_function value Args: - state: A State object or list of State objects + state (State): A State object or list of State objects Return: - actions: the action or list of actions that maximize + actions (Action): the action or list of actions that maximize the q_function for each state """ if isinstance(state, State): @@ -235,11 +262,12 @@ def evaluate_q_function(self, state): """ Return q_values for for given state(s) Args: - state: A State object or list of State objects + state (State): A State object or list of State objects Return: - q_values: An ndarray of size(action_list) for a state object - An ndarray of # States by size(action_list) for a list + q_values (ndarray): Either + * An ndarray of size(action_list) for a state object + * An ndarray of # States by size(action_list) for a list """ if isinstance(state, State): diff --git a/deep_rf/q_graph.py b/deep_rf/q_graph.py index 2296194..9766963 100644 --- a/deep_rf/q_graph.py +++ b/deep_rf/q_graph.py @@ -3,9 +3,19 @@ class QGraph(object): - """ - q_input: (tf.placeholder float [None, board_height, board_width, num_frames]) - tf placeholder for state - q_output: (tf.Tensor of action_values [None, num_actions]) - Q function output to evaluated with tf.run() + """ Data structure for the Q Function + + Args: + q_input (tf.placeholder float [None, board_height, board_width, num_frames]): + tf placeholder for frame input + q_output (tf.Tensor of action_values [None, num_actions]): + Q function output to evaluated with tf.run() + + Attributes: + q_input (tf.placeholder) : input to Q function + q_output (tf.Tensor): Q function output + graph (tf.Graph): tensorflow graph containing the Q function + var_list (list): list of names for Q function weights """ def __init__(self, q_input, q_output): @@ -19,14 +29,17 @@ def __init__(self, q_input, q_output): def default_q_graph(game, num_frames): """ initialize Q function input & output + Parameters: + game (SinglePlayerGame): a game object + num_frames (int): number of past frames to keep in memory + Returns: - QGraph: a q graph + QGraph (QGraph): a q graph """ g = tf.Graph() with g.as_default(): - # input layer q_input = tf.placeholder(dtype=tf.float32, shape=[None, game.frame_height, @@ -65,4 +78,4 @@ def default_q_graph(game, num_frames): b_fc2 = _utils.init_fc_bias(length=len(game.action_list)) q_output = tf.matmul(h_fc1, w_fc2) + b_fc2 - return QGraph(q_input, q_output) \ No newline at end of file + return QGraph(q_input, q_output) diff --git a/deep_rf/single_player_game.py b/deep_rf/single_player_game.py index 255e649..8ebd6b2 100644 --- a/deep_rf/single_player_game.py +++ b/deep_rf/single_player_game.py @@ -4,9 +4,25 @@ """ +class SinglePlayerGame: + """ A virtual class for single player games + This contains the class skeleton for a single player game. + To be used in deep reinforcement learning. + + Args: + action_list (list): set of game actions + frame_width (int): non-negative size of game window width + frame_height (int): non-negative size of game window height + + Attributes: + action_list (list): set of game actions + action_dict (dict): enumeration of action_list + + See Also: + deep_rf.deep_rf_learner.DeepRFLearner + """ -class SinglePlayerGame: def __init__(self, action_list, frame_height, frame_width): self.action_list = action_list self.action_dict = {self.action_list[i]: i for i in @@ -16,25 +32,47 @@ def __init__(self, action_list, frame_height, frame_width): @property def frame_height(self): + """ frame_height (int): non-negative size of game window height """ return self._frame_height @property def frame_width(self): + """ frame_width (int): non-negative size of game window width """ return self._frame_width @property def score(self): + """ score (double): current score of game """ raise NotImplementedError('Subclass should define get_score()') def do_action(self, action): + """ Apply player's selected action to current game. + + Args: + action (Action): action to perform + + Returns: + None: applies action to game + """ raise NotImplementedError('Subclass should define do_action()') def get_frame(self): + """ Return the pixels for the current game + + Returns: + frame (ndarray): returns the frame_height by frame_width game window. + """ raise NotImplementedError('Subclass should define get_frame()') def is_game_over(self): + """ Return whether the game has ended + + Returns: + isGameOver (bool): returns whether the game has ended + """ raise NotImplementedError('Subclass should define is_game_over()') def reset(self): + """ Start a new game. """ raise NotImplementedError('Subclass should define reset()') diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..28215ee --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,230 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) + $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " epub3 to make an epub3" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + @echo " dummy to check syntax errors of document sources" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/DeepRF.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/DeepRF.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/DeepRF" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/DeepRF" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: epub3 +epub3: + $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 + @echo + @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +.PHONY: dummy +dummy: + $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy + @echo + @echo "Build finished. Dummy builder generates no files." diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..d1de987 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,64 @@ +## Quick-Start + +* Install `sphinx` using `pip` +* Follow the Google style guide for docstrings +* Call `sphinx-apidoc` on the directory with our python modules +to generate '.rst' files +* Call `make html` within the 'docs' folder to build 'html' documentation + +See below for more details. + +## Docstring Style Guide +There are many different docstring style guidelines. + +We choose to follow the Google style guide for readability: +* [Google style guide](https://google.github.io/styleguide/pyguide.html) +* [Sphinx example module](http://www.sphinx-doc.org/en/stable/ext/example_google.html) + +## Auto-Doc Generation with `sphinx` +To create 'html' and 'pdf' documentation of our module, we use [`sphinx`](http://www.sphinx-doc.org/en/stable/tutorial.html). +Sphinx is a utility for generating python documentation that uses reStructured text (rst) files. + +### Installation +Install `sphinx` using `pip` +``` +$ pip install Sphinx +``` + +### Setting Up `sphinx` +To setup the documentation sources, first switch to the `docs` directory of the project. Then call `sphinx-quickstart` +``` +$ mkdir docs +$ cd docs +$ sphinx-quickstart +``` +Be sure to say yes to the "autodoc" extension. It is also recommend to separate the `source` and `build` directories. + +`sphinx-quickstart` should generate +* `Makefile` - a makefile for building the documention (i.e. call `$ make html$` to build html documentation) +* `conf.py` - a configuration file for `sphinx`. Edit this to change settings and add extensions +* `index.rst` - the home page for documentation + +To get "autodoc" to recognize the Google style, enable `sphinx.ext.napoleon` by editing `conf.py`. +See [sphinx support for Google style docstrings](http://www.sphinx-doc.org/en/stable/ext/napoleon.html) for more details. + +### Adding Module Documentation +To add documentation, we can either modify `rst` files manually or use `sphinx-apidoc` ([example_link](https://codeandchaos.wordpress.com/2012/07/30/sphinx-autodoc-tutorial-for-dummies/), [apidoc documentation](http://www.sphinx-doc.org/en/stable/man/sphinx-apidoc.html)) . + +I recommend using `sphinx-apidoc` to generate the `rst` files and then modify them manually. +To create documentation call +``` +$ sphinx-apidoc -o +``` +where `` is the path to the module folder (e.g. `src/`) and `` is `docs/source`. +Note that `sphinx-apidoc` will import each module in ``. + +Useful `sphinx-apidoc` options (`--separate` for separate pages and `--private` to include all functions) + + +### Documentation Output Formatting +To change the html output style, change sthe `html_theme` value in `conf.py`. +(http://www.sphinx-doc.org/en/stable/theming.html) + + + diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..4f03842 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,303 @@ +# -*- coding: utf-8 -*- +# +# Deep RF documentation build configuration file, created by +# sphinx-quickstart on Sun Sep 4 15:10:00 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', +] + +# napoleon settings +napoleon_google_docstring = True +napoleon_numpy_docstring = True +napoleon_include_init_with_doc = False +napoleon_include_private_with_doc = False +napoleon_include_special_with_doc = False +napoleon_use_admonition_for_examples = False +napoleon_use_admonition_for_notes = False +napoleon_use_admonition_for_references = False +napoleon_use_ivar = False +napoleon_use_param = True +napoleon_use_rtype = True +napoleon_use_keyword = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Deep RF' +copyright = u'2016, Solstat' +author = u'Solstat' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0.1.0' +# The full version, including alpha/beta/rc tags. +release = u'0.1.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'classic' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. +# " v documentation" by default. +#html_title = u'Deep RF v0.1.0' + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (relative to this directory) to use as a favicon of +# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not None, a 'Last updated on:' timestamp is inserted at every page +# bottom, using the given strftime format. +# The empty string is equivalent to '%b %d, %Y'. +#html_last_updated_fmt = None + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# 'ja' uses this config value. +# 'zh' user can custom change `jieba` dictionary path. +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'DeepRFdoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'DeepRF.tex', u'Deep RF Documentation', + u'Solstat', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'deeprf', u'Deep RF Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'DeepRF', u'Deep RF Documentation', + author, 'DeepRF', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/docs/source/deep_rf.deep_rf_learner.rst b/docs/source/deep_rf.deep_rf_learner.rst new file mode 100644 index 0000000..6e811ef --- /dev/null +++ b/docs/source/deep_rf.deep_rf_learner.rst @@ -0,0 +1,7 @@ +deep_rf.deep_rf_learner module +============================== + +.. automodule:: deep_rf.deep_rf_learner + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/deep_rf.epsilon_method.rst b/docs/source/deep_rf.epsilon_method.rst new file mode 100644 index 0000000..d9b77ae --- /dev/null +++ b/docs/source/deep_rf.epsilon_method.rst @@ -0,0 +1,7 @@ +deep_rf.epsilon_method module +============================= + +.. automodule:: deep_rf.epsilon_method + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/deep_rf.q_graph.rst b/docs/source/deep_rf.q_graph.rst new file mode 100644 index 0000000..3e491c6 --- /dev/null +++ b/docs/source/deep_rf.q_graph.rst @@ -0,0 +1,7 @@ +deep_rf.q_graph module +====================== + +.. automodule:: deep_rf.q_graph + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/deep_rf.rst b/docs/source/deep_rf.rst new file mode 100644 index 0000000..666bb1a --- /dev/null +++ b/docs/source/deep_rf.rst @@ -0,0 +1,46 @@ +deep_rf package +=============== + +Submodules +---------- + +deep_rf.deep_rf_learner module +------------------------------ + +.. automodule:: deep_rf.deep_rf_learner + :members: + :undoc-members: + :show-inheritance: + +deep_rf.epsilon_method module +----------------------------- + +.. automodule:: deep_rf.epsilon_method + :members: + :undoc-members: + :show-inheritance: + +deep_rf.q_graph module +---------------------- + +.. automodule:: deep_rf.q_graph + :members: + :undoc-members: + :show-inheritance: + +deep_rf.single_player_game module +--------------------------------- + +.. automodule:: deep_rf.single_player_game + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: deep_rf + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/deep_rf.single_player_game.rst b/docs/source/deep_rf.single_player_game.rst new file mode 100644 index 0000000..aa8ea5c --- /dev/null +++ b/docs/source/deep_rf.single_player_game.rst @@ -0,0 +1,7 @@ +deep_rf.single_player_game module +================================= + +.. automodule:: deep_rf.single_player_game + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..3ffad1b --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,20 @@ +.. Deep RF documentation master file, created by + sphinx-quickstart on Sun Sep 4 15:10:00 2016. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Deep RF's documentation! +=================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/source/modules.rst b/docs/source/modules.rst new file mode 100644 index 0000000..bf30452 --- /dev/null +++ b/docs/source/modules.rst @@ -0,0 +1,7 @@ +deep_rf +======= + +.. toctree:: + :maxdepth: 4 + + deep_rf diff --git a/docs/source/snake.rst b/docs/source/snake.rst new file mode 100644 index 0000000..9e24daf --- /dev/null +++ b/docs/source/snake.rst @@ -0,0 +1,17 @@ +snake package +============= + +Submodules +---------- + +.. toctree:: + + snake.snake_game + +Module contents +--------------- + +.. automodule:: snake + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/snake.snake_game.rst b/docs/source/snake.snake_game.rst new file mode 100644 index 0000000..c244896 --- /dev/null +++ b/docs/source/snake.snake_game.rst @@ -0,0 +1,7 @@ +snake.snake_game module +======================= + +.. automodule:: snake.snake_game + :members: + :undoc-members: + :show-inheritance: diff --git a/snake/__init__.py b/snake/__init__.py index e69de29..6ba0be0 100644 --- a/snake/__init__.py +++ b/snake/__init__.py @@ -0,0 +1 @@ +import snake_game diff --git a/snake/snake_game.py b/snake/snake_game.py index 404e306..17c2b10 100644 --- a/snake/snake_game.py +++ b/snake/snake_game.py @@ -1,6 +1,6 @@ """ -Single Player Game class +Snake Game Implementation for deep reinforcement learning """ @@ -10,6 +10,7 @@ _SNAKE_ACTION_LIST = ['UP', 'DOWN', 'LEFT', 'RIGHT'] class SnakeGame(SinglePlayerGame): + """ Implementation of Snake as a SinglePlayerGame """ def __init__(self, board_height=20, board_width=20): SinglePlayerGame.__init__(self, action_list=_SNAKE_ACTION_LIST, frame_height = board_height, @@ -157,16 +158,19 @@ def __str__(self): class Snake(object): - """ - Args: - initial_location (dictionary {'x': int, 'y': int}) - Attributes: - body (list of dictionaries {'x': int, 'y': int}) - direction (int from action_dict) - Methods: - move - move body in direction - grow - append point to end of body - is_self_collision - check if snake has collided with itself + """ Class that handles the location, size, and direction of the snake + + Snake extended description + + Parameters + ---------- + initial_location : dict {'x': int, 'y': int} + initial location of snake + + Attributes + ---------- + body : list of dict + list of dictionaries {'x': int, 'y': int}) """ def __init__(self, initial_location): self.body = [initial_location] @@ -175,6 +179,9 @@ def __init__(self, initial_location): @property def direction(self): + """ direction : {'UP', 'DOWN', 'LEFT', 'RIGHT'} + direction of Snake + """ return self._direction @direction.setter @@ -183,6 +190,7 @@ def direction(self, value): self._direction = value def move(self): + """ Move the snake one step in its current direction """ head = self.body[0].copy() # deep copy if self._direction == 'UP': head['y'] = head['y'] + 1 @@ -197,10 +205,24 @@ def move(self): self.body.pop() def grow(self): + """ Increase the length of the snake + + Appends a pixel to the end of the snake's current body + + Parameters + ---------- + None + + Returns + ------- + None + + """ tail = self.body[-1] self.body.append(tail) def is_self_collision(self): + """ Checks if the snake's head intersects its body """ head = self.body[0] if self.body.count(head) > 1: return True