feat: add Markdown formatter

Signed-off-by: Yordis Prieto <[email protected]>

Author: yordis

lib/ex_doc.ex

@@ -35,11 +35,16 @@ defmodule ExDoc do
   end
 
   defp find_formatter(name) do
-    [ExDoc.Formatter, String.upcase(name)]
+    [ExDoc.Formatter, format_module_name(name)]
     |> Module.concat()
     |> check_formatter_module(name)
   end
 
+  defp format_module_name("html"), do: "HTML"
+  defp format_module_name("epub"), do: "EPUB"
+  defp format_module_name("markdown"), do: "Markdown"
+  defp format_module_name(name), do: String.upcase(name)
+
   defp check_formatter_module(modname, argname) do
     if Code.ensure_loaded?(modname) do
       modname

lib/ex_doc/doc_ast.ex

@@ -65,6 +65,67 @@ defmodule ExDoc.DocAST do
     Enum.map(attrs, fn {key, val} -> " #{key}=\"#{ExDoc.Utils.h(val)}\"" end)
   end
 
+  @doc """
+  Transform AST into markdown string.
+  """
+  def to_markdown(ast)
+
+  def to_markdown(binary) when is_binary(binary) do
+    ExDoc.Utils.h(binary)
+  end
+
+  def to_markdown(list) when is_list(list) do
+    Enum.map_join(list, "", &to_markdown/1)
+  end
+
+  def to_markdown({:comment, _attrs, inner, _meta}) do
+    "<!--#{inner}-->"
+  end
+
+  def to_markdown({:code, attrs, inner, _meta}) do
+    lang = attrs[:class] || ""
+
+    """
+    ```#{lang}
+    #{inner}
+    ```
+    """
+  end
+
+  def to_markdown({:a, attrs, inner, _meta}) do
+    "[#{to_markdown(inner)}](#{attrs[:href]})"
+  end
+
+  def to_markdown({:hr, _attrs, _inner, _meta}) do
+    "\n\n---\n\n"
+  end
+
+  def to_markdown({:p, _attrs, inner, _meta}) do
+    to_markdown(inner) <> "\n\n"
+  end
+
+  def to_markdown({:br, _attrs, _inner, _meta}) do
+    "\n\n"
+  end
+
+  def to_markdown({:img, attrs, _inner, _meta}) do
+    alt = attrs[:alt] || ""
+    title = attrs[:title] || ""
+    "![#{alt}](#{attrs[:src]} \"#{title}\")"
+  end
+
+  def to_markdown({tag, _attrs, _inner, _meta}) when tag in @void_elements do
+    ""
+  end
+
+  def to_markdown({_tag, _attrs, inner, %{verbatim: true}}) do
+    Enum.join(inner, "")
+  end
+
+  def to_markdown({_tag, _attrs, inner, _meta}) do
+    to_markdown(inner)
+  end
+
   ## parse markdown
 
   defp parse_markdown(markdown, opts) do

lib/ex_doc/formatter.ex

@@ -48,14 +48,14 @@ defmodule ExDoc.Formatter do
 
                 specs = Enum.map(child_node.specs, &language.autolink_spec(&1, autolink_opts))
                 child_node = %{child_node | specs: specs}
-                render_doc(child_node, language, autolink_opts, opts)
+                render_doc(child_node, ext, language, autolink_opts, opts)
               end
 
-            %{render_doc(group, language, autolink_opts, opts) | docs: docs}
+            %{render_doc(group, ext, language, autolink_opts, opts) | docs: docs}
           end
 
         %{
-          render_doc(node, language, [{:id, node.id} | autolink_opts], opts)
+          render_doc(node, ext, language, [{:id, node.id} | autolink_opts], opts)
           | docs_groups: docs_groups
         }
       end,
@@ -117,11 +117,11 @@ defmodule ExDoc.Formatter do
 
   # Helper functions
 
-  defp render_doc(%{doc: nil} = node, _language, _autolink_opts, _opts),
+  defp render_doc(%{doc: nil} = node, _ext, _language, _autolink_opts, _opts),
     do: node
 
-  defp render_doc(%{doc: doc} = node, language, autolink_opts, opts) do
-    doc = autolink_and_highlight(doc, language, autolink_opts, opts)
+  defp render_doc(%{doc: doc} = node, ext, language, autolink_opts, opts) do
+    doc = autolink_and_render(doc, ext, language, autolink_opts, opts)
     %{node | doc: doc}
   end
 
@@ -137,7 +137,13 @@ defmodule ExDoc.Formatter do
     mod_id <> "." <> id
   end
 
-  defp autolink_and_highlight(doc, language, autolink_opts, opts) do
+  defp autolink_and_render(doc, ".md", language, autolink_opts, opts) do
+    doc
+    |> language.autolink_doc(autolink_opts)
+    |> ExDoc.DocAST.highlight(language, opts)
+  end
+
+  defp autolink_and_render(doc, _html_ext, language, autolink_opts, opts) do
     doc
     |> language.autolink_doc(autolink_opts)
     |> ExDoc.DocAST.highlight(language, opts)
@@ -187,6 +193,7 @@ defmodule ExDoc.Formatter do
 
     source_file = validate_extra_string!(input_options, :source) || input
     opts = [file: source_file, line: 1]
+    ext = Keyword.fetch!(autolink_opts, :ext)
 
     {extension, source, ast} =
       case extension_name(input) do
@@ -202,7 +209,7 @@ defmodule ExDoc.Formatter do
             source
             |> Markdown.to_ast(opts)
             |> ExDoc.DocAST.add_ids_to_headers([:h2, :h3])
-            |> autolink_and_highlight(language, [file: input] ++ autolink_opts, opts)
+            |> autolink_and_render(ext, language, [file: input] ++ autolink_opts, opts)
 
           {extension, source, ast}
 

lib/ex_doc/formatter/epub/templates.ex

@@ -13,14 +13,14 @@ defmodule ExDoc.Formatter.EPUB.Templates do
   defp render_doc(ast), do: ast && ExDoc.DocAST.to_string(ast)
 
   @doc """
-  Generate content from the module template for a given `node`
+  Generate content from the module template for a given `node`.
   """
   def module_page(config, module_node) do
     module_template(config, module_node)
   end
 
   @doc """
-  Generated ID for static file
+  Generated ID for static file.
   """
   def static_file_to_id(static_file) do
     static_file |> Path.basename() |> text_to_id()

lib/ex_doc/formatter/html/templates.ex

@@ -13,7 +13,7 @@ defmodule ExDoc.Formatter.HTML.Templates do
     ]
 
   @doc """
-  Generate content from the module template for a given `node`
+  Generate content from the module template for a given `node`.
   """
   def module_page(module_node, config) do
     module_template(config, module_node)

lib/ex_doc/formatter/markdown.ex

@@ -0,0 +1,220 @@
+defmodule ExDoc.Formatter.Markdown do
+  @moduledoc false
+
+  alias __MODULE__.{Templates}
+  alias ExDoc.Formatter
+  alias ExDoc.Utils
+
+  @doc """
+  Generates Markdown documentation for the given modules.
+  """
+  @spec run([ExDoc.ModuleNode.t()], [ExDoc.ModuleNode.t()], ExDoc.Config.t()) :: String.t()
+  def run(project_nodes, filtered_modules, config) when is_map(config) do
+    Utils.unset_warned()
+
+    config = normalize_config(config)
+    build = Path.join(config.output, ".build-markdown")
+    output_setup(build, config)
+
+    extras = Formatter.build_extras(config, ".md")
+
+    project_nodes =
+      project_nodes
+      |> Formatter.render_all(filtered_modules, ".md", config, highlight_tag: "samp")
+
+    nodes_map = %{
+      modules: Formatter.filter_list(:module, project_nodes),
+      tasks: Formatter.filter_list(:task, project_nodes)
+    }
+
+    config = %{config | extras: extras}
+
+    all_files =
+      [generate_nav(config, nodes_map)] ++
+        generate_extras(config) ++
+        generate_list(config, nodes_map.modules) ++
+        generate_list(config, nodes_map.tasks) ++
+        [generate_llm_index(config, nodes_map)]
+
+    generate_build(List.flatten(all_files), build)
+    config.output |> Path.join("index.md") |> Path.relative_to_cwd()
+  end
+
+  defp normalize_config(config) do
+    output = Path.expand(config.output)
+    %{config | output: output}
+  end
+
+  defp output_setup(build, config) do
+    if File.exists?(build) do
+      build
+      |> File.read!()
+      |> String.split("\n", trim: true)
+      |> Enum.map(&Path.join(config.output, &1))
+      |> Enum.each(&File.rm/1)
+
+      File.rm(build)
+    else
+      # Only remove markdown files, not HTML/EPUB files
+      File.mkdir_p!(config.output)
+
+      if File.exists?(config.output) do
+        config.output
+        |> Path.join("*.md")
+        |> Path.wildcard()
+        |> Enum.each(&File.rm/1)
+
+        llms_file = Path.join(config.output, "llms.txt")
+        if File.exists?(llms_file), do: File.rm(llms_file)
+      end
+    end
+  end
+
+  defp generate_build(files, build) do
+    entries =
+      files
+      |> Enum.uniq()
+      |> Enum.sort()
+      |> Enum.map(&[&1, "\n"])
+
+    File.write!(build, entries)
+  end
+
+  defp normalize_output(output) do
+    output
+    |> String.replace(~r/\r\n?/, "\n")
+    |> String.replace(~r/\n{3,}/, "\n\n")
+  end
+
+  defp generate_nav(config, nodes) do
+    nodes =
+      Map.update!(nodes, :modules, fn modules ->
+        modules |> Enum.chunk_by(& &1.group) |> Enum.map(&{hd(&1).group, &1})
+      end)
+
+    content =
+      Templates.nav_template(config, nodes)
+      |> normalize_output()
+
+    filename = "index.md"
+    File.write("#{config.output}/#{filename}", content)
+    filename
+  end
+
+  defp generate_extras(config) do
+    for {_title, extras} <- config.extras,
+        %{id: id, source: content} <- extras,
+        not is_map_key(%{id: id, source: content}, :url) do
+      filename = "#{id}.md"
+      output = "#{config.output}/#{filename}"
+
+      if File.regular?(output) do
+        Utils.warn("file #{Path.relative_to_cwd(output)} already exists", [])
+      end
+
+      File.write!(output, normalize_output(content))
+      filename
+    end
+  end
+
+  defp generate_list(config, nodes) do
+    nodes
+    |> Task.async_stream(&generate_module_page(&1, config), timeout: :infinity)
+    |> Enum.map(&elem(&1, 1))
+  end
+
+  ## Helpers
+
+  defp generate_module_page(module_node, config) do
+    content =
+      Templates.module_page(config, module_node)
+      |> normalize_output()
+
+    filename = "#{module_node.id}.md"
+    File.write("#{config.output}/#{filename}", content)
+    filename
+  end
+
+  defp generate_llm_index(config, nodes_map) do
+    content = generate_llm_index_content(config, nodes_map)
+    filename = "llms.txt"
+    File.write("#{config.output}/#{filename}", content)
+    filename
+  end
+
+  defp generate_llm_index_content(config, nodes_map) do
+    project_info = """
+    # #{config.project} #{config.version}
+
+    #{config.project} documentation index for Large Language Models.
+
+    ## Modules
+
+    """
+
+    modules_info =
+      nodes_map.modules
+      |> Enum.map(fn module_node ->
+        "- **#{module_node.title}** (#{module_node.id}.md): #{module_node.doc |> ExDoc.DocAST.synopsis() |> extract_plain_text()}"
+      end)
+      |> Enum.join("\n")
+
+    tasks_info =
+      if length(nodes_map.tasks) > 0 do
+        tasks_list =
+          nodes_map.tasks
+          |> Enum.map(fn task_node ->
+            "- **#{task_node.title}** (#{task_node.id}.md): #{task_node.doc |> ExDoc.DocAST.synopsis() |> extract_plain_text()}"
+          end)
+          |> Enum.join("\n")
+
+        "\n\n## Mix Tasks\n\n" <> tasks_list
+      else
+        ""
+      end
+
+    extras_info =
+      if is_list(config.extras) and length(config.extras) > 0 do
+        extras_list =
+          config.extras
+          |> Enum.flat_map(fn
+            {_group, extras} when is_list(extras) -> extras
+            _ -> []
+          end)
+          |> Enum.map(fn extra ->
+            "- **#{extra.title}** (#{extra.id}.md): #{extra.title}"
+          end)
+          |> Enum.join("\n")
+
+        if extras_list == "" do
+          ""
+        else
+          "\n\n## Guides\n\n" <> extras_list
+        end
+      else
+        ""
+      end
+
+    project_info <> modules_info <> tasks_info <> extras_info
+  end
+
+  defp extract_plain_text(html) when is_binary(html) do
+    html
+    |> String.replace(~r/<[^>]*>/, "")
+    |> String.replace(~r/\s+/, " ")
+    |> String.trim()
+    |> case do
+      "" ->
+        "No documentation available"
+
+      text ->
+        if String.length(text) > 150 do
+          String.slice(text, 0, 150) <> "..."
+        else
+          text
+        end
+    end
+  end
+
+  defp extract_plain_text(_), do: "No documentation available"
+end