pax_global_header 0000666 0000000 0000000 00000000064 15106421726 0014516 g ustar 00root root 0000000 0000000 52 comment=241ed02bc1a5d567ecf486de7d84bb74db0068d2
python-markdownify-1.2.2/ 0000775 0000000 0000000 00000000000 15106421726 0015371 5 ustar 00root root 0000000 0000000 python-markdownify-1.2.2/.github/ 0000775 0000000 0000000 00000000000 15106421726 0016731 5 ustar 00root root 0000000 0000000 python-markdownify-1.2.2/.github/workflows/ 0000775 0000000 0000000 00000000000 15106421726 0020766 5 ustar 00root root 0000000 0000000 python-markdownify-1.2.2/.github/workflows/python-app.yml 0000664 0000000 0000000 00000002362 15106421726 0023613 0 ustar 00root root 0000000 0000000 # This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: Python application
on:
push:
branches: [ develop ]
pull_request:
branches: [ develop ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.8
uses: actions/setup-python@v2
with:
python-version: 3.8
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install --upgrade setuptools setuptools_scm wheel build tox
- name: Lint and test
run: |
tox
- name: Build
run: |
python -m build -nwsx .
types:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.8
uses: actions/setup-python@v2
with:
python-version: 3.8
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install --upgrade setuptools setuptools_scm wheel build tox mypy types-beautifulsoup4
- name: Check types
run: |
mypy .
mypy --strict tests/types.py
python-markdownify-1.2.2/.github/workflows/python-publish.yml 0000664 0000000 0000000 00000001565 15106421726 0024505 0 ustar 00root root 0000000 0000000 # This workflow will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
name: Upload Python Package
on:
release:
types: [created]
jobs:
deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install --upgrade setuptools setuptools_scm wheel build twine
- name: Build and publish
env:
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
python -m build -nwsx .
twine upload dist/*
python-markdownify-1.2.2/.gitignore 0000664 0000000 0000000 00000000150 15106421726 0017355 0 ustar 00root root 0000000 0000000 *.pyc
*.egg
.eggs/
*.egg-info/
.DS_Store
/.env
/dist
/MANIFEST
/venv
build/
.vscode/settings.json
.tox/
python-markdownify-1.2.2/LICENSE 0000664 0000000 0000000 00000002073 15106421726 0016400 0 ustar 00root root 0000000 0000000 The MIT License (MIT)
Copyright 2012-2018 Matthew Tretter
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
python-markdownify-1.2.2/MANIFEST.in 0000664 0000000 0000000 00000000037 15106421726 0017127 0 ustar 00root root 0000000 0000000 include README.rst
prune tests
python-markdownify-1.2.2/README.rst 0000664 0000000 0000000 00000021356 15106421726 0017067 0 ustar 00root root 0000000 0000000 |build| |version| |license| |downloads|
.. |build| image:: https://img.shields.io/github/actions/workflow/status/matthewwithanm/python-markdownify/python-app.yml?branch=develop
:alt: GitHub Workflow Status
:target: https://github.com/matthewwithanm/python-markdownify/actions/workflows/python-app.yml?query=workflow%3A%22Python+application%22
.. |version| image:: https://img.shields.io/pypi/v/markdownify
:alt: Pypi version
:target: https://pypi.org/project/markdownify/
.. |license| image:: https://img.shields.io/pypi/l/markdownify
:alt: License
:target: https://github.com/matthewwithanm/python-markdownify/blob/develop/LICENSE
.. |downloads| image:: https://pepy.tech/badge/markdownify
:alt: Pypi Downloads
:target: https://pepy.tech/project/markdownify
Installation
============
``pip install markdownify``
Usage
=====
Convert some HTML to Markdown:
.. code:: python
from markdownify import markdownify as md
md('YayGitHub') # > '**Yay** [GitHub](http://github.com)'
Specify tags to exclude:
.. code:: python
from markdownify import markdownify as md
md('YayGitHub', strip=['a']) # > '**Yay** GitHub'
\...or specify the tags you want to include:
.. code:: python
from markdownify import markdownify as md
md('YayGitHub', convert=['b']) # > '**Yay** GitHub'
Options
=======
Markdownify supports the following options:
strip
A list of tags to strip. This option can't be used with the
``convert`` option.
convert
A list of tags to convert. This option can't be used with the
``strip`` option.
autolinks
A boolean indicating whether the "automatic link" style should be used when
a ``a`` tag's contents match its href. Defaults to ``True``.
default_title
A boolean to enable setting the title of a link to its href, if no title is
given. Defaults to ``False``.
heading_style
Defines how headings should be converted. Accepted values are ``ATX``,
``ATX_CLOSED``, ``SETEXT``, and ``UNDERLINED`` (which is an alias for
``SETEXT``). Defaults to ``UNDERLINED``.
bullets
An iterable (string, list, or tuple) of bullet styles to be used. If the
iterable only contains one item, it will be used regardless of how deeply
lists are nested. Otherwise, the bullet will alternate based on nesting
level. Defaults to ``'*+-'``.
strong_em_symbol
In markdown, both ``*`` and ``_`` are used to encode **strong** or
*emphasized* texts. Either of these symbols can be chosen by the options
``ASTERISK`` (default) or ``UNDERSCORE`` respectively.
sub_symbol, sup_symbol
Define the chars that surround ```` and ```` text. Defaults to an
empty string, because this is non-standard behavior. Could be something like
``~`` and ``^`` to result in ``~sub~`` and ``^sup^``. If the value starts
with ``<`` and ends with ``>``, it is treated as an HTML tag and a ``/`` is
inserted after the ``<`` in the string used after the text; this allows
specifying ```` to use raw HTML in the output for subscripts, for
example.
newline_style
Defines the style of marking linebreaks (`` ``) in markdown. The default
value ``SPACES`` of this option will adopt the usual two spaces and a newline,
while ``BACKSLASH`` will convert a linebreak to ``\\n`` (a backslash and a
newline). While the latter convention is non-standard, it is commonly
preferred and supported by a lot of interpreters.
code_language
Defines the language that should be assumed for all ``
`` sections.
Useful, if all code on a page is in the same programming language and
should be annotated with `````python`` or similar.
Defaults to ``''`` (empty string) and can be any string.
code_language_callback
When the HTML code contains ``pre`` tags that in some way provide the code
language, for example as class, this callback can be used to extract the
language from the tag and prefix it to the converted ``pre`` tag.
The callback gets one single argument, a BeautifulSoup object, and returns
a string containing the code language, or ``None``.
An example to use the class name as code language could be::
def callback(el):
return el['class'][0] if el.has_attr('class') else None
Defaults to ``None``.
escape_asterisks
If set to ``False``, do not escape ``*`` to ``\*`` in text.
Defaults to ``True``.
escape_underscores
If set to ``False``, do not escape ``_`` to ``\_`` in text.
Defaults to ``True``.
escape_misc
If set to ``True``, escape miscellaneous punctuation characters
that sometimes have Markdown significance in text.
Defaults to ``False``.
keep_inline_images_in
Images are converted to their alt-text when the images are located inside
headlines or table cells. If some inline images should be converted to
markdown images instead, this option can be set to a list of parent tags
that should be allowed to contain inline images, for example ``['td']``.
Defaults to an empty list.
table_infer_header
Controls handling of tables with no header row (as indicated by ````
or ``
``). When set to ``True``, the first body row is used as the header row.
Defaults to ``False``, which leaves the header row empty.
wrap, wrap_width
If ``wrap`` is set to ``True``, all text paragraphs are wrapped at
``wrap_width`` characters. Defaults to ``False`` and ``80``.
Use with ``newline_style=BACKSLASH`` to keep line breaks in paragraphs.
A `wrap_width` value of `None` reflows lines to unlimited line length.
strip_document
Controls whether leading and/or trailing separation newlines are removed from
the final converted document. Supported values are ``LSTRIP`` (leading),
``RSTRIP`` (trailing), ``STRIP`` (both), and ``None`` (neither). Newlines
within the document are unaffected.
Defaults to ``STRIP``.
strip_pre
Controls whether leading/trailing blank lines are removed from ``
``
tags. Supported values are ``STRIP`` (all leading/trailing blank lines),
``STRIP_ONE`` (one leading/trailing blank line), and ``None`` (neither).
Defaults to ``STRIP``.
bs4_options
Specify additional configuration options for the ``BeautifulSoup`` object
used to interpret the HTML markup. String and list values (such as ``lxml``
or ``html5lib``) are treated as ``features`` arguments to control parser
selection. Dictionary values (such as ``{"from_encoding": "iso-8859-8"}``)
are treated as full kwargs to be used for the BeautifulSoup constructor,
allowing specification of any parameter. For parameter details, see the
Beautiful Soup documentation at:
.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/bs4/doc/
Options may be specified as kwargs to the ``markdownify`` function, or as a
nested ``Options`` class in ``MarkdownConverter`` subclasses.
Converting BeautifulSoup objects
================================
.. code:: python
from markdownify import MarkdownConverter
# Create shorthand method for conversion
def md(soup, **options):
return MarkdownConverter(**options).convert_soup(soup)
Creating Custom Converters
==========================
If you have a special usecase that calls for a special conversion, you can
always inherit from ``MarkdownConverter`` and override the method you want to
change.
The function that handles a HTML tag named ``abc`` is called
``convert_abc(self, el, text, parent_tags)`` and returns a string
containing the converted HTML tag.
The ``MarkdownConverter`` object will handle the conversion based on the
function names:
.. code:: python
from markdownify import MarkdownConverter
class ImageBlockConverter(MarkdownConverter):
"""
Create a custom MarkdownConverter that adds two newlines after an image
"""
def convert_img(self, el, text, parent_tags):
return super().convert_img(el, text, parent_tags) + '\n\n'
# Create shorthand method for conversion
def md(html, **options):
return ImageBlockConverter(**options).convert(html)
.. code:: python
from markdownify import MarkdownConverter
class IgnoreParagraphsConverter(MarkdownConverter):
"""
Create a custom MarkdownConverter that ignores paragraphs
"""
def convert_p(self, el, text, parent_tags):
return ''
# Create shorthand method for conversion
def md(html, **options):
return IgnoreParagraphsConverter(**options).convert(html)
Command Line Interface
======================
Use ``markdownify example.html > example.md`` or pipe input from stdin
(``cat example.html | markdownify > example.md``).
Call ``markdownify -h`` to see all available options.
They are the same as listed above and take the same arguments.
Development
===========
To run tests and the linter run ``pip install tox`` once, then ``tox``.
python-markdownify-1.2.2/markdownify/ 0000775 0000000 0000000 00000000000 15106421726 0017723 5 ustar 00root root 0000000 0000000 python-markdownify-1.2.2/markdownify/__init__.py 0000664 0000000 0000000 00000073235 15106421726 0022046 0 ustar 00root root 0000000 0000000 from bs4 import BeautifulSoup, Comment, Doctype, NavigableString, Tag
from textwrap import fill
import re
import six
# General-purpose regex patterns
re_convert_heading = re.compile(r'convert_h(\d+)')
re_line_with_content = re.compile(r'^(.*)', flags=re.MULTILINE)
re_whitespace = re.compile(r'[\t ]+')
re_all_whitespace = re.compile(r'[\t \r\n]+')
re_newline_whitespace = re.compile(r'[\t \r\n]*[\r\n][\t \r\n]*')
re_html_heading = re.compile(r'h(\d+)')
re_pre_lstrip1 = re.compile(r'^ *\n')
re_pre_rstrip1 = re.compile(r'\n *$')
re_pre_lstrip = re.compile(r'^[ \n]*\n')
re_pre_rstrip = re.compile(r'[ \n]*$')
# Pattern for creating convert_ function names from tag names
re_make_convert_fn_name = re.compile(r'[\[\]:-]')
# Extract (leading_nl, content, trailing_nl) from a string
# (functionally equivalent to r'^(\n*)(.*?)(\n*)$', but greedy is faster than reluctant here)
re_extract_newlines = re.compile(r'^(\n*)((?:.*[^\n])?)(\n*)$', flags=re.DOTALL)
# Escape miscellaneous special Markdown characters
re_escape_misc_chars = re.compile(r'([]\\&<`[>~=+|])')
# Escape sequence of one or more consecutive '-', preceded
# and followed by whitespace or start/end of fragment, as it
# might be confused with an underline of a header, or with a
# list marker
re_escape_misc_dash_sequences = re.compile(r'(\s|^)(-+(?:\s|$))')
# Escape sequence of up to six consecutive '#', preceded
# and followed by whitespace or start/end of fragment, as
# it might be confused with an ATX heading
re_escape_misc_hashes = re.compile(r'(\s|^)(#{1,6}(?:\s|$))')
# Escape '.' or ')' preceded by up to nine digits, as it might be
# confused with a list item
re_escape_misc_list_items = re.compile(r'((?:\s|^)[0-9]{1,9})([.)](?:\s|$))')
# Find consecutive backtick sequences in a string
re_backtick_runs = re.compile(r'`+')
# Heading styles
ATX = 'atx'
ATX_CLOSED = 'atx_closed'
UNDERLINED = 'underlined'
SETEXT = UNDERLINED
# Newline style
SPACES = 'spaces'
BACKSLASH = 'backslash'
# Strong and emphasis style
ASTERISK = '*'
UNDERSCORE = '_'
# Document/pre strip styles
LSTRIP = 'lstrip'
RSTRIP = 'rstrip'
STRIP = 'strip'
STRIP_ONE = 'strip_one'
def strip1_pre(text):
"""Strip one leading and trailing newline from a
string."""
text = re_pre_lstrip1.sub('', text)
text = re_pre_rstrip1.sub('', text)
return text
def strip_pre(text):
"""Strip all leading and trailing newlines from a
string."""
text = re_pre_lstrip.sub('', text)
text = re_pre_rstrip.sub('', text)
return text
def chomp(text):
"""
If the text in an inline tag like b, a, or em contains a leading or trailing
space, strip the string and return a space as suffix of prefix, if needed.
This function is used to prevent conversions like
foo => ** foo**
"""
prefix = ' ' if text and text[0] == ' ' else ''
suffix = ' ' if text and text[-1] == ' ' else ''
text = text.strip()
return (prefix, suffix, text)
def abstract_inline_conversion(markup_fn):
"""
This abstracts all simple inline tags like b, em, del, ...
Returns a function that wraps the chomped text in a pair of the string
that is returned by markup_fn, with '/' inserted in the string used after
the text if it looks like an HTML tag. markup_fn is necessary to allow for
references to self.strong_em_symbol etc.
"""
def implementation(self, el, text, parent_tags):
markup_prefix = markup_fn(self)
if markup_prefix.startswith('<') and markup_prefix.endswith('>'):
markup_suffix = '), ignore adjacent whitespace elements.
return True
elif should_remove_whitespace_outside(el.previous_sibling) or should_remove_whitespace_outside(el.next_sibling):
# Outside block elements (including
), ignore adjacent whitespace elements.
return True
else:
return False
elif el is None:
return True
else:
raise ValueError('Unexpected element type: %s' % type(el))
children_to_convert = [el for el in node.children if not _can_ignore(el)]
# Create a copy of this tag's parent context, then update it to include this tag
# to propagate down into the children.
parent_tags_for_children = set(parent_tags)
parent_tags_for_children.add(node.name)
# if this tag is a heading or table cell, add an '_inline' parent pseudo-tag
if (
re_html_heading.match(node.name) is not None # headings
or node.name in {'td', 'th'} # table cells
):
parent_tags_for_children.add('_inline')
# if this tag is a preformatted element, add a '_noformat' parent pseudo-tag
if node.name in {'pre', 'code', 'kbd', 'samp'}:
parent_tags_for_children.add('_noformat')
# Convert the children elements into a list of result strings.
child_strings = [
self.process_element(el, parent_tags=parent_tags_for_children)
for el in children_to_convert
]
# Remove empty string values.
child_strings = [s for s in child_strings if s]
# Collapse newlines at child element boundaries, if needed.
if node.name == 'pre' or node.find_parent('pre'):
# Inside
blocks, do not collapse newlines.
pass
else:
# Collapse newlines at child element boundaries.
updated_child_strings = [''] # so the first lookback works
for child_string in child_strings:
# Separate the leading/trailing newlines from the content.
leading_nl, content, trailing_nl = re_extract_newlines.match(child_string).groups()
# If the last child had trailing newlines and this child has leading newlines,
# use the larger newline count, limited to 2.
if updated_child_strings[-1] and leading_nl:
prev_trailing_nl = updated_child_strings.pop() # will be replaced by the collapsed value
num_newlines = min(2, max(len(prev_trailing_nl), len(leading_nl)))
leading_nl = '\n' * num_newlines
# Add the results to the updated child string list.
updated_child_strings.extend([leading_nl, content, trailing_nl])
child_strings = updated_child_strings
# Join all child text strings into a single string.
text = ''.join(child_strings)
# apply this tag's final conversion function
convert_fn = self.get_conv_fn_cached(node.name)
if convert_fn is not None:
text = convert_fn(node, text, parent_tags=parent_tags)
return text
def convert__document_(self, el, text, parent_tags):
"""Final document-level formatting for BeautifulSoup object (node.name == "[document]")"""
if self.options['strip_document'] == LSTRIP:
text = text.lstrip('\n') # remove leading separation newlines
elif self.options['strip_document'] == RSTRIP:
text = text.rstrip('\n') # remove trailing separation newlines
elif self.options['strip_document'] == STRIP:
text = text.strip('\n') # remove leading and trailing separation newlines
elif self.options['strip_document'] is None:
pass # leave leading and trailing separation newlines as-is
else:
raise ValueError('Invalid value for strip_document: %s' % self.options['strip_document'])
return text
def process_text(self, el, parent_tags=None):
# For the top-level element, initialize the parent context with an empty set.
if parent_tags is None:
parent_tags = set()
text = six.text_type(el) or ''
# normalize whitespace if we're not inside a preformatted element
if 'pre' not in parent_tags:
if self.options['wrap']:
text = re_all_whitespace.sub(' ', text)
else:
text = re_newline_whitespace.sub('\n', text)
text = re_whitespace.sub(' ', text)
# escape special characters if we're not inside a preformatted or code element
if '_noformat' not in parent_tags:
text = self.escape(text, parent_tags)
# remove leading whitespace at the start or just after a
# block-level element; remove traliing whitespace at the end
# or just before a block-level element.
if (should_remove_whitespace_outside(el.previous_sibling)
or (should_remove_whitespace_inside(el.parent)
and not el.previous_sibling)):
text = text.lstrip(' \t\r\n')
if (should_remove_whitespace_outside(el.next_sibling)
or (should_remove_whitespace_inside(el.parent)
and not el.next_sibling)):
text = text.rstrip()
return text
def get_conv_fn_cached(self, tag_name):
"""Given a tag name, return the conversion function using the cache."""
# If conversion function is not in cache, add it
if tag_name not in self.convert_fn_cache:
self.convert_fn_cache[tag_name] = self.get_conv_fn(tag_name)
# Return the cached entry
return self.convert_fn_cache[tag_name]
def get_conv_fn(self, tag_name):
"""Given a tag name, find and return the conversion function."""
tag_name = tag_name.lower()
# Handle strip/convert exclusion options
if not self.should_convert_tag(tag_name):
return None
# Look for an explicitly defined conversion function by tag name first
convert_fn_name = "convert_%s" % re_make_convert_fn_name.sub("_", tag_name)
convert_fn = getattr(self, convert_fn_name, None)
if convert_fn:
return convert_fn
# If tag is any heading, handle with convert_hN() function
match = re_html_heading.match(tag_name)
if match:
n = int(match.group(1)) # get value of N from
return lambda el, text, parent_tags: self.convert_hN(n, el, text, parent_tags)
# No conversion function was found
return None
def should_convert_tag(self, tag):
"""Given a tag name, return whether to convert based on strip/convert options."""
strip = self.options['strip']
convert = self.options['convert']
if strip is not None:
return tag not in strip
elif convert is not None:
return tag in convert
else:
return True
def escape(self, text, parent_tags):
if not text:
return ''
if self.options['escape_misc']:
text = re_escape_misc_chars.sub(r'\\\1', text)
text = re_escape_misc_dash_sequences.sub(r'\1\\\2', text)
text = re_escape_misc_hashes.sub(r'\1\\\2', text)
text = re_escape_misc_list_items.sub(r'\1\\\2', text)
if self.options['escape_asterisks']:
text = text.replace('*', r'\*')
if self.options['escape_underscores']:
text = text.replace('_', r'\_')
return text
def underline(self, text, pad_char):
text = (text or '').rstrip()
return '\n\n%s\n%s\n\n' % (text, pad_char * len(text)) if text else ''
def convert_a(self, el, text, parent_tags):
if '_noformat' in parent_tags:
return text
prefix, suffix, text = chomp(text)
if not text:
return ''
href = el.get('href')
title = el.get('title')
# For the replacement see #29: text nodes underscores are escaped
if (self.options['autolinks']
and text.replace(r'\_', '_') == href
and not title
and not self.options['default_title']):
# Shortcut syntax
return '<%s>' % href
if self.options['default_title'] and not title:
title = href
title_part = ' "%s"' % title.replace('"', r'\"') if title else ''
return '%s[%s](%s%s)%s' % (prefix, text, href, title_part, suffix) if href else text
convert_b = abstract_inline_conversion(lambda self: 2 * self.options['strong_em_symbol'])
def convert_blockquote(self, el, text, parent_tags):
# handle some early-exit scenarios
text = (text or '').strip(' \t\r\n')
if '_inline' in parent_tags:
return ' ' + text + ' '
if not text:
return "\n"
# indent lines with blockquote marker
def _indent_for_blockquote(match):
line_content = match.group(1)
return '> ' + line_content if line_content else '>'
text = re_line_with_content.sub(_indent_for_blockquote, text)
return '\n' + text + '\n\n'
def convert_br(self, el, text, parent_tags):
if '_inline' in parent_tags:
return ' '
if self.options['newline_style'].lower() == BACKSLASH:
return '\\\n'
else:
return ' \n'
def convert_code(self, el, text, parent_tags):
if '_noformat' in parent_tags:
return text
prefix, suffix, text = chomp(text)
if not text:
return ''
# Find the maximum number of consecutive backticks in the text, then
# delimit the code span with one more backtick than that
max_backticks = max((len(match) for match in re.findall(re_backtick_runs, text)), default=0)
markup_delimiter = '`' * (max_backticks + 1)
# If the maximum number of backticks is greater than zero, add a space
# to avoid interpretation of inside backticks as literals
if max_backticks > 0:
text = " " + text + " "
return '%s%s%s%s%s' % (prefix, markup_delimiter, text, markup_delimiter, suffix)
convert_del = abstract_inline_conversion(lambda self: '~~')
def convert_div(self, el, text, parent_tags):
if '_inline' in parent_tags:
return ' ' + text.strip() + ' '
text = text.strip()
return '\n\n%s\n\n' % text if text else ''
convert_article = convert_div
convert_section = convert_div
convert_em = abstract_inline_conversion(lambda self: self.options['strong_em_symbol'])
convert_kbd = convert_code
def convert_dd(self, el, text, parent_tags):
text = (text or '').strip()
if '_inline' in parent_tags:
return ' ' + text + ' '
if not text:
return '\n'
# indent definition content lines by four spaces
def _indent_for_dd(match):
line_content = match.group(1)
return ' ' + line_content if line_content else ''
text = re_line_with_content.sub(_indent_for_dd, text)
# insert definition marker into first-line indent whitespace
text = ':' + text[1:]
return '%s\n' % text
# definition lists are formatted as follows:
# https://pandoc.org/MANUAL.html#definition-lists
# https://michelf.ca/projects/php-markdown/extra/#def-list
convert_dl = convert_div
def convert_dt(self, el, text, parent_tags):
# remove newlines from term text
text = (text or '').strip()
text = re_all_whitespace.sub(' ', text)
if '_inline' in parent_tags:
return ' ' + text + ' '
if not text:
return '\n'
# TODO - format consecutive
elements as directly adjacent lines):
# https://michelf.ca/projects/php-markdown/extra/#def-list
return '\n\n%s\n' % text
def convert_hN(self, n, el, text, parent_tags):
# convert_hN() converts tags, where N is any integer
if '_inline' in parent_tags:
return text
# Markdown does not support heading depths of n > 6
n = max(1, min(6, n))
style = self.options['heading_style'].lower()
text = text.strip()
if style == UNDERLINED and n <= 2:
line = '=' if n == 1 else '-'
return self.underline(text, line)
text = re_all_whitespace.sub(' ', text)
hashes = '#' * n
if style == ATX_CLOSED:
return '\n\n%s %s %s\n\n' % (hashes, text, hashes)
return '\n\n%s %s\n\n' % (hashes, text)
def convert_hr(self, el, text, parent_tags):
return '\n\n---\n\n'
convert_i = convert_em
def convert_img(self, el, text, parent_tags):
alt = el.attrs.get('alt', None) or ''
src = el.attrs.get('src', None) or ''
title = el.attrs.get('title', None) or ''
title_part = ' "%s"' % title.replace('"', r'\"') if title else ''
if ('_inline' in parent_tags
and el.parent.name not in self.options['keep_inline_images_in']):
return alt
return '' % (alt, src, title_part)
def convert_video(self, el, text, parent_tags):
if ('_inline' in parent_tags
and el.parent.name not in self.options['keep_inline_images_in']):
return text
src = el.attrs.get('src', None) or ''
if not src:
sources = el.find_all('source', attrs={'src': True})
if sources:
src = sources[0].attrs.get('src', None) or ''
poster = el.attrs.get('poster', None) or ''
if src and poster:
return '[](%s)' % (text, poster, src)
if src:
return '[%s](%s)' % (text, src)
if poster:
return '' % (text, poster)
return text
def convert_list(self, el, text, parent_tags):
# Converting a list to inline is undefined.
# Ignoring inline conversion parents for list.
before_paragraph = False
next_sibling = _next_block_content_sibling(el)
if next_sibling and next_sibling.name not in ['ul', 'ol']:
before_paragraph = True
if 'li' in parent_tags:
# remove trailing newline if we're in a nested list
return '\n' + text.rstrip()
return '\n\n' + text + ('\n' if before_paragraph else '')
convert_ul = convert_list
convert_ol = convert_list
def convert_li(self, el, text, parent_tags):
# handle some early-exit scenarios
text = (text or '').strip()
if not text:
return "\n"
# determine list item bullet character to use
parent = el.parent
if parent is not None and parent.name == 'ol':
if parent.get("start") and str(parent.get("start")).isnumeric():
start = int(parent.get("start"))
else:
start = 1
bullet = '%s.' % (start + len(el.find_previous_siblings('li')))
else:
depth = -1
while el:
if el.name == 'ul':
depth += 1
el = el.parent
bullets = self.options['bullets']
bullet = bullets[depth % len(bullets)]
bullet = bullet + ' '
bullet_width = len(bullet)
bullet_indent = ' ' * bullet_width
# indent content lines by bullet width
def _indent_for_li(match):
line_content = match.group(1)
return bullet_indent + line_content if line_content else ''
text = re_line_with_content.sub(_indent_for_li, text)
# insert bullet into first-line indent whitespace
text = bullet + text[bullet_width:]
return '%s\n' % text
def convert_p(self, el, text, parent_tags):
if '_inline' in parent_tags:
return ' ' + text.strip(' \t\r\n') + ' '
text = text.strip(' \t\r\n')
if self.options['wrap']:
# Preserve newlines (and preceding whitespace) resulting
# from tags. Newlines in the input have already been
# replaced by spaces.
if self.options['wrap_width'] is not None:
lines = text.split('\n')
new_lines = []
for line in lines:
line = line.lstrip(' \t\r\n')
line_no_trailing = line.rstrip()
trailing = line[len(line_no_trailing):]
line = fill(line,
width=self.options['wrap_width'],
break_long_words=False,
break_on_hyphens=False)
new_lines.append(line + trailing)
text = '\n'.join(new_lines)
return '\n\n%s\n\n' % text if text else ''
def convert_pre(self, el, text, parent_tags):
if not text:
return ''
code_language = self.options['code_language']
if self.options['code_language_callback']:
code_language = self.options['code_language_callback'](el) or code_language
if self.options['strip_pre'] == STRIP:
text = strip_pre(text) # remove all leading/trailing newlines
elif self.options['strip_pre'] == STRIP_ONE:
text = strip1_pre(text) # remove one leading/trailing newline
elif self.options['strip_pre'] is None:
pass # leave leading and trailing newlines as-is
else:
raise ValueError('Invalid value for strip_pre: %s' % self.options['strip_pre'])
return '\n\n```%s\n%s\n```\n\n' % (code_language, text)
def convert_q(self, el, text, parent_tags):
return '"' + text + '"'
def convert_script(self, el, text, parent_tags):
return ''
def convert_style(self, el, text, parent_tags):
return ''
convert_s = convert_del
convert_strong = convert_b
convert_samp = convert_code
convert_sub = abstract_inline_conversion(lambda self: self.options['sub_symbol'])
convert_sup = abstract_inline_conversion(lambda self: self.options['sup_symbol'])
def convert_table(self, el, text, parent_tags):
return '\n\n' + text.strip() + '\n\n'
def convert_caption(self, el, text, parent_tags):
return text.strip() + '\n\n'
def convert_figcaption(self, el, text, parent_tags):
return '\n\n' + text.strip() + '\n\n'
def convert_td(self, el, text, parent_tags):
colspan = 1
if 'colspan' in el.attrs and el['colspan'].isdigit():
colspan = max(1, min(1000, int(el['colspan'])))
return ' ' + text.strip().replace("\n", " ") + ' |' * colspan
def convert_th(self, el, text, parent_tags):
colspan = 1
if 'colspan' in el.attrs and el['colspan'].isdigit():
colspan = max(1, min(1000, int(el['colspan'])))
return ' ' + text.strip().replace("\n", " ") + ' |' * colspan
def convert_tr(self, el, text, parent_tags):
cells = el.find_all(['td', 'th'])
is_first_row = el.find_previous_sibling() is None
is_headrow = (
all([cell.name == 'th' for cell in cells])
or (el.parent.name == 'thead'
# avoid multiple tr in thead
and len(el.parent.find_all('tr')) == 1)
)
is_head_row_missing = (
(is_first_row and not el.parent.name == 'tbody')
or (is_first_row and el.parent.name == 'tbody' and len(el.parent.parent.find_all(['thead'])) < 1)
)
overline = ''
underline = ''
full_colspan = 0
for cell in cells:
if 'colspan' in cell.attrs and cell['colspan'].isdigit():
full_colspan += max(1, min(1000, int(cell['colspan'])))
else:
full_colspan += 1
if ((is_headrow
or (is_head_row_missing
and self.options['table_infer_header']))
and is_first_row):
# first row and:
# - is headline or
# - headline is missing and header inference is enabled
# print headline underline
underline += '| ' + ' | '.join(['---'] * full_colspan) + ' |' + '\n'
elif ((is_head_row_missing
and not self.options['table_infer_header'])
or (is_first_row
and (el.parent.name == 'table'
or (el.parent.name == 'tbody'
and not el.parent.find_previous_sibling())))):
# headline is missing and header inference is disabled or:
# first row, not headline, and:
# - the parent is table or
# - the parent is tbody at the beginning of a table.
# print empty headline above this row
overline += '| ' + ' | '.join([''] * full_colspan) + ' |' + '\n'
overline += '| ' + ' | '.join(['---'] * full_colspan) + ' |' + '\n'
return overline + '|' + text + '\n' + underline
def markdownify(html, **options):
return MarkdownConverter(**options).convert(html)
python-markdownify-1.2.2/markdownify/__init__.pyi 0000664 0000000 0000000 00000004061 15106421726 0022206 0 ustar 00root root 0000000 0000000 from _typeshed import Incomplete
from typing import Callable, Union
ATX: str
ATX_CLOSED: str
UNDERLINED: str
SETEXT = UNDERLINED
SPACES: str
BACKSLASH: str
ASTERISK: str
UNDERSCORE: str
LSTRIP: str
RSTRIP: str
STRIP: str
STRIP_ONE: str
def markdownify(
html: str,
autolinks: bool = ...,
bs4_options: str = ...,
bullets: str = ...,
code_language: str = ...,
code_language_callback: Union[Callable[[Incomplete], Union[str, None]], None] = ...,
convert: Union[list[str], None] = ...,
default_title: bool = ...,
escape_asterisks: bool = ...,
escape_underscores: bool = ...,
escape_misc: bool = ...,
heading_style: str = ...,
keep_inline_images_in: list[str] = ...,
newline_style: str = ...,
strip: Union[list[str], None] = ...,
strip_document: Union[str, None] = ...,
strip_pre: str = ...,
strong_em_symbol: str = ...,
sub_symbol: str = ...,
sup_symbol: str = ...,
table_infer_header: bool = ...,
wrap: bool = ...,
wrap_width: int = ...,
) -> str: ...
class MarkdownConverter:
def __init__(
self,
autolinks: bool = ...,
bs4_options: str = ...,
bullets: str = ...,
code_language: str = ...,
code_language_callback: Union[Callable[[Incomplete], Union[str, None]], None] = ...,
convert: Union[list[str], None] = ...,
default_title: bool = ...,
escape_asterisks: bool = ...,
escape_underscores: bool = ...,
escape_misc: bool = ...,
heading_style: str = ...,
keep_inline_images_in: list[str] = ...,
newline_style: str = ...,
strip: Union[list[str], None] = ...,
strip_document: Union[str, None] = ...,
strip_pre: str = ...,
strong_em_symbol: str = ...,
sub_symbol: str = ...,
sup_symbol: str = ...,
table_infer_header: bool = ...,
wrap: bool = ...,
wrap_width: int = ...,
) -> None:
...
def convert(self, html: str) -> str:
...
def convert_soup(self, soup: Incomplete) -> str:
...
python-markdownify-1.2.2/markdownify/main.py 0000775 0000000 0000000 00000010662 15106421726 0021231 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
import argparse
import sys
from markdownify import markdownify, ATX, ATX_CLOSED, UNDERLINED, \
SPACES, BACKSLASH, ASTERISK, UNDERSCORE
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
prog='markdownify',
description='Converts html to markdown.',
)
parser.add_argument('html', nargs='?', type=argparse.FileType('r'),
default=sys.stdin,
help="The html file to convert. Defaults to STDIN if not "
"provided.")
parser.add_argument('-s', '--strip', nargs='*',
help="A list of tags to strip. This option can't be used with "
"the --convert option.")
parser.add_argument('-c', '--convert', nargs='*',
help="A list of tags to convert. This option can't be used with "
"the --strip option.")
parser.add_argument('-a', '--autolinks', action='store_true',
help="A boolean indicating whether the 'automatic link' style "
"should be used when a 'a' tag's contents match its href.")
parser.add_argument('--default-title', action='store_false',
help="A boolean to enable setting the title of a link to its "
"href, if no title is given.")
parser.add_argument('--heading-style', default=UNDERLINED,
choices=(ATX, ATX_CLOSED, UNDERLINED),
help="Defines how headings should be converted.")
parser.add_argument('-b', '--bullets', default='*+-',
help="A string of bullet styles to use; the bullet will "
"alternate based on nesting level.")
parser.add_argument('--strong-em-symbol', default=ASTERISK,
choices=(ASTERISK, UNDERSCORE),
help="Use * or _ to convert strong and italics text"),
parser.add_argument('--sub-symbol', default='',
help="Define the chars that surround ''.")
parser.add_argument('--sup-symbol', default='',
help="Define the chars that surround ''.")
parser.add_argument('--newline-style', default=SPACES,
choices=(SPACES, BACKSLASH),
help="Defines the style of conversions: two spaces "
"or backslash at the and of the line thet should break.")
parser.add_argument('--code-language', default='',
help="Defines the language that should be assumed for all "
"'
' sections.")
parser.add_argument('--no-escape-asterisks', dest='escape_asterisks',
action='store_false',
help="Do not escape '*' to '\\*' in text.")
parser.add_argument('--no-escape-underscores', dest='escape_underscores',
action='store_false',
help="Do not escape '_' to '\\_' in text.")
parser.add_argument('-i', '--keep-inline-images-in',
default=[],
nargs='*',
help="Images are converted to their alt-text when the images are "
"located inside headlines or table cells. If some inline images "
"should be converted to markdown images instead, this option can "
"be set to a list of parent tags that should be allowed to "
"contain inline images.")
parser.add_argument('--table-infer-header', dest='table_infer_header',
action='store_true',
help="When a table has no header row (as indicated by '' "
"or '
'), use the first body row as the header row.")
parser.add_argument('-w', '--wrap', action='store_true',
help="Wrap all text paragraphs at --wrap-width characters.")
parser.add_argument('--wrap-width', type=int, default=80)
parser.add_argument('--bs4-options',
default='html.parser',
help="Specifies the parser that BeautifulSoup should use to parse "
"the HTML markup. Examples include 'html5.parser', 'lxml', and "
"'html5lib'.")
args = parser.parse_args(argv)
print(markdownify(**vars(args)))
if __name__ == '__main__':
main()
python-markdownify-1.2.2/markdownify/py.typed 0000664 0000000 0000000 00000000001 15106421726 0021411 0 ustar 00root root 0000000 0000000
python-markdownify-1.2.2/pyproject.toml 0000664 0000000 0000000 00000002373 15106421726 0020312 0 ustar 00root root 0000000 0000000 [build-system]
requires = ["setuptools>=61.2", "setuptools_scm[toml]>=3.4.3"]
build-backend = "setuptools.build_meta"
[project]
name = "markdownify"
version = "1.2.2"
authors = [{name = "Matthew Tretter", email = "m@tthewwithanm.com"}]
description = "Convert HTML to markdown."
readme = "README.rst"
classifiers = [
"Environment :: Web Environment",
"Framework :: Django",
"Intended Audience :: Developers",
"License :: OSI Approved :: MIT License",
"Operating System :: OS Independent",
"Programming Language :: Python :: 2.5",
"Programming Language :: Python :: 2.6",
"Programming Language :: Python :: 2.7",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Topic :: Utilities",
]
dependencies = [
"beautifulsoup4>=4.9,<5",
"six>=1.15,<2"
]
[project.urls]
Homepage = "http://github.com/matthewwithanm/python-markdownify"
Download = "http://github.com/matthewwithanm/python-markdownify/tarball/master"
[project.scripts]
markdownify = "markdownify.main:main"
[tool.setuptools]
zip-safe = false
include-package-data = true
[tool.setuptools.packages.find]
include = ["markdownify", "markdownify.*"]
namespaces = false
[tool.setuptools_scm]
python-markdownify-1.2.2/shell.nix 0000664 0000000 0000000 00000000322 15106421726 0017215 0 ustar 00root root 0000000 0000000 { pkgs ? import {} }:
pkgs.mkShell {
name = "python-shell";
buildInputs = with pkgs; [
python38
python38Packages.tox
python38Packages.setuptools
python38Packages.virtualenv
];
}
python-markdownify-1.2.2/tests/ 0000775 0000000 0000000 00000000000 15106421726 0016533 5 ustar 00root root 0000000 0000000 python-markdownify-1.2.2/tests/__init__.py 0000664 0000000 0000000 00000000000 15106421726 0020632 0 ustar 00root root 0000000 0000000 python-markdownify-1.2.2/tests/test_advanced.py 0000664 0000000 0000000 00000002341 15106421726 0021711 0 ustar 00root root 0000000 0000000 from .utils import md
def test_chomp():
assert md(' ') == ' '
assert md(' ') == ' '
assert md(' ') == ' '
assert md(' ') == ' '
assert md(' s ') == ' **s** '
assert md(' s ') == ' **s** '
assert md(' s ') == ' **s** '
assert md(' s ') == ' **s** '
def test_nested():
text = md('
') == '\n> Hello\n\nhandsome\n\n'
def test_blockquote_nested():
text = md('
And she was like
Hello
')
assert text == '\n> And she was like\n> > Hello\n\n'
def test_br():
assert md('a b c') == 'a \nb \nc'
assert md('a b c', newline_style=BACKSLASH) == 'a\\\nb\\\nc'
assert md('
B