path: root/flit/validate.py

"""Validate various pieces of packaging data"""

import errno
import io
import logging
import os
from pathlib import Path
import re
import requests
import sys

from .vendorized.readme.rst import render

log = logging.getLogger(__name__)

CUSTOM_CLASSIFIERS = frozenset({
    # https://github.com/pypa/warehouse/pull/5440
    'Private :: Do Not Upload',
})


def get_cache_dir() -> Path:
    """Locate a platform-appropriate cache directory for flit to use

    Does not ensure that the cache directory exists.
    """
    # Linux, Unix, AIX, etc.
    if os.name == 'posix' and sys.platform != 'darwin':
        # use ~/.cache if empty OR not set
        xdg = os.environ.get("XDG_CACHE_HOME", None) \
              or os.path.expanduser('~/.cache')
        return Path(xdg, 'flit')

    # Mac OS
    elif sys.platform == 'darwin':
        return Path(os.path.expanduser('~'), 'Library/Caches/flit')

    # Windows (hopefully)
    else:
        local = os.environ.get('LOCALAPPDATA', None) \
                or os.path.expanduser('~\\AppData\\Local')
        return Path(local, 'flit')


def _read_classifiers_cached():
    """Reads classifiers from cached file"""
    with (get_cache_dir() / 'classifiers.lst').open(encoding='utf-8') as f:
        valid_classifiers = set(l.strip() for l in f)
    return valid_classifiers


def _download_and_cache_classifiers():
    """Get the list of valid trove classifiers from PyPI"""
    log.info('Fetching list of valid trove classifiers')
    resp = requests.get(
        'https://pypi.org/pypi?%3Aaction=list_classifiers')
    resp.raise_for_status()

    cache_dir = get_cache_dir()
    try:
        cache_dir.mkdir(parents=True)
    except (FileExistsError, PermissionError):
        pass
    except OSError as e:
        # readonly mounted file raises OSError, only these should be captured
        if e.errno != errno.EROFS:
            raise

    try:
        with (cache_dir / 'classifiers.lst').open('wb') as f:
            f.write(resp.content)
    except (PermissionError, FileNotFoundError):
        # cache file could not be created
        pass
    except OSError as e:
        # readonly mounted file raises OSError, only these should be captured
        if e.errno != errno.EROFS:
            raise

    valid_classifiers = set(l.strip() for l in resp.text.splitlines())
    return valid_classifiers


def _verify_classifiers(classifiers, valid_classifiers):
    """Check classifiers against a set of known classifiers"""
    invalid = classifiers - valid_classifiers
    return ["Unrecognised classifier: {!r}".format(c)
            for c in sorted(invalid)]


def validate_classifiers(classifiers):
    """Verify trove classifiers from config file.

    Fetches and caches a list of known classifiers from PyPI. Setting the
    environment variable FLIT_NO_NETWORK=1 will skip this if the classifiers
    are not already cached.
    """
    if not classifiers:
        return []

    problems = []
    classifiers = set(classifiers)
    try:
        valid_classifiers = _read_classifiers_cached()
        valid_classifiers.update(CUSTOM_CLASSIFIERS)
        problems = _verify_classifiers(classifiers, valid_classifiers)
    except (FileNotFoundError, PermissionError) as e1:
        # We haven't yet got the classifiers cached or couldn't read it
        pass
    else:
        if not problems:
            return []

    # Either we don't have the list, or there were unexpected classifiers
    # which might have been added since we last fetched it. Fetch and cache.

    if os.environ.get('FLIT_NO_NETWORK', ''):
        log.warning(
            "Not checking classifiers, because FLIT_NO_NETWORK is set")
        return []

    # Try to download up-to-date list of classifiers
    try:
        valid_classifiers = _download_and_cache_classifiers()
    except requests.ConnectionError:
        # The error you get on a train, going through Oregon, without wifi
        log.warning(
            "Couldn't get list of valid classifiers to check against")
        return problems
    valid_classifiers.update(CUSTOM_CLASSIFIERS)
    return _verify_classifiers(classifiers, valid_classifiers)


def validate_entrypoints(entrypoints):
    """Check that the loaded entrypoints are valid.

    Expects a dict of dicts, e.g.::

        {'console_scripts': {'flit': 'flit:main'}}
    """

    def _is_identifier_attr(s):
        return all(n.isidentifier() for n in s.split('.'))

    problems = []
    for groupname, group in entrypoints.items():
        for k, v in group.items():
            if ':' in v:
                mod, obj = v.split(':', 1)
                valid = _is_identifier_attr(mod) and _is_identifier_attr(obj)
            else:
                valid = _is_identifier_attr(v)

            if not valid:
                problems.append('Invalid entry point in group {}: '
                                '{} = {}'.format(groupname, k, v))
    return problems

# Distribution name, not quite the same as a Python identifier
NAME = re.compile(r'^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$', re.IGNORECASE)
r''
VERSION_SPEC = re.compile(r'(~=|===?|!=|<=?|>=?)\s*[A-Z0-9\-_.*+!]+$', re.IGNORECASE)
REQUIREMENT = re.compile(NAME.pattern[:-1] +  # Trim '$'
     r"""\s*(?P<extras>\[.*\])?
         \s*(?P<version>[(=~<>!@][^;]*)?
         \s*(?P<envmark>;.*)?
     $""", re.IGNORECASE | re.VERBOSE)
MARKER_OP = re.compile(r'(~=|===?|!=|<=?|>=?|\s+in\s+|\s+not in\s+)')

def validate_name(metadata):
    name = metadata.get('name', None)
    if name is None or NAME.match(name):
        return []
    return ['Invalid name: {!r}'.format(name)]


def _valid_version_specifier(s):
    for clause in s.split(','):
        if not VERSION_SPEC.match(clause.strip()):
            return False
    return True

def validate_requires_python(metadata):
    spec = metadata.get('requires_python', None)
    if spec is None or _valid_version_specifier(spec):
        return []
    return ['Invalid requires-python: {!r}'.format(spec)]

MARKER_VARS = {
    'python_version', 'python_full_version', 'os_name', 'sys_platform',
    'platform_release', 'platform_system', 'platform_version', 'platform_machine',
    'platform_python_implementation', 'implementation_name',
    'implementation_version', 'extra',
}

def validate_environment_marker(em):
    clauses = re.split(r'\s+(?:and|or)\s+', em)
    problems = []
    for c in clauses:
        # TODO: validate parentheses properly. They're allowed by PEP 508.
        parts = MARKER_OP.split(c.strip('()'))
        if len(parts) != 3:
            problems.append("Invalid expression in environment marker: {!r}".format(c))
            continue
        l, op, r = parts
        for var in (l.strip(), r.strip()):
            if var[:1] in {'"', "'"}:
                if len(var) < 2 or var[-1:] != var[:1]:
                    problems.append("Invalid string in environment marker: {}".format(var))
            elif var not in MARKER_VARS:
                problems.append("Invalid variable name in environment marker: {!r}".format(var))
    return problems

def validate_requires_dist(metadata):
    probs = []
    for req in metadata.get('requires_dist', []):
        m = REQUIREMENT.match(req)
        if not m:
            probs.append("Could not parse requirement: {!r}".format(req))
            continue

        extras, version, envmark = m.group('extras', 'version', 'envmark')
        if not (extras is None or all(NAME.match(e.strip())
                                      for e in extras[1:-1].split(','))):
            probs.append("Invalid extras in requirement: {!r}".format(req))
        if version is not None:
            if version.startswith('(') and version.endswith(')'):
                version = version[1:-1]
            if version.startswith('@'):
                pass  # url specifier  TODO: validate URL
            elif not _valid_version_specifier(version):
                print((extras, version, envmark))
                probs.append("Invalid version specifier {!r} in requirement {!r}"
                             .format(version, req))
        if envmark is not None:
            probs.extend(validate_environment_marker(envmark[1:]))
    return probs

def validate_url(url):
    if url is None:
        return []
    probs = []
    if not url.startswith(('http://', 'https://')):
        probs.append("URL {!r} doesn't start with https:// or http://"
                     .format(url))
    elif not url.split('//', 1)[1]:
        probs.append("URL missing address")
    return probs

def validate_project_urls(metadata):
    probs = []
    for prurl in metadata.get('project_urls', []):
        name, url = prurl.split(',', 1)
        url = url.lstrip()
        if not name:
            probs.append("No name for project URL {!r}".format(url))
        elif len(name) > 32:
            probs.append("Project URL name {!r} longer than 32 characters"
                         .format(name))
        probs.extend(validate_url(url))

    return probs


def validate_readme_rst(metadata):
    mimetype = metadata.get('description_content_type', '')

    if mimetype != 'text/x-rst':
        return []

    # rst check
    raw_desc = metadata.get('description', '')
    stream = io.StringIO()
    res = render(raw_desc, stream)
    if not res:
        return [
            ("The file description seems not to be valid rst for PyPI;"
             " it will be interpreted as plain text"),
            stream.getvalue(),
        ]

    return []  # rst rendered OK


def validate_config(config_info):
    i = config_info
    problems = sum([
        validate_classifiers(i.metadata.get('classifiers')),
        validate_entrypoints(i.entrypoints),
        validate_name(i.metadata),
        validate_requires_python(i.metadata),
        validate_requires_dist(i.metadata),
        validate_url(i.metadata.get('home_page', None)),
        validate_project_urls(i.metadata),
        validate_readme_rst(i.metadata)
                   ], [])

    for p in problems:
        log.error(p)
    return problems