resolver.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299
  1. import functools
  2. import logging
  3. import os
  4. from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, cast
  5. from pip._vendor.packaging.utils import canonicalize_name
  6. from pip._vendor.resolvelib import BaseReporter, ResolutionImpossible
  7. from pip._vendor.resolvelib import Resolver as RLResolver
  8. from pip._vendor.resolvelib.structs import DirectedGraph
  9. from pip._internal.cache import WheelCache
  10. from pip._internal.index.package_finder import PackageFinder
  11. from pip._internal.operations.prepare import RequirementPreparer
  12. from pip._internal.req.req_install import InstallRequirement
  13. from pip._internal.req.req_set import RequirementSet
  14. from pip._internal.resolution.base import BaseResolver, InstallRequirementProvider
  15. from pip._internal.resolution.resolvelib.provider import PipProvider
  16. from pip._internal.resolution.resolvelib.reporter import (
  17. PipDebuggingReporter,
  18. PipReporter,
  19. )
  20. from .base import Candidate, Requirement
  21. from .factory import Factory
  22. if TYPE_CHECKING:
  23. from pip._vendor.resolvelib.resolvers import Result as RLResult
  24. Result = RLResult[Requirement, Candidate, str]
  25. logger = logging.getLogger(__name__)
  26. class Resolver(BaseResolver):
  27. _allowed_strategies = {"eager", "only-if-needed", "to-satisfy-only"}
  28. def __init__(
  29. self,
  30. preparer: RequirementPreparer,
  31. finder: PackageFinder,
  32. wheel_cache: Optional[WheelCache],
  33. make_install_req: InstallRequirementProvider,
  34. use_user_site: bool,
  35. ignore_dependencies: bool,
  36. ignore_installed: bool,
  37. ignore_requires_python: bool,
  38. force_reinstall: bool,
  39. upgrade_strategy: str,
  40. py_version_info: Optional[Tuple[int, ...]] = None,
  41. ):
  42. super().__init__()
  43. assert upgrade_strategy in self._allowed_strategies
  44. self.factory = Factory(
  45. finder=finder,
  46. preparer=preparer,
  47. make_install_req=make_install_req,
  48. wheel_cache=wheel_cache,
  49. use_user_site=use_user_site,
  50. force_reinstall=force_reinstall,
  51. ignore_installed=ignore_installed,
  52. ignore_requires_python=ignore_requires_python,
  53. py_version_info=py_version_info,
  54. )
  55. self.ignore_dependencies = ignore_dependencies
  56. self.upgrade_strategy = upgrade_strategy
  57. self._result: Optional[Result] = None
  58. def resolve(
  59. self, root_reqs: List[InstallRequirement], check_supported_wheels: bool
  60. ) -> RequirementSet:
  61. collected = self.factory.collect_root_requirements(root_reqs)
  62. provider = PipProvider(
  63. factory=self.factory,
  64. constraints=collected.constraints,
  65. ignore_dependencies=self.ignore_dependencies,
  66. upgrade_strategy=self.upgrade_strategy,
  67. user_requested=collected.user_requested,
  68. )
  69. if "PIP_RESOLVER_DEBUG" in os.environ:
  70. reporter: BaseReporter = PipDebuggingReporter()
  71. else:
  72. reporter = PipReporter()
  73. resolver: RLResolver[Requirement, Candidate, str] = RLResolver(
  74. provider,
  75. reporter,
  76. )
  77. try:
  78. limit_how_complex_resolution_can_be = 200000
  79. result = self._result = resolver.resolve(
  80. collected.requirements, max_rounds=limit_how_complex_resolution_can_be
  81. )
  82. except ResolutionImpossible as e:
  83. error = self.factory.get_installation_error(
  84. cast("ResolutionImpossible[Requirement, Candidate]", e),
  85. collected.constraints,
  86. )
  87. raise error from e
  88. req_set = RequirementSet(check_supported_wheels=check_supported_wheels)
  89. for candidate in result.mapping.values():
  90. ireq = candidate.get_install_requirement()
  91. if ireq is None:
  92. continue
  93. # Check if there is already an installation under the same name,
  94. # and set a flag for later stages to uninstall it, if needed.
  95. installed_dist = self.factory.get_dist_to_uninstall(candidate)
  96. if installed_dist is None:
  97. # There is no existing installation -- nothing to uninstall.
  98. ireq.should_reinstall = False
  99. elif self.factory.force_reinstall:
  100. # The --force-reinstall flag is set -- reinstall.
  101. ireq.should_reinstall = True
  102. elif installed_dist.version != candidate.version:
  103. # The installation is different in version -- reinstall.
  104. ireq.should_reinstall = True
  105. elif candidate.is_editable or installed_dist.editable:
  106. # The incoming distribution is editable, or different in
  107. # editable-ness to installation -- reinstall.
  108. ireq.should_reinstall = True
  109. elif candidate.source_link and candidate.source_link.is_file:
  110. # The incoming distribution is under file://
  111. if candidate.source_link.is_wheel:
  112. # is a local wheel -- do nothing.
  113. logger.info(
  114. "%s is already installed with the same version as the "
  115. "provided wheel. Use --force-reinstall to force an "
  116. "installation of the wheel.",
  117. ireq.name,
  118. )
  119. continue
  120. # is a local sdist or path -- reinstall
  121. ireq.should_reinstall = True
  122. else:
  123. continue
  124. link = candidate.source_link
  125. if link and link.is_yanked:
  126. # The reason can contain non-ASCII characters, Unicode
  127. # is required for Python 2.
  128. msg = (
  129. "The candidate selected for download or install is a "
  130. "yanked version: {name!r} candidate (version {version} "
  131. "at {link})\nReason for being yanked: {reason}"
  132. ).format(
  133. name=candidate.name,
  134. version=candidate.version,
  135. link=link,
  136. reason=link.yanked_reason or "<none given>",
  137. )
  138. logger.warning(msg)
  139. req_set.add_named_requirement(ireq)
  140. reqs = req_set.all_requirements
  141. self.factory.preparer.prepare_linked_requirements_more(reqs)
  142. for req in reqs:
  143. req.prepared = True
  144. req.needs_more_preparation = False
  145. return req_set
  146. def get_installation_order(
  147. self, req_set: RequirementSet
  148. ) -> List[InstallRequirement]:
  149. """Get order for installation of requirements in RequirementSet.
  150. The returned list contains a requirement before another that depends on
  151. it. This helps ensure that the environment is kept consistent as they
  152. get installed one-by-one.
  153. The current implementation creates a topological ordering of the
  154. dependency graph, giving more weight to packages with less
  155. or no dependencies, while breaking any cycles in the graph at
  156. arbitrary points. We make no guarantees about where the cycle
  157. would be broken, other than it *would* be broken.
  158. """
  159. assert self._result is not None, "must call resolve() first"
  160. if not req_set.requirements:
  161. # Nothing is left to install, so we do not need an order.
  162. return []
  163. graph = self._result.graph
  164. weights = get_topological_weights(graph, set(req_set.requirements.keys()))
  165. sorted_items = sorted(
  166. req_set.requirements.items(),
  167. key=functools.partial(_req_set_item_sorter, weights=weights),
  168. reverse=True,
  169. )
  170. return [ireq for _, ireq in sorted_items]
  171. def get_topological_weights(
  172. graph: "DirectedGraph[Optional[str]]", requirement_keys: Set[str]
  173. ) -> Dict[Optional[str], int]:
  174. """Assign weights to each node based on how "deep" they are.
  175. This implementation may change at any point in the future without prior
  176. notice.
  177. We first simplify the dependency graph by pruning any leaves and giving them
  178. the highest weight: a package without any dependencies should be installed
  179. first. This is done again and again in the same way, giving ever less weight
  180. to the newly found leaves. The loop stops when no leaves are left: all
  181. remaining packages have at least one dependency left in the graph.
  182. Then we continue with the remaining graph, by taking the length for the
  183. longest path to any node from root, ignoring any paths that contain a single
  184. node twice (i.e. cycles). This is done through a depth-first search through
  185. the graph, while keeping track of the path to the node.
  186. Cycles in the graph result would result in node being revisited while also
  187. being on its own path. In this case, take no action. This helps ensure we
  188. don't get stuck in a cycle.
  189. When assigning weight, the longer path (i.e. larger length) is preferred.
  190. We are only interested in the weights of packages that are in the
  191. requirement_keys.
  192. """
  193. path: Set[Optional[str]] = set()
  194. weights: Dict[Optional[str], int] = {}
  195. def visit(node: Optional[str]) -> None:
  196. if node in path:
  197. # We hit a cycle, so we'll break it here.
  198. return
  199. # Time to visit the children!
  200. path.add(node)
  201. for child in graph.iter_children(node):
  202. visit(child)
  203. path.remove(node)
  204. if node not in requirement_keys:
  205. return
  206. last_known_parent_count = weights.get(node, 0)
  207. weights[node] = max(last_known_parent_count, len(path))
  208. # Simplify the graph, pruning leaves that have no dependencies.
  209. # This is needed for large graphs (say over 200 packages) because the
  210. # `visit` function is exponentially slower then, taking minutes.
  211. # See https://github.com/pypa/pip/issues/10557
  212. # We will loop until we explicitly break the loop.
  213. while True:
  214. leaves = set()
  215. for key in graph:
  216. if key is None:
  217. continue
  218. for _child in graph.iter_children(key):
  219. # This means we have at least one child
  220. break
  221. else:
  222. # No child.
  223. leaves.add(key)
  224. if not leaves:
  225. # We are done simplifying.
  226. break
  227. # Calculate the weight for the leaves.
  228. weight = len(graph) - 1
  229. for leaf in leaves:
  230. if leaf not in requirement_keys:
  231. continue
  232. weights[leaf] = weight
  233. # Remove the leaves from the graph, making it simpler.
  234. for leaf in leaves:
  235. graph.remove(leaf)
  236. # Visit the remaining graph.
  237. # `None` is guaranteed to be the root node by resolvelib.
  238. visit(None)
  239. # Sanity check: all requirement keys should be in the weights,
  240. # and no other keys should be in the weights.
  241. difference = set(weights.keys()).difference(requirement_keys)
  242. assert not difference, difference
  243. return weights
  244. def _req_set_item_sorter(
  245. item: Tuple[str, InstallRequirement],
  246. weights: Dict[Optional[str], int],
  247. ) -> Tuple[int, str]:
  248. """Key function used to sort install requirements for installation.
  249. Based on the "weight" mapping calculated in ``get_installation_order()``.
  250. The canonical package name is returned as the second member as a tie-
  251. breaker to ensure the result is predictable, which is useful in tests.
  252. """
  253. name = canonicalize_name(item[0])
  254. return weights[name], name