Coverage for src/gitlabracadabra/packages/pypi.py: 79%

1#

2# Copyright (C) 2019-2025 Mathieu Parent <math.parent@gmail.com>

3#

4# This program is free software: you can redistribute it and/or modify

5# it under the terms of the GNU Lesser General Public License as published by

6# the Free Software Foundation, either version 3 of the License, or

7# (at your option) any later version.

8#

9# This program is distributed in the hope that it will be useful,

10# but WITHOUT ANY WARRANTY; without even the implied warranty of

11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

12# GNU Lesser General Public License for more details.

13#

14# You should have received a copy of the GNU Lesser General Public License

15# along with this program. If not, see <http://www.gnu.org/licenses/>.

16

17 from __future__ import annotations

18

19 from html import unescape

20 from logging import getLogger

21 from posixpath import join as posixpath_join

22 from typing import TYPE_CHECKING, Any

23 from urllib.parse import quote as urlquote

24 from urllib.parse import urljoin, urlparse, urlunparse

25 from urllib.request import parse_keqv_list

26

27 from html5lib import parse as html5lib_parse

28 from packaging.requirements import InvalidRequirement, Requirement

29 from packaging.utils import canonicalize_name

30 from packaging.version import InvalidVersion, Version

31 from requests import codes

32

33 from gitlabracadabra.packages.package_file import PackageFile

34 from gitlabracadabra.packages.pip import extract_version_from_fragment

35 from gitlabracadabra.packages.source import Source

36

37 if TYPE_CHECKING: 37 ↛ 38line 37 didn't jump to line 38 because the condition on line 37 was never true

38 from requests.models import Response

39

40 from gitlabracadabra.packages.destination import Destination

41

42 try:

43 from packaging.utils import parse_wheel_filename

44

45 HAS_PACKAGING_PARSERS = True

46 except ImportError: # packaging << 20.9

47 HAS_PACKAGING_PARSERS = False

48

49 logger = getLogger(__name__)

50

51

52 class PyPI(Source):

53 """PyPI repository."""

54

55 def __init__(

56 self,

57 *,

58 log_prefix: str = "",

59 index_url: str | None = None,

60 requirements: str | list[str],

61 ) -> None:

62 """Initialize a PyPI repository object.

63

64 Args:

65 log_prefix: Log prefix.

66 index_url: index-url (default to https://pypi.org/simple).

67 requirements: Python requirements as list or string.

68 """

69 super().__init__()

70 self._log_prefix = log_prefix

71 self._index_url = index_url or "https://pypi.org/simple"

72 if isinstance(requirements, str):

73 self._requirements = requirements.splitlines()

74 else:

75 self._requirements = [req for reqs in requirements for req in reqs.splitlines()]

76

77 def __str__(self) -> str:

78 """Return string representation.

79

80 Returns:

81 A string.

82 """

83 return "PyPI repository"

84

85 def package_files(

86 self,

87 destination: Destination, # noqa: ARG002

88 ) -> list[PackageFile]:

89 """Return list of package files.

90

91 Returns:

92 List of package files.

93 """

94 package_files: list[PackageFile] = []

95 if not HAS_PACKAGING_PARSERS: 95 ↛ 96line 95 didn't jump to line 96 because the condition on line 95 was never true

96 logger.error(

97 "%sPyPI packages mirroring requires packaging >= 20.9",

98 self._log_prefix,

99 )

100 return package_files

101 for requirement_string in self._requirements:

102 if requirement_string.lstrip().startswith("#"):

103 continue

104 package_files_from_requirement_string = self._package_files_from_requirement_string(requirement_string)

105 if not package_files_from_requirement_string: 105 ↛ 106line 105 didn't jump to line 106 because the condition on line 105 was never true

106 logger.warning(

107 "%sNo package files matching found for requirement: %s",

108 self._log_prefix,

109 requirement_string,

110 )

111 package_files.extend(package_files_from_requirement_string)

112 return package_files

113

114 def _package_files_from_requirement_string(self, requirement_string: str) -> list[PackageFile]:

115 try:

116 req = Requirement(requirement_string)

117 except InvalidRequirement:

118 logger.warning(

119 '%sInvalid requirement "%s"',

120 self._log_prefix,

121 requirement_string,

122 )

123 return []

124 return self._package_files_from_requirement(req)

125

126 def _package_files_from_requirement(self, req: Requirement) -> list[PackageFile]:

127 index_url = self._get_index_url(req.name)

128 index_response = self.session.request("get", index_url)

129 if index_response.status_code != codes["ok"]: 129 ↛ 130line 129 didn't jump to line 130 because the condition on line 129 was never true

130 logger.warning(

131 "%sUnexpected HTTP status for PyPI index %s: received %i %s",

132 self._log_prefix,

133 index_url,

134 index_response.status_code,

135 index_response.reason,

136 )

137 return []

138 return self._package_files_from_requirement_and_response(req, index_response)

139

140 def _get_index_url(self, project_name: str) -> str:

141 loc = posixpath_join(

142 self._index_url,

143 urlquote(canonicalize_name(project_name)),

144 )

145 if not loc.endswith("/"): 145 ↛ 147line 145 didn't jump to line 147 because the condition on line 145 was always true

146 loc = f"{loc}/"

147 return loc

148

149 def _package_files_from_requirement_and_response(

150 self,

151 req: Requirement,

152 response: Response,

153 ) -> list[PackageFile]:

154 document = html5lib_parse(

155 response.content,

156 transport_encoding=response.encoding,

157 namespaceHTMLElements=False,

158 )

159

160 base_url = self._get_base_url(response, document)

161

162 package_files: dict[Version, list[PackageFile]] = {}

163 for anchor in document.findall(".//a"):

164 version, package_file = self._package_file_from_requirement_and_anchor(req, anchor, base_url)

165 if version and package_file:

166 if version not in package_files:

167 package_files[version] = []

168 package_files[version].append(package_file)

169

170 try:

171 best_match = sorted(package_files, reverse=True)[0]

172 except IndexError:

173 return []

174 return package_files[best_match]

175

176 def _get_base_url(self, response: Response, document: Any) -> str:

177 base_url = response.url

178 for base in document.findall(".//base"): 178 ↛ 179line 178 didn't jump to line 179 because the loop on line 178 never started

179 href = base.get("href")

180 if href is not None:

181 base_url = href

182 break

183 return base_url

184

185 def _package_file_from_requirement_and_anchor(

186 self,

187 req: Requirement,

188 anchor: Any,

189 base_url: str,

190 ) -> tuple[Version | None, PackageFile | None]:

191 if "href" not in anchor.keys(): # noqa: SIM118 191 ↛ 192line 191 didn't jump to line 192 because the condition on line 191 was never true

192 return None, None

193 if anchor.get("data-yanked") and not str(req.specifier).startswith("=="):

194 return None, None

195

196 parsed_url = urlparse(urljoin(base_url, anchor.get("href")))

197

198 filename = parsed_url.path.split("/")[-1]

199 try:

200 name, ver = self._parse_filename(filename, canonicalize_name(req.name))

201 except InvalidVersion:

202 # Ignore invalid versions, like in pbr-0.5.2.5.g5b3e942.tar.gz

203 logger.debug(

204 "%sIgnoring invalid version for filename %s",

205 self._log_prefix,

206 filename,

207 )

208 return None, None

209

210 if name is None or ver is None or ver not in req.specifier:

211 return None, None

212

213 metadata = parse_keqv_list(parsed_url.fragment.split("&"))

214

215 if "data-requires-python" in anchor.keys(): # noqa: SIM118

216 metadata["requires-python"] = unescape(anchor.get("data-requires-python"))

217

218 return ver, PackageFile(

219 urlunparse(parsed_url._replace(fragment="")),

220 "pypi",

221 name,

222 str(ver),

223 filename,

224 metadata=metadata,

225 )

226

227 def _parse_filename(self, filename: str, canonical_name: str) -> tuple[str | None, Version | None]:

228 if filename.endswith(".whl"):

229 name, ver, _, _ = parse_wheel_filename(filename)

230 return name, ver

231 if filename.endswith(".egg"):

232 # Ignore egg files for now

233 return None, None

234 if filename.endswith(".tar.gz"): 234 ↛ 238line 234 didn't jump to line 238 because the condition on line 234 was always true

235 ver_str = extract_version_from_fragment(filename[:-7], canonical_name)

236 if ver_str: 236 ↛ 238line 236 didn't jump to line 238 because the condition on line 236 was always true

237 return canonical_name, Version(ver_str)

238 return None, None