pep508_rs-0.2.3/.cargo_vcs_info.json0000644000000001360000000000100125630ustar { "git": { "sha1": "0e705a2682a10fed966de69ea0860041a980c79f" }, "path_in_vcs": "" }pep508_rs-0.2.3/Cargo.toml0000644000000042300000000000100105600ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2021" name = "pep508_rs" version = "0.2.3" include = [ "/src", "Changelog.md", "License-Apache", "License-BSD", "Readme.md", "pyproject.toml", ] description = "A library for python dependency specifiers, better known as PEP 508" readme = "Readme.md" license = "Apache-2.0 OR BSD-2-Clause" repository = "https://github.com/konstin/pep508_rs" [profile.maturin] inherits = "release" strip = true [profile.release] debug = 2 [lib] name = "pep508_rs" crate-type = [ "cdylib", "rlib", ] [dependencies.anyhow] version = "1.0.75" optional = true [dependencies.once_cell] version = "1.18.0" [dependencies.pep440_rs] version = "0.3.11" [dependencies.pyo3] version = "0.19.2" features = [ "abi3", "extension-module", ] optional = true [dependencies.pyo3-log] version = "0.8.3" optional = true [dependencies.regex] version = "1.9.5" features = ["std"] default-features = false [dependencies.serde] version = "1.0.188" features = ["derive"] optional = true [dependencies.serde_json] version = "1.0.107" optional = true [dependencies.thiserror] version = "1.0.49" [dependencies.toml] version = "0.8.1" optional = true [dependencies.tracing] version = "0.1.37" features = ["log"] [dependencies.unicode-width] version = "0.1.11" [dependencies.url] version = "2.4.1" features = ["serde"] [dev-dependencies.indoc] version = "2.0.4" [dev-dependencies.log] version = "0.4.20" [dev-dependencies.serde_json] version = "1.0.107" [dev-dependencies.testing_logger] version = "0.1.1" [features] default = [] modern = [ "serde", "toml", "anyhow", ] pyo3 = [ "dep:pyo3", "pep440_rs/pyo3", "pyo3-log", ] serde = [ "dep:serde", "pep440_rs/serde", ] pep508_rs-0.2.3/Cargo.toml.orig000064400000000000000000000026571046102023000142540ustar 00000000000000[package] name = "pep508_rs" version = "0.2.3" description = "A library for python dependency specifiers, better known as PEP 508" edition = "2021" include = ["/src", "Changelog.md", "License-Apache", "License-BSD", "Readme.md", "pyproject.toml"] # Same license as pypa/packaging where the tests are from license = "Apache-2.0 OR BSD-2-Clause" readme = "Readme.md" repository = "https://github.com/konstin/pep508_rs" [dependencies] anyhow = { version = "1.0.75", optional = true } once_cell = "1.18.0" pep440_rs = "0.3.11" pyo3 = { version = "0.19.2", optional = true, features = ["abi3", "extension-module"] } pyo3-log = { version = "0.8.3", optional = true } regex = { version = "1.9.5", default-features = false, features = ["std"] } serde = { version = "1.0.188", features = ["derive"], optional = true } serde_json = { version = "1.0.107", optional = true } thiserror = "1.0.49" toml = { version = "0.8.1", optional = true } tracing = { version = "0.1.37", features = ["log"] } unicode-width = "0.1.11" url = { version = "2.4.1", features = ["serde"] } [dev-dependencies] indoc = "2.0.4" log = "0.4.20" testing_logger = "0.1.1" serde_json = "1.0.107" [features] pyo3 = ["dep:pyo3", "pep440_rs/pyo3", "pyo3-log"] serde = ["dep:serde", "pep440_rs/serde"] modern = ["serde", "toml", "anyhow"] default = [] [lib] name = "pep508_rs" crate-type = ["cdylib", "rlib"] [profile.release] debug = true [profile.maturin] inherits = "release" strip = true pep508_rs-0.2.3/License-Apache000064400000000000000000000236751046102023000140540ustar 00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS pep508_rs-0.2.3/License-BSD000064400000000000000000000024151046102023000132700ustar 00000000000000Copyright (c) 2023 konstin Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. pep508_rs-0.2.3/Readme.md000064400000000000000000000057371046102023000131060ustar 00000000000000# Dependency specifiers (PEP 508) in Rust [![Crates.io](https://img.shields.io/crates/v/pep508_rs.svg?logo=rust&style=flat-square)](https://crates.io/crates/pep508_rs) [![PyPI](https://img.shields.io/pypi/v/pep508_rs.svg?logo=python&style=flat-square)](https://pypi.org/project/pep508_rs) A library for python [dependency specifiers](https://packaging.python.org/en/latest/specifications/dependency-specifiers/), better known as [PEP 508](https://peps.python.org/pep-0508/). ## Usage **In Rust** ```rust use std::str::FromStr; use pep508_rs::Requirement; let marker = r#"requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8""#; let dependency_specification = Requirement::from_str(marker).unwrap(); assert_eq!(dependency_specification.name, "requests"); assert_eq!(dependency_specification.extras, Some(vec!["security".to_string(), "tests".to_string()])); ``` **In Python** ```python from pep508_rs import Requirement requests = Requirement( 'requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"' ) assert requests.name == "requests" assert requests.extras == ["security", "tests"] assert [str(i) for i in requests.version_or_url] == [">= 2.8.1", "== 2.8.*"] ``` Python bindings are built with [maturin](https://github.com/PyO3/maturin), but you can also use the normal `pip install .` `Version` and `VersionSpecifier` from [pep440_rs](https://github.com/konstin/pep440-rs) are reexported to avoid type mismatches. ## Markers Markers allow you to install dependencies only in specific environments (python version, operating system, architecture, etc.) or when a specific feature is activated. E.g. you can say `importlib-metadata ; python_version < "3.8"` or `itsdangerous (>=1.1.0) ; extra == 'security'`. Unfortunately, the marker grammar has some oversights (e.g. ) and the design of comparisons (PEP 440 comparisons with lexicographic fallback) leads to confusing outcomes. This implementation tries to carefully validate everything and emit warnings whenever bogus comparisons with unintended semantics are made. In python, warnings are by default sent to the normal python logging infrastructure: ```python from pep508_rs import Requirement, MarkerEnvironment env = MarkerEnvironment.current() assert not Requirement("numpy; extra == 'science'").evaluate_markers(env, []) assert Requirement("numpy; extra == 'science'").evaluate_markers(env, ["science"]) assert not Requirement( "numpy; extra == 'science' and extra == 'arrays'" ).evaluate_markers(env, ["science"]) assert Requirement( "numpy; extra == 'science' or extra == 'arrays'" ).evaluate_markers(env, ["science"]) ``` ```python from pep508_rs import Requirement, MarkerEnvironment env = MarkerEnvironment.current() Requirement("numpy; python_version >= '3.9.'").evaluate_markers(env, []) # This will log: # "Expected PEP 440 version to compare with python_version, found '3.9.', " # "evaluating to false: Version `3.9.` doesn't match PEP 440 rules" ``` pep508_rs-0.2.3/pyproject.toml000064400000000000000000000012241046102023000142660ustar 00000000000000[project] name = "pep508_rs" version = "0.2.2" description = "A library for python dependency specifiers, better known as PEP 508" readme = "Readme.md" [tool.poetry] name = "pep508_rs" version = "0.1.1" description = "" authors = ["konstin "] readme = "Readme.md" [tool.poetry.dependencies] python = ">=3.7" [tool.poetry.group.dev.dependencies] black = { extras = ["jupyter"], version = "^23.1.0" } maturin = "^1.0.0" pytest = "^7.2.0" ruff = "^0.0.270" [tool.maturin] features = ["pyo3"] [tool.pytest.ini_options] minversion = "7.2.0" addopts = "--tb=short" [build-system] requires = ["maturin>=1.0,<2.0"] build-backend = "maturin" pep508_rs-0.2.3/src/lib.rs000064400000000000000000001270131046102023000132620ustar 00000000000000//! A library for python [dependency specifiers](https://packaging.python.org/en/latest/specifications/dependency-specifiers/) //! better known as [PEP 508](https://peps.python.org/pep-0508/) //! //! ## Usage //! //! ``` //! use std::str::FromStr; //! use pep508_rs::Requirement; //! //! let marker = r#"requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8""#; //! let dependency_specification = Requirement::from_str(marker).unwrap(); //! assert_eq!(dependency_specification.name, "requests"); //! assert_eq!(dependency_specification.extras, Some(vec!["security".to_string(), "tests".to_string()])); //! ``` #![deny(missing_docs)] mod marker; #[cfg(feature = "modern")] pub mod modern; pub use marker::{ MarkerEnvironment, MarkerExpression, MarkerOperator, MarkerTree, MarkerValue, MarkerValueString, MarkerValueVersion, MarkerWarningKind, StringVersion, }; #[cfg(feature = "pyo3")] use pep440_rs::PyVersion; use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers}; #[cfg(feature = "pyo3")] use pyo3::{ basic::CompareOp, create_exception, exceptions::PyNotImplementedError, pyclass, pymethods, pymodule, types::PyModule, IntoPy, PyObject, PyResult, Python, }; #[cfg(feature = "serde")] use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; #[cfg(feature = "pyo3")] use std::collections::hash_map::DefaultHasher; use std::collections::HashSet; use std::fmt::{Display, Formatter}; #[cfg(feature = "pyo3")] use std::hash::{Hash, Hasher}; use std::str::{Chars, FromStr}; use thiserror::Error; use unicode_width::UnicodeWidthStr; use url::Url; /// Error with a span attached. Not that those aren't `String` but `Vec` indices. #[derive(Debug, Clone, Eq, PartialEq)] pub struct Pep508Error { /// Either we have an error string from our parser or an upstream error from `url` pub message: Pep508ErrorSource, /// Span start index pub start: usize, /// Span length pub len: usize, /// The input string so we can print it underlined pub input: String, } /// Either we have an error string from our parser or an upstream error from `url` #[derive(Debug, Error, Clone, Eq, PartialEq)] pub enum Pep508ErrorSource { /// An error from our parser String(String), /// A url parsing error #[error(transparent)] UrlError(#[from] url::ParseError), } impl Display for Pep508ErrorSource { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { Pep508ErrorSource::String(string) => string.fmt(f), Pep508ErrorSource::UrlError(parse_err) => parse_err.fmt(f), } } } impl Display for Pep508Error { /// Pretty formatting with underline fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { // We can use char indices here since it's a Vec let start_offset = self .input .chars() .take(self.start) .collect::() .width(); let underline_len = if self.start == self.input.len() { // We also allow 0 here for convenience assert!( self.len <= 1, "Can only go one past the input not {}", self.len ); 1 } else { self.input .chars() .skip(self.start) .take(self.len) .collect::() .width() }; write!( f, "{}\n{}\n{}{}", self.message, self.input, " ".repeat(start_offset), "^".repeat(underline_len) ) } } /// We need this to allow e.g. anyhow's `.context()` impl std::error::Error for Pep508Error {} #[cfg(feature = "pyo3")] create_exception!( pep508, PyPep508Error, pyo3::exceptions::PyValueError, "A PEP 508 parser error with span information" ); /// A PEP 508 dependency specification #[cfg_attr(feature = "pyo3", pyclass(module = "pep508"))] #[derive(Hash, Debug, Clone, Eq, PartialEq)] pub struct Requirement { /// The distribution name such as `numpy` in /// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"` pub name: String, /// The list of extras such as `security`, `tests` in /// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"` pub extras: Option>, /// The version specifier such as `>= 2.8.1`, `== 2.8.*` in /// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"` /// or a url pub version_or_url: Option, /// The markers such as `python_version > "3.8"` in /// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`. /// Those are a nested and/or tree pub marker: Option, } impl Display for Requirement { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.name)?; if let Some(extras) = &self.extras { write!(f, "[{}]", extras.join(","))?; } if let Some(version_or_url) = &self.version_or_url { match version_or_url { VersionOrUrl::VersionSpecifier(version_specifier) => { let version_specifier: Vec = version_specifier.iter().map(ToString::to_string).collect(); write!(f, " {}", version_specifier.join(", "))?; } VersionOrUrl::Url(url) => { // We add the space for markers later if necessary write!(f, " @ {}", url)?; } } } if let Some(marker) = &self.marker { write!(f, " ; {}", marker)?; } Ok(()) } } /// https://github.com/serde-rs/serde/issues/908#issuecomment-298027413 #[cfg(feature = "serde")] impl<'de> Deserialize<'de> for Requirement { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { let s = String::deserialize(deserializer)?; FromStr::from_str(&s).map_err(de::Error::custom) } } /// https://github.com/serde-rs/serde/issues/1316#issue-332908452 #[cfg(feature = "serde")] impl Serialize for Requirement { fn serialize(&self, serializer: S) -> Result where S: Serializer, { serializer.collect_str(self) } } #[cfg(feature = "pyo3")] #[pymethods] impl Requirement { /// The distribution name such as `numpy` in /// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"` #[getter] pub fn name(&self) -> String { self.name.clone() } /// The list of extras such as `security`, `tests` in /// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"` #[getter] pub fn extras(&self) -> Option> { self.extras.clone() } /// The marker expression such as `python_version > "3.8"` in /// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"` #[getter] pub fn marker(&self) -> Option { self.marker.as_ref().map(|m| m.to_string()) } /// Parses a PEP 440 string #[new] pub fn py_new(requirement: &str) -> PyResult { Self::from_str(requirement).map_err(|err| PyPep508Error::new_err(err.to_string())) } #[getter] fn version_or_url(&self, py: Python<'_>) -> PyObject { match &self.version_or_url { None => py.None(), Some(VersionOrUrl::VersionSpecifier(version_specifier)) => version_specifier .iter() .map(|x| x.clone().into_py(py)) .collect::>() .into_py(py), Some(VersionOrUrl::Url(url)) => url.to_string().into_py(py), } } fn __str__(&self) -> String { self.to_string() } fn __repr__(&self) -> String { format!(r#""{}""#, self) } fn __richcmp__(&self, other: &Self, op: CompareOp) -> PyResult { let err = PyNotImplementedError::new_err("Requirement only supports equality comparisons"); match op { CompareOp::Lt => Err(err), CompareOp::Le => Err(err), CompareOp::Eq => Ok(self == other), CompareOp::Ne => Ok(self != other), CompareOp::Gt => Err(err), CompareOp::Ge => Err(err), } } fn __hash__(&self) -> u64 { let mut hasher = DefaultHasher::new(); self.hash(&mut hasher); hasher.finish() } /// Returns whether the markers apply for the given environment #[pyo3(name = "evaluate_markers")] pub fn py_evaluate_markers(&self, env: &MarkerEnvironment, extras: Vec) -> bool { self.evaluate_markers(env, extras) } /// Returns whether the requirement would be satisfied, independent of environment markers, i.e. /// if there is potentially an environment that could activate this requirement. /// /// Note that unlike [Self::evaluate_markers] this does not perform any checks for bogus /// expressions but will simply return true. As caller you should separately perform a check /// with an environment and forward all warnings. #[pyo3(name = "evaluate_extras_and_python_version")] pub fn py_evaluate_extras_and_python_version( &self, extras: HashSet, python_versions: Vec, ) -> bool { self.evaluate_extras_and_python_version( extras, python_versions .into_iter() .map(|py_version| py_version.0) .collect(), ) } /// Returns whether the markers apply for the given environment #[pyo3(name = "evaluate_markers_and_report")] pub fn py_evaluate_markers_and_report( &self, env: &MarkerEnvironment, extras: Vec, ) -> (bool, Vec<(MarkerWarningKind, String, String)>) { self.evaluate_markers_and_report(env, extras) } } impl Requirement { /// Returns whether the markers apply for the given environment pub fn evaluate_markers(&self, env: &MarkerEnvironment, extras: Vec) -> bool { if let Some(marker) = &self.marker { marker.evaluate( env, &extras.iter().map(String::as_str).collect::>(), ) } else { true } } /// Returns whether the requirement would be satisfied, independent of environment markers, i.e. /// if there is potentially an environment that could activate this requirement. /// /// Note that unlike [Self::evaluate_markers] this does not perform any checks for bogus /// expressions but will simply return true. As caller you should separately perform a check /// with an environment and forward all warnings. pub fn evaluate_extras_and_python_version( &self, extras: HashSet, python_versions: Vec, ) -> bool { if let Some(marker) = &self.marker { marker.evaluate_extras_and_python_version(&extras, &python_versions) } else { true } } /// Returns whether the markers apply for the given environment pub fn evaluate_markers_and_report( &self, env: &MarkerEnvironment, extras: Vec, ) -> (bool, Vec<(MarkerWarningKind, String, String)>) { if let Some(marker) = &self.marker { marker.evaluate_collect_warnings( env, &extras.iter().map(|x| x.as_str()).collect::>(), ) } else { (true, Vec::new()) } } } impl FromStr for Requirement { type Err = Pep508Error; /// Parse a [Dependency Specifier](https://packaging.python.org/en/latest/specifications/dependency-specifiers/) fn from_str(input: &str) -> Result { parse(&mut CharIter::new(input)) } } impl Requirement { /// Parse a [Dependency Specifier](https://packaging.python.org/en/latest/specifications/dependency-specifiers/) pub fn parse(input: &mut CharIter) -> Result { parse(input) } } /// The actual version specifier or url to install #[derive(Debug, Clone, Eq, Hash, PartialEq)] pub enum VersionOrUrl { /// A PEP 440 version specifier set VersionSpecifier(VersionSpecifiers), /// A installable URL Url(Url), } /// A `Vec` and an index inside of it. Like [String], but with utf-8 aware indexing pub struct CharIter<'a> { input: &'a str, chars: Chars<'a>, /// char-based (not byte-based) position pos: usize, } impl<'a> CharIter<'a> { /// Convert from `&str` pub fn new(input: &'a str) -> Self { Self { input, chars: input.chars(), pos: 0, } } fn copy_chars(&self) -> String { self.input.to_string() } fn peek(&self) -> Option<(usize, char)> { self.chars.clone().next().map(|char| (self.pos, char)) } fn eat(&mut self, token: char) -> Option { let (start_pos, peek_char) = self.peek()?; if peek_char == token { self.next(); Some(start_pos) } else { None } } fn next(&mut self) -> Option<(usize, char)> { let next = (self.pos, self.chars.next()?); self.pos += 1; Some(next) } fn peek_char(&self) -> Option { self.chars.clone().next() } fn get_pos(&self) -> usize { self.pos } fn peek_while(&mut self, condition: impl Fn(char) -> bool) -> (String, usize, usize) { let peeker = self.chars.clone(); let start = self.get_pos(); let mut len = 0; let substring = peeker .take_while(|c| { if condition(*c) { len += 1; true } else { false } }) .collect::(); (substring, start, len) } fn take_while(&mut self, condition: impl Fn(char) -> bool) -> (String, usize, usize) { // no pretty, but works let mut substring = String::new(); let start = self.get_pos(); let mut len = 0; while let Some(char) = self.peek_char() { if !condition(char) { break; } else { substring.push(char); self.next(); len += 1; } } (substring, start, len) } fn next_expect_char(&mut self, expected: char, span_start: usize) -> Result<(), Pep508Error> { match self.next() { None => Err(Pep508Error { message: Pep508ErrorSource::String(format!( "Expected '{}', found end of dependency specification", expected )), start: span_start, len: 1, input: self.copy_chars(), }), Some((_, value)) if value == expected => Ok(()), Some((pos, other)) => Err(Pep508Error { message: Pep508ErrorSource::String(format!( "Expected '{}', found '{}'", expected, other )), start: pos, len: 1, input: self.copy_chars(), }), } } fn eat_whitespace(&mut self) { while let Some(char) = self.peek_char() { if char.is_whitespace() { self.next(); } else { return; } } } } fn parse_name(chars: &mut CharIter) -> Result { // https://peps.python.org/pep-0508/#names // ^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$ with re.IGNORECASE let mut name = String::new(); if let Some((index, char)) = chars.next() { if matches!(char, 'A'..='Z' | 'a'..='z' | '0'..='9') { name.push(char); } else { return Err(Pep508Error { message: Pep508ErrorSource::String(format!( "Expected package name starting with an alphanumeric character, found '{}'", char )), start: index, len: 1, input: chars.copy_chars(), }); } } else { return Err(Pep508Error { message: Pep508ErrorSource::String("Empty field is not allowed for PEP508".to_string()), start: 0, len: 1, input: chars.copy_chars(), }); } loop { match chars.peek() { Some((index, char @ ('A'..='Z' | 'a'..='z' | '0'..='9' | '.' | '-' | '_'))) => { name.push(char); chars.next(); // [.-_] can't be the final character if chars.peek().is_none() && matches!(char, '.' | '-' | '_') { return Err(Pep508Error { message: Pep508ErrorSource::String(format!( "Package name must end with an alphanumeric character, not '{}'", char )), start: index, len: 1, input: chars.copy_chars(), }); } } Some(_) | None => return Ok(name), } } } /// parses extras in the `[extra1,extra2] format` fn parse_extras(chars: &mut CharIter) -> Result>, Pep508Error> { let bracket_pos = match chars.eat('[') { Some(pos) => pos, None => return Ok(None), }; let mut extras = Vec::new(); loop { // wsp* before the identifier chars.eat_whitespace(); let mut buffer = String::new(); let early_eof_error = Pep508Error { message: Pep508ErrorSource::String( "Missing closing bracket (expected ']', found end of dependency specification)" .to_string(), ), start: bracket_pos, len: 1, input: chars.copy_chars(), }; // First char of the identifier match chars.next() { // letterOrDigit Some((_, alphanumeric @ ('a'..='z' | 'A'..='Z' | '0'..='9'))) => { buffer.push(alphanumeric) } Some((pos, other)) => { return Err(Pep508Error { message: Pep508ErrorSource::String(format!( "Expected an alphanumeric character starting the extra name, found '{}'", other )), start: pos, len: 1, input: chars.copy_chars(), }) } None => return Err(early_eof_error), } // Parse from the second char of the identifier // We handle the illegal character case below // identifier_end = letterOrDigit | (('-' | '_' | '.' )* letterOrDigit) // identifier_end* buffer.push_str( &chars .take_while( |char| matches!(char, 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.'), ) .0, ); match chars.peek() { Some((pos, char)) if char != ',' && char != ']' && !char.is_whitespace() => { return Err(Pep508Error { message: Pep508ErrorSource::String(format!( "Invalid character in extras name, expected an alphanumeric character, '-', '_', '.', ',' or ']', found '{}'", char )), start: pos, len: 1, input: chars.copy_chars(), }) } _=>{} }; // wsp* after the identifier chars.eat_whitespace(); // end or next identifier? match chars.next() { Some((_, ',')) => { extras.push(buffer); } Some((_, ']')) => { extras.push(buffer); break; } Some((pos, other)) => { return Err(Pep508Error { message: Pep508ErrorSource::String(format!( "Expected either ',' (separating extras) or ']' (ending the extras section), found '{other}'" )), start: pos, len: 1, input: chars.copy_chars(), }) } None => return Err(early_eof_error), } } Ok(Some(extras)) } fn parse_url(chars: &mut CharIter) -> Result { // wsp* chars.eat_whitespace(); // let (url, start, len) = chars.take_while(|char| !char.is_whitespace()); if url.is_empty() { return Err(Pep508Error { message: Pep508ErrorSource::String("Expected URL".to_string()), start, len, input: chars.copy_chars(), }); } let url = Url::parse(&url).map_err(|err| Pep508Error { message: Pep508ErrorSource::UrlError(err), start, len, input: chars.copy_chars(), })?; Ok(VersionOrUrl::Url(url)) } /// PEP 440 wrapper fn parse_specifier( chars: &mut CharIter, buffer: &str, start: usize, end: usize, ) -> Result { VersionSpecifier::from_str(buffer).map_err(|err| Pep508Error { message: Pep508ErrorSource::String(err), start, len: end - start, input: chars.copy_chars(), }) } /// Such as `>=1.19,<2.0`, either delimited by the end of the specifier or a `;` for the marker part /// /// ```text /// version_one (wsp* ',' version_one)* /// ``` fn parse_version_specifier(chars: &mut CharIter) -> Result, Pep508Error> { let mut start = chars.get_pos(); let mut specifiers = Vec::new(); let mut buffer = String::new(); let requirement_kind = loop { match chars.peek() { Some((end, ',')) => { let specifier = parse_specifier(chars, &buffer, start, end)?; specifiers.push(specifier); buffer.clear(); chars.next(); start = end + 1; } Some((_, ';')) | None => { let end = chars.get_pos(); let specifier = parse_specifier(chars, &buffer, start, end)?; specifiers.push(specifier); break Some(VersionOrUrl::VersionSpecifier( specifiers.into_iter().collect(), )); } Some((_, char)) => { buffer.push(char); chars.next(); } } }; Ok(requirement_kind) } /// Such as `(>=1.19,<2.0)` /// /// ```text /// '(' version_one (wsp* ',' version_one)* ')' /// ``` fn parse_version_specifier_parentheses( chars: &mut CharIter, ) -> Result, Pep508Error> { let brace_pos = chars.get_pos(); chars.next(); // Makes for slightly better error underline chars.eat_whitespace(); let mut start = chars.get_pos(); let mut specifiers = Vec::new(); let mut buffer = String::new(); let requirement_kind = loop { match chars.next() { Some((end, ',')) => { let specifier = parse_specifier(chars, &buffer, start, end)?; specifiers.push(specifier); buffer.clear(); start = end + 1; } Some((end, ')')) => { let specifier = parse_specifier(chars, &buffer, start, end)?; specifiers.push(specifier); break Some(VersionOrUrl::VersionSpecifier(specifiers.into_iter().collect())); }, Some((_, char)) => buffer.push(char), None => return Err(Pep508Error { message: Pep508ErrorSource::String("Missing closing parenthesis (expected ')', found end of dependency specification)".to_string()), start: brace_pos, len: 1, input: chars.copy_chars(), }), } }; Ok(requirement_kind) } /// Parse a [dependency specifier](https://packaging.python.org/en/latest/specifications/dependency-specifiers) fn parse(chars: &mut CharIter) -> Result { // Technically, the grammar is: // ```text // name_req = name wsp* extras? wsp* versionspec? wsp* quoted_marker? // url_req = name wsp* extras? wsp* urlspec wsp+ quoted_marker? // specification = wsp* ( url_req | name_req ) wsp* // ``` // So we can merge this into: // ```text // specification = wsp* name wsp* extras? wsp* (('@' wsp* url_req) | ('(' versionspec ')') | (versionspec)) wsp* (';' wsp* marker)? wsp* // ``` // Where the extras start with '[' if any, then we have '@', '(' or one of the version comparison // operators. Markers start with ';' if any // wsp* chars.eat_whitespace(); // name let name = parse_name(chars)?; // wsp* chars.eat_whitespace(); // extras? let extras = parse_extras(chars)?; // wsp* chars.eat_whitespace(); // ( url_req | name_req )? let requirement_kind = match chars.peek_char() { Some('@') => { chars.next(); Some(parse_url(chars)?) } Some('(') => parse_version_specifier_parentheses(chars)?, Some('<' | '=' | '>' | '~' | '!') => parse_version_specifier(chars)?, Some(';') | None => None, Some(other) => { return Err(Pep508Error { message: Pep508ErrorSource::String(format!( "Expected one of `@`, `(`, `<`, `=`, `>`, `~`, `!`, `;`, found `{}`", other )), start: chars.get_pos(), len: 1, input: chars.copy_chars(), }) } }; // wsp* chars.eat_whitespace(); // quoted_marker? let marker = if chars.peek_char() == Some(';') { // Skip past the semicolon chars.next(); Some(marker::parse_markers_impl(chars)?) } else { None }; // wsp* chars.eat_whitespace(); if let Some((pos, char)) = chars.next() { return Err(Pep508Error { message: Pep508ErrorSource::String(if marker.is_none() { format!(r#"Expected end of input or ';', found '{}'"#, char) } else { format!(r#"Expected end of input, found '{}'"#, char) }), start: pos, len: 1, input: chars.copy_chars(), }); } Ok(Requirement { name, extras, version_or_url: requirement_kind, marker, }) } /// A library for [dependency specifiers](https://packaging.python.org/en/latest/specifications/dependency-specifiers/) /// as originally specified in [PEP 508](https://peps.python.org/pep-0508/) /// /// This has `Version` and `VersionSpecifier` included. That is because /// `pep440_rs.Version("1.2.3") != pep508_rs.Requirement("numpy==1.2.3").version_or_url` as the /// `Version`s come from two different binaries and can therefore never be equal. #[cfg(feature = "pyo3")] #[pymodule] #[pyo3(name = "pep508_rs")] pub fn python_module(py: Python<'_>, m: &PyModule) -> PyResult<()> { // Allowed to fail if we embed this module in another #[allow(unused_must_use)] { pyo3_log::try_init(); } m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add("Pep508Error", py.get_type::())?; Ok(()) } /// Half of these tests are copied from https://github.com/pypa/packaging/pull/624 #[cfg(test)] mod tests { use crate::marker::{ parse_markers_impl, MarkerExpression, MarkerOperator, MarkerTree, MarkerValue, MarkerValueString, MarkerValueVersion, }; use crate::{CharIter, Requirement, VersionOrUrl}; use indoc::indoc; use pep440_rs::{Operator, Version, VersionSpecifier}; use std::str::FromStr; use url::Url; fn assert_err(input: &str, error: &str) { assert_eq!(Requirement::from_str(input).unwrap_err().to_string(), error); } #[test] fn error_empty() { assert_err( "", indoc! {" Empty field is not allowed for PEP508 ^" }, ); } #[test] fn error_start() { assert_err( "_name", indoc! {" Expected package name starting with an alphanumeric character, found '_' _name ^" }, ); } #[test] fn error_end() { assert_err( "name_", indoc! {" Package name must end with an alphanumeric character, not '_' name_ ^" }, ); } #[test] fn basic_examples() { let input = r#"requests[security,tests] >=2.8.1, ==2.8.* ; python_version < '2.7'"#; let requests = Requirement::from_str(input).unwrap(); assert_eq!(input, requests.to_string()); let expected = Requirement { name: "requests".to_string(), extras: Some(vec!["security".to_string(), "tests".to_string()]), version_or_url: Some(VersionOrUrl::VersionSpecifier( [ VersionSpecifier::new( Operator::GreaterThanEqual, Version { epoch: 0, release: vec![2, 8, 1], pre: None, post: None, dev: None, local: None, }, false, ) .unwrap(), VersionSpecifier::new( Operator::Equal, Version { epoch: 0, release: vec![2, 8], pre: None, post: None, dev: None, local: None, }, true, ) .unwrap(), ] .into_iter() .collect(), )), marker: Some(MarkerTree::Expression(MarkerExpression { l_value: MarkerValue::MarkerEnvVersion(MarkerValueVersion::PythonVersion), operator: MarkerOperator::LessThan, r_value: MarkerValue::QuotedString("2.7".to_string()), })), }; assert_eq!(requests, expected); } #[test] fn parenthesized_single() { let numpy = Requirement::from_str("numpy ( >=1.19 )").unwrap(); assert_eq!(numpy.name, "numpy"); } #[test] fn parenthesized_double() { let numpy = Requirement::from_str("numpy ( >=1.19, <2.0 )").unwrap(); assert_eq!(numpy.name, "numpy"); } #[test] fn versions_single() { let numpy = Requirement::from_str("numpy >=1.19 ").unwrap(); assert_eq!(numpy.name, "numpy"); } #[test] fn versions_double() { let numpy = Requirement::from_str("numpy >=1.19, <2.0 ").unwrap(); assert_eq!(numpy.name, "numpy"); } #[test] fn error_extras_eof1() { assert_err( "black[", indoc! {" Missing closing bracket (expected ']', found end of dependency specification) black[ ^" }, ); } #[test] fn error_extras_eof2() { assert_err( "black[d", indoc! {" Missing closing bracket (expected ']', found end of dependency specification) black[d ^" }, ); } #[test] fn error_extras_eof3() { assert_err( "black[d,", indoc! {" Missing closing bracket (expected ']', found end of dependency specification) black[d, ^" }, ); } #[test] fn error_extras_illegal_start1() { assert_err( "black[ö]", indoc! {" Expected an alphanumeric character starting the extra name, found 'ö' black[ö] ^" }, ); } #[test] fn error_extras_illegal_start2() { assert_err( "black[_d]", indoc! {" Expected an alphanumeric character starting the extra name, found '_' black[_d] ^" }, ); } #[test] fn error_extras_illegal_character() { assert_err( "black[jüpyter]", indoc! {" Invalid character in extras name, expected an alphanumeric character, '-', '_', '.', ',' or ']', found 'ü' black[jüpyter] ^" }, ); } #[test] fn error_extras1() { let numpy = Requirement::from_str("black[d]").unwrap(); assert_eq!(numpy.extras, Some(vec!["d".to_string()])); } #[test] fn error_extras2() { let numpy = Requirement::from_str("black[d,jupyter]").unwrap(); assert_eq!( numpy.extras, Some(vec!["d".to_string(), "jupyter".to_string()]) ); } #[test] fn error_parenthesized_pep440() { assert_err( "numpy ( ><1.19 )", indoc! {" Version specifier `><1.19 ` doesn't match PEP 440 rules numpy ( ><1.19 ) ^^^^^^^" }, ); } #[test] fn error_parenthesized_parenthesis() { assert_err( "numpy ( >=1.19 ", indoc! {" Missing closing parenthesis (expected ')', found end of dependency specification) numpy ( >=1.19 ^" }, ); } #[test] fn error_whats_that() { assert_err( "numpy % 1.16", indoc! {" Expected one of `@`, `(`, `<`, `=`, `>`, `~`, `!`, `;`, found `%` numpy % 1.16 ^" }, ); } #[test] fn url() { let pip_url = Requirement::from_str("pip @ https://github.com/pypa/pip/archive/1.3.1.zip#sha1=da9234ee9982d4bbb3c72346a6de940a148ea686") .unwrap(); let url = "https://github.com/pypa/pip/archive/1.3.1.zip#sha1=da9234ee9982d4bbb3c72346a6de940a148ea686"; let expected = Requirement { name: "pip".to_string(), extras: None, marker: None, version_or_url: Some(VersionOrUrl::Url(Url::parse(url).unwrap())), }; assert_eq!(pip_url, expected); } #[test] fn test_marker_parsing() { let marker = r#"python_version == "2.7" and (sys_platform == "win32" or (os_name == "linux" and implementation_name == 'cpython'))"#; let actual = parse_markers_impl(&mut CharIter::new(marker)).unwrap(); let expected = MarkerTree::And(vec![ MarkerTree::Expression(MarkerExpression { l_value: MarkerValue::MarkerEnvVersion(MarkerValueVersion::PythonVersion), operator: MarkerOperator::Equal, r_value: MarkerValue::QuotedString("2.7".to_string()), }), MarkerTree::Or(vec![ MarkerTree::Expression(MarkerExpression { l_value: MarkerValue::MarkerEnvString(MarkerValueString::SysPlatform), operator: MarkerOperator::Equal, r_value: MarkerValue::QuotedString("win32".to_string()), }), MarkerTree::And(vec![ MarkerTree::Expression(MarkerExpression { l_value: MarkerValue::MarkerEnvString(MarkerValueString::OsName), operator: MarkerOperator::Equal, r_value: MarkerValue::QuotedString("linux".to_string()), }), MarkerTree::Expression(MarkerExpression { l_value: MarkerValue::MarkerEnvString( MarkerValueString::ImplementationName, ), operator: MarkerOperator::Equal, r_value: MarkerValue::QuotedString("cpython".to_string()), }), ]), ]), ]); assert_eq!(expected, actual); } #[test] fn name_and_marker() { Requirement::from_str(r#"numpy; sys_platform == "win32" or (os_name == "linux" and implementation_name == 'cpython')"#).unwrap(); } #[test] fn error_marker_incomplete1() { assert_err( r#"numpy; sys_platform"#, indoc! {" Expected a valid marker operator (such as '>=' or 'not in'), found '' numpy; sys_platform ^" }, ); } #[test] fn error_marker_incomplete2() { assert_err( r#"numpy; sys_platform == "#, indoc! {" Expected marker value, found end of dependency specification numpy; sys_platform == ^" }, ); } #[test] fn error_marker_incomplete3() { assert_err( r#"numpy; sys_platform == "win32" or "#, indoc! {r#" Expected marker value, found end of dependency specification numpy; sys_platform == "win32" or ^"#}, ); } #[test] fn error_marker_incomplete4() { assert_err( r#"numpy; sys_platform == "win32" or (os_name == "linux""#, indoc! {r#" Expected ')', found end of dependency specification numpy; sys_platform == "win32" or (os_name == "linux" ^"#}, ); } #[test] fn error_marker_incomplete5() { assert_err( r#"numpy; sys_platform == "win32" or (os_name == "linux" and "#, indoc! {r#" Expected marker value, found end of dependency specification numpy; sys_platform == "win32" or (os_name == "linux" and ^"#}, ); } #[test] fn error_pep440() { assert_err( r#"numpy >=1.1.*"#, indoc! {" Operator >= must not be used in version ending with a star numpy >=1.1.* ^^^^^^^" }, ); } #[test] fn error_no_name() { assert_err( r#"==0.0"#, indoc! {" Expected package name starting with an alphanumeric character, found '=' ==0.0 ^" }, ); } #[test] fn error_no_comma_between_extras() { assert_err( r#"name[bar baz]"#, indoc! {" Expected either ',' (separating extras) or ']' (ending the extras section), found 'b' name[bar baz] ^" }, ); } #[test] fn error_extra_comma_after_extras() { assert_err( r#"name[bar, baz,]"#, indoc! {" Expected an alphanumeric character starting the extra name, found ']' name[bar, baz,] ^" }, ); } #[test] fn error_extras_not_closed() { assert_err( r#"name[bar, baz >= 1.0"#, indoc! {" Expected either ',' (separating extras) or ']' (ending the extras section), found '>' name[bar, baz >= 1.0 ^" }, ); } #[test] fn error_no_space_after_url() { assert_err( r#"name @ https://example.com/; extra == 'example'"#, indoc! {" Expected end of input or ';', found 'e' name @ https://example.com/; extra == 'example' ^" }, ); } #[test] fn error_name_at_nothing() { assert_err( r#"name @ "#, indoc! {" Expected URL name @ ^" }, ); } #[test] fn test_error_invalid_marker_key() { assert_err( r#"name; invalid_name"#, indoc! {" Expected a valid marker name, found 'invalid_name' name; invalid_name ^^^^^^^^^^^^" }, ); } #[test] fn error_markers_invalid_order() { assert_err( "name; '3.7' <= invalid_name", indoc! {" Expected a valid marker name, found 'invalid_name' name; '3.7' <= invalid_name ^^^^^^^^^^^^" }, ); } #[test] fn error_markers_notin() { assert_err( "name; '3.7' notin python_version", indoc! {" Expected a valid marker operator (such as '>=' or 'not in'), found 'notin' name; '3.7' notin python_version ^^^^^" }, ); } #[test] fn error_markers_inpython_version() { assert_err( "name; '3.6'inpython_version", indoc! {" Expected a valid marker operator (such as '>=' or 'not in'), found 'inpython_version' name; '3.6'inpython_version ^^^^^^^^^^^^^^^^" }, ); } #[test] fn error_markers_not_python_version() { assert_err( "name; '3.7' not python_version", indoc! {" Expected 'i', found 'p' name; '3.7' not python_version ^" }, ); } #[test] fn error_markers_invalid_operator() { assert_err( "name; '3.7' ~ python_version", indoc! {" Expected a valid marker operator (such as '>=' or 'not in'), found '~' name; '3.7' ~ python_version ^" }, ); } #[test] fn error_invalid_prerelease() { assert_err( "name==1.0.org1", indoc! {" Version specifier `==1.0.org1` doesn't match PEP 440 rules name==1.0.org1 ^^^^^^^^^^" }, ); } #[test] fn error_no_version_value() { assert_err( "name==", indoc! {" Version specifier `==` doesn't match PEP 440 rules name== ^^" }, ); } #[test] fn error_no_version_operator() { assert_err( "name 1.0", indoc! {" Expected one of `@`, `(`, `<`, `=`, `>`, `~`, `!`, `;`, found `1` name 1.0 ^" }, ); } #[test] fn error_random_char() { assert_err( "name >= 1.0 #", indoc! {" Version specifier `>= 1.0 #` doesn't match PEP 440 rules name >= 1.0 # ^^^^^^^^" }, ); } } pep508_rs-0.2.3/src/marker.rs000064400000000000000000001751201046102023000137770ustar 00000000000000//! PEP 508 markers implementations with validation and warnings //! //! Markers allow you to install dependencies only in specific environments (python version, //! operating system, architecture, etc.) or when a specific feature is activated. E.g. you can //! say `importlib-metadata ; python_version < "3.8"` or //! `itsdangerous (>=1.1.0) ; extra == 'security'`. Unfortunately, the marker grammar has some //! oversights (e.g. ) and //! the design of comparisons (PEP 440 comparisons with lexicographic fallback) leads to confusing //! outcomes. This implementation tries to carefully validate everything and emit warnings whenever //! bogus comparisons with unintended semantics are made. use crate::{CharIter, Pep508Error, Pep508ErrorSource}; use pep440_rs::{Version, VersionSpecifier}; #[cfg(feature = "pyo3")] use pyo3::{ basic::CompareOp, exceptions::PyValueError, pyclass, pymethods, PyAny, PyResult, Python, }; #[cfg(feature = "serde")] use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; use std::collections::HashSet; use std::fmt::{Display, Formatter}; use std::ops::Deref; use std::str::FromStr; use tracing::warn; /// Ways in which marker evaluation can fail #[cfg_attr(feature = "pyo3", pyclass(module = "pep508"))] #[derive(Debug, Eq, Hash, Ord, PartialOrd, PartialEq, Clone, Copy)] pub enum MarkerWarningKind { /// Using an old name from PEP 345 instead of the modern equivalent /// DeprecatedMarkerName, /// Doing an operation other than `==` and `!=` on a quoted string with `extra`, such as /// `extra > "perf"` or `extra == os_name` ExtraInvalidComparison, /// Comparing a string valued marker and a string lexicographically, such as `"3.9" > "3.10"` LexicographicComparison, /// Comparing two markers, such as `os_name != sys_implementation` MarkerMarkerComparison, /// Failed to parse a PEP 440 version or version specifier, e.g. `>=1<2` Pep440Error, /// Comparing two strings, such as `"3.9" > "3.10"` StringStringComparison, } #[cfg(feature = "pyo3")] #[pymethods] impl MarkerWarningKind { fn __hash__(&self) -> u8 { *self as u8 } fn __richcmp__(&self, other: &Self, op: CompareOp) -> bool { op.matches(self.cmp(other)) } } /// Those environment markers with a PEP 440 version as value such as `python_version` #[derive(Clone, Debug, Eq, Hash, PartialEq)] #[allow(clippy::enum_variant_names)] pub enum MarkerValueVersion { /// `implementation_version` ImplementationVersion, /// `python_full_version` PythonFullVersion, /// `python_version` PythonVersion, } impl Display for MarkerValueVersion { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { Self::ImplementationVersion => f.write_str("implementation_version"), Self::PythonFullVersion => f.write_str("python_full_version"), Self::PythonVersion => f.write_str("python_version"), } } } /// Those environment markers with an arbitrary string as value such as `sys_platform` #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub enum MarkerValueString { /// `implementation_name` ImplementationName, /// `os_name` OsName, /// /// Deprecated `os.name` from https://peps.python.org/pep-0345/#environment-markers OsNameDeprecated, /// `platform_machine` PlatformMachine, /// /// Deprecated `platform.machine` from https://peps.python.org/pep-0345/#environment-markers PlatformMachineDeprecated, /// `platform_python_implementation` PlatformPythonImplementation, /// /// Deprecated `platform.python_implementation` from https://peps.python.org/pep-0345/#environment-markers PlatformPythonImplementationDeprecated, /// `platform_release` PlatformRelease, /// `platform_system` PlatformSystem, /// `platform_version` PlatformVersion, /// /// Deprecated `platform.version` from https://peps.python.org/pep-0345/#environment-markers PlatformVersionDeprecated, /// `sys_platform` SysPlatform, /// /// Deprecated `sys.platform` from https://peps.python.org/pep-0345/#environment-markers SysPlatformDeprecated, } impl Display for MarkerValueString { /// Normalizes deprecated names to the proper ones fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { Self::ImplementationName => f.write_str("implementation_name"), Self::OsName | Self::OsNameDeprecated => f.write_str("os_name"), Self::PlatformMachine | Self::PlatformMachineDeprecated => { f.write_str("platform_machine") } Self::PlatformPythonImplementation | Self::PlatformPythonImplementationDeprecated => { f.write_str("platform_python_implementation") } Self::PlatformRelease => f.write_str("platform_release"), Self::PlatformSystem => f.write_str("platform_system"), Self::PlatformVersion | Self::PlatformVersionDeprecated => { f.write_str("platform_version") } Self::SysPlatform | Self::SysPlatformDeprecated => f.write_str("sys_platform"), } } } /// One of the predefined environment values /// /// #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub enum MarkerValue { /// Those environment markers with a PEP 440 version as value such as `python_version` MarkerEnvVersion(MarkerValueVersion), /// Those environment markers with an arbitrary string as value such as `sys_platform` MarkerEnvString(MarkerValueString), /// `extra`. This one is special because it's a list and not env but user given Extra, /// Not a constant, but a user given quoted string with a value inside such as '3.8' or "windows" QuotedString(String), } impl MarkerValue { fn string_value(value: String) -> Self { Self::QuotedString(value) } } impl FromStr for MarkerValue { type Err = String; /// This is specifically for the reserved values fn from_str(s: &str) -> Result { let value = match s { "implementation_name" => Self::MarkerEnvString(MarkerValueString::ImplementationName), "implementation_version" => { Self::MarkerEnvVersion(MarkerValueVersion::ImplementationVersion) } "os_name" => Self::MarkerEnvString(MarkerValueString::OsName), "os.name" => Self::MarkerEnvString(MarkerValueString::OsNameDeprecated), "platform_machine" => Self::MarkerEnvString(MarkerValueString::PlatformMachine), "platform.machine" => { Self::MarkerEnvString(MarkerValueString::PlatformMachineDeprecated) } "platform_python_implementation" => { Self::MarkerEnvString(MarkerValueString::PlatformPythonImplementation) } "platform.python_implementation" => { Self::MarkerEnvString(MarkerValueString::PlatformPythonImplementationDeprecated) } "platform_release" => Self::MarkerEnvString(MarkerValueString::PlatformRelease), "platform_system" => Self::MarkerEnvString(MarkerValueString::PlatformSystem), "platform_version" => Self::MarkerEnvString(MarkerValueString::PlatformVersion), "platform.version" => { Self::MarkerEnvString(MarkerValueString::PlatformVersionDeprecated) } "python_full_version" => Self::MarkerEnvVersion(MarkerValueVersion::PythonFullVersion), "python_version" => Self::MarkerEnvVersion(MarkerValueVersion::PythonVersion), "sys_platform" => Self::MarkerEnvString(MarkerValueString::SysPlatform), "sys.platform" => Self::MarkerEnvString(MarkerValueString::SysPlatformDeprecated), "extra" => Self::Extra, _ => return Err(format!("Invalid key: {}", s)), }; Ok(value) } } impl Display for MarkerValue { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { Self::MarkerEnvVersion(marker_value_version) => marker_value_version.fmt(f), Self::MarkerEnvString(marker_value_string) => marker_value_string.fmt(f), Self::Extra => f.write_str("extra"), Self::QuotedString(value) => write!(f, "'{}'", value), } } } /// How to compare key and value, such as by `==`, `>` or `not in` #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub enum MarkerOperator { /// `==` Equal, /// `!=` NotEqual, /// `>` GreaterThan, /// `>=` GreaterEqual, /// `<` LessThan, /// `<=` LessEqual, /// `~=` TildeEqual, /// `in` In, /// `not in` NotIn, } impl MarkerOperator { /// Compare two versions, returning None for `in` and `not in` fn to_pep440_operator(&self) -> Option { match self { MarkerOperator::Equal => Some(pep440_rs::Operator::Equal), MarkerOperator::NotEqual => Some(pep440_rs::Operator::NotEqual), MarkerOperator::GreaterThan => Some(pep440_rs::Operator::GreaterThan), MarkerOperator::GreaterEqual => Some(pep440_rs::Operator::GreaterThanEqual), MarkerOperator::LessThan => Some(pep440_rs::Operator::LessThan), MarkerOperator::LessEqual => Some(pep440_rs::Operator::LessThanEqual), MarkerOperator::TildeEqual => Some(pep440_rs::Operator::TildeEqual), MarkerOperator::In => None, MarkerOperator::NotIn => None, } } } impl FromStr for MarkerOperator { type Err = String; /// PEP 508 allows arbitrary whitespace between "not" and "in", and so do we fn from_str(s: &str) -> Result { let value = match s { "==" => Self::Equal, "!=" => Self::NotEqual, ">" => Self::GreaterThan, ">=" => Self::GreaterEqual, "<" => Self::LessThan, "<=" => Self::LessEqual, "~=" => Self::TildeEqual, "in" => Self::In, not_space_in if not_space_in // start with not .strip_prefix("not") // ends with in .and_then(|space_in| space_in.strip_suffix("in")) // and has only whitespace in between .map(|space| !space.is_empty() && space.trim().is_empty()) .unwrap_or_default() => { Self::NotIn } other => return Err(format!("Invalid comparator: {}", other)), }; Ok(value) } } impl Display for MarkerOperator { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.write_str(match self { Self::Equal => "==", Self::NotEqual => "!=", Self::GreaterThan => ">", Self::GreaterEqual => ">=", Self::LessThan => "<", Self::LessEqual => "<=", Self::TildeEqual => "~=", Self::In => "in", Self::NotIn => "not in", }) } } /// Helper type with a [Version] and its original text #[cfg_attr(feature = "pyo3", pyclass(get_all, module = "pep508"))] #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub struct StringVersion { /// Original unchanged string pub string: String, /// Parsed version pub version: Version, } impl FromStr for StringVersion { type Err = String; fn from_str(s: &str) -> Result { Ok(Self { string: s.to_string(), version: Version::from_str(s)?, }) } } #[cfg(feature = "serde")] impl Serialize for StringVersion { fn serialize(&self, serializer: S) -> Result where S: Serializer, { serializer.serialize_str(&self.string) } } #[cfg(feature = "serde")] impl<'de> Deserialize<'de> for StringVersion { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { let string = String::deserialize(deserializer)?; Self::from_str(&string).map_err(de::Error::custom) } } impl Deref for StringVersion { type Target = Version; fn deref(&self) -> &Self::Target { &self.version } } /// The marker values for a python interpreter, normally the current one /// /// /// /// Some are `(String, Version)` because we have to support version comparison #[allow(missing_docs)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] #[cfg_attr(feature = "pyo3", pyclass(get_all, module = "pep508"))] #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub struct MarkerEnvironment { pub implementation_name: String, pub implementation_version: StringVersion, pub os_name: String, pub platform_machine: String, pub platform_python_implementation: String, pub platform_release: String, pub platform_system: String, pub platform_version: String, pub python_full_version: StringVersion, pub python_version: StringVersion, pub sys_platform: String, } impl MarkerEnvironment { /// Returns of the PEP 440 version typed value of the key in the current environment fn get_version(&self, key: &MarkerValueVersion) -> &Version { match key { MarkerValueVersion::ImplementationVersion => &self.implementation_version.version, MarkerValueVersion::PythonFullVersion => &self.python_full_version.version, MarkerValueVersion::PythonVersion => &self.python_version.version, } } /// Returns of the stringly typed value of the key in the current environment fn get_string(&self, key: &MarkerValueString) -> &str { match key { MarkerValueString::ImplementationName => &self.implementation_name, MarkerValueString::OsName | MarkerValueString::OsNameDeprecated => &self.os_name, MarkerValueString::PlatformMachine | MarkerValueString::PlatformMachineDeprecated => { &self.platform_machine } MarkerValueString::PlatformPythonImplementation | MarkerValueString::PlatformPythonImplementationDeprecated => { &self.platform_python_implementation } MarkerValueString::PlatformRelease => &self.platform_release, MarkerValueString::PlatformSystem => &self.platform_system, MarkerValueString::PlatformVersion | MarkerValueString::PlatformVersionDeprecated => { &self.platform_version } MarkerValueString::SysPlatform | MarkerValueString::SysPlatformDeprecated => { &self.sys_platform } } } } #[cfg(feature = "pyo3")] #[pymethods] impl MarkerEnvironment { /// Construct your own marker environment #[new] #[pyo3(signature = (*, implementation_name, implementation_version, os_name, platform_machine, platform_python_implementation, platform_release, platform_system, platform_version, python_full_version, python_version, sys_platform ))] #[allow(clippy::too_many_arguments)] fn py_new( implementation_name: &str, implementation_version: &str, os_name: &str, platform_machine: &str, platform_python_implementation: &str, platform_release: &str, platform_system: &str, platform_version: &str, python_full_version: &str, python_version: &str, sys_platform: &str, ) -> PyResult { let implementation_version = StringVersion::from_str(implementation_version).map_err(|err| { PyValueError::new_err(format!( "implementation_version is not a valid PEP440 version: {}", err )) })?; let python_full_version = StringVersion::from_str(python_full_version).map_err(|err| { PyValueError::new_err(format!( "python_full_version is not a valid PEP440 version: {}", err )) })?; let python_version = StringVersion::from_str(python_version).map_err(|err| { PyValueError::new_err(format!( "python_version is not a valid PEP440 version: {}", err )) })?; Ok(Self { implementation_name: implementation_name.to_string(), implementation_version, os_name: os_name.to_string(), platform_machine: platform_machine.to_string(), platform_python_implementation: platform_python_implementation.to_string(), platform_release: platform_release.to_string(), platform_system: platform_system.to_string(), platform_version: platform_version.to_string(), python_full_version, python_version, sys_platform: sys_platform.to_string(), }) } /// Query the current python interpreter to get the correct marker value #[staticmethod] fn current(py: Python<'_>) -> PyResult { let os = py.import("os")?; let platform = py.import("platform")?; let sys = py.import("sys")?; let python_version_tuple: (String, String, String) = platform .getattr("python_version_tuple")? .call0()? .extract()?; // See pseudocode at // https://packaging.python.org/en/latest/specifications/dependency-specifiers/#environment-markers let name = sys.getattr("implementation")?.getattr("name")?.extract()?; let info: &PyAny = sys.getattr("implementation")?.getattr("version")?; let kind = info.getattr("releaselevel")?.extract::()?; let implementation_version: String = format!( "{}.{}.{}{}", info.getattr("major")?.extract::()?, info.getattr("minor")?.extract::()?, info.getattr("micro")?.extract::()?, if kind != "final" { format!("{}{}", kind, info.getattr("serial")?.extract::()?) } else { "".to_string() } ); let python_full_version: String = platform.getattr("python_version")?.call0()?.extract()?; let python_version = format!("{}.{}", python_version_tuple.0, python_version_tuple.1); // This is not written down in PEP 508, but it's the only reasonable assumption to make let implementation_version = StringVersion::from_str(&implementation_version).map_err(|err| { PyValueError::new_err(format!( "Broken python implementation, implementation_version is not a valid PEP440 version: {}", err )) })?; let python_full_version = StringVersion::from_str(&python_full_version).map_err(|err| { PyValueError::new_err(format!( "Broken python implementation, python_full_version is not a valid PEP440 version: {}", err )) })?; let python_version = StringVersion::from_str(&python_version).map_err(|err| { PyValueError::new_err(format!( "Broken python implementation, python_version is not a valid PEP440 version: {}", err )) })?; Ok(Self { implementation_name: name, implementation_version, os_name: os.getattr("name")?.extract()?, platform_machine: platform.getattr("machine")?.call0()?.extract()?, platform_python_implementation: platform .getattr("python_implementation")? .call0()? .extract()?, platform_release: platform.getattr("release")?.call0()?.extract()?, platform_system: platform.getattr("system")?.call0()?.extract()?, platform_version: platform.getattr("version")?.call0()?.extract()?, python_full_version, python_version, sys_platform: sys.getattr("platform")?.extract()?, }) } } /// Represents one clause such as `python_version > "3.8"` in the form /// ```text /// /// ``` #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub struct MarkerExpression { /// A name from the PEP508 list or a string pub l_value: MarkerValue, /// an operator, such as `>=` or `not in` pub operator: MarkerOperator, /// A name from the PEP508 list or a string pub r_value: MarkerValue, } impl MarkerExpression { /// Evaluate a expression fn evaluate( &self, env: &MarkerEnvironment, extras: &[&str], reporter: &mut impl FnMut(MarkerWarningKind, String, &MarkerExpression), ) -> bool { match &self.l_value { // The only sound choice for this is ` ` MarkerValue::MarkerEnvVersion(l_key) => { let value = &self.r_value; let (r_version, r_star) = if let MarkerValue::QuotedString(r_string) = &value { match Version::from_str_star(r_string) { Ok((version, star)) => (version, star), Err(err) => { reporter(MarkerWarningKind::Pep440Error, format!( "Expected PEP 440 version to compare with {}, found {}, evaluating to false: {}", l_key, self.r_value, err ), self); return false; } } } else { reporter(MarkerWarningKind::Pep440Error, format!( "Expected double quoted PEP 440 version to compare with {}, found {}, evaluating to false", l_key, self.r_value ), self); return false; }; let operator = match self.operator.to_pep440_operator() { None => { reporter(MarkerWarningKind::Pep440Error, format!( "Expected PEP 440 version operator to compare {} with '{}', found '{}', evaluating to false", l_key, r_version, self.operator ), self); return false; } Some(operator) => operator, }; let specifier = match VersionSpecifier::new(operator, r_version, r_star) { Ok(specifier) => specifier, Err(err) => { reporter( MarkerWarningKind::Pep440Error, format!("Invalid operator/version combination: {}", err), self, ); return false; } }; let l_version = env.get_version(l_key); specifier.contains(l_version) } // This is half the same block as above inverted MarkerValue::MarkerEnvString(l_key) => { let r_string = match &self.r_value { MarkerValue::Extra | MarkerValue::MarkerEnvVersion(_) | MarkerValue::MarkerEnvString(_) => { reporter(MarkerWarningKind::MarkerMarkerComparison, "Comparing two markers with each other doesn't make any sense, evaluating to false".to_string(), self); return false; } MarkerValue::QuotedString(r_string) => r_string, }; let l_string = env.get_string(l_key); self.compare_strings(l_string, r_string, reporter) } // `extra == '...'` MarkerValue::Extra => { let r_value_string = match &self.r_value { MarkerValue::MarkerEnvVersion(_) | MarkerValue::MarkerEnvString(_) | MarkerValue::Extra => { reporter(MarkerWarningKind::ExtraInvalidComparison, "Comparing extra with something other than a quoted string is wrong, evaluating to false".to_string(), self); return false; } MarkerValue::QuotedString(r_value_string) => r_value_string, }; self.marker_compare(r_value_string, extras, reporter) } // This is either MarkerEnvVersion, MarkerEnvString or Extra inverted MarkerValue::QuotedString(l_string) => { match &self.r_value { // The only sound choice for this is ` ` MarkerValue::MarkerEnvVersion(r_key) => { let l_version = match Version::from_str(l_string) { Ok(l_version) => l_version, Err(err) => { reporter(MarkerWarningKind::Pep440Error, format!( "Expected double quoted PEP 440 version to compare with {}, found {}, evaluating to false: {}", l_string, self.r_value, err ), self); return false; } }; let r_version = env.get_version(r_key); let operator = match self.operator.to_pep440_operator() { None => { reporter(MarkerWarningKind::Pep440Error, format!( "Expected PEP 440 version operator to compare '{}' with {}, found '{}', evaluating to false", l_string, r_key, self.operator ), self); return false; } Some(operator) => operator, }; let specifier = match VersionSpecifier::new(operator, r_version.clone(), false) { Ok(specifier) => specifier, Err(err) => { reporter( MarkerWarningKind::Pep440Error, format!("Invalid operator/version combination: {}", err), self, ); return false; } }; specifier.contains(&l_version) } // This is half the same block as above inverted MarkerValue::MarkerEnvString(r_key) => { let r_string = env.get_string(r_key); self.compare_strings(l_string, r_string, reporter) } // `'...' == extra` MarkerValue::Extra => self.marker_compare(l_string, extras, reporter), // `'...' == '...'`, doesn't make much sense MarkerValue::QuotedString(_) => { // Not even pypa/packaging 22.0 supports this // https://github.com/pypa/packaging/issues/632 reporter(MarkerWarningKind::StringStringComparison, format!( "Comparing two quoted strings with each other doesn't make sense: {}, evaluating to false", self ), self); false } } } } } /// Evaluates only the extras and python version part of the markers. We use this during /// dependency resolution when we want to have packages for all possible environments but /// already know the extras and the possible python versions (from `requires-python`) /// /// This considers only expression in the from `extra == '...'`, `'...' == extra`, /// `python_version '...'` and /// `'...' python_version`. /// /// Note that unlike [Self::evaluate] this does not perform any checks for bogus expressions but /// will simply return true. /// /// ```rust /// # use std::collections::HashSet; /// # use std::str::FromStr; /// # use pep508_rs::{MarkerTree, Pep508Error}; /// # use pep440_rs::Version; /// /// # fn main() -> Result<(), Pep508Error> { /// let marker_tree = MarkerTree::from_str(r#"("linux" in sys_platform) and extra == 'day'"#)?; /// let versions: Vec = (8..12).map(|minor| Version::from_release(vec![3, minor])).collect(); /// assert!(marker_tree.evaluate_extras_and_python_version(&["day".to_string()].into(), &versions)); /// assert!(!marker_tree.evaluate_extras_and_python_version(&["night".to_string()].into(), &versions)); /// /// let marker_tree = MarkerTree::from_str(r#"extra == 'day' and python_version < '3.11' and '3.10' <= python_version"#)?; /// assert!(!marker_tree.evaluate_extras_and_python_version(&["day".to_string()].into(), &vec![Version::from_release(vec![3, 9])])); /// assert!(marker_tree.evaluate_extras_and_python_version(&["day".to_string()].into(), &vec![Version::from_release(vec![3, 10])])); /// assert!(!marker_tree.evaluate_extras_and_python_version(&["day".to_string()].into(), &vec![Version::from_release(vec![3, 11])])); /// # Ok(()) /// # } /// ``` fn evaluate_extras_and_python_version( &self, extras: &HashSet, python_versions: &[Version], ) -> bool { match (&self.l_value, &self.operator, &self.r_value) { // `extra == '...'` (MarkerValue::Extra, MarkerOperator::Equal, MarkerValue::QuotedString(r_string)) => { extras.contains(r_string) } // `'...' == extra` (MarkerValue::QuotedString(l_string), MarkerOperator::Equal, MarkerValue::Extra) => { extras.contains(l_string) } // `extra != '...'` (MarkerValue::Extra, MarkerOperator::NotEqual, MarkerValue::QuotedString(r_string)) => { !extras.contains(r_string) } // `'...' != extra` (MarkerValue::QuotedString(l_string), MarkerOperator::NotEqual, MarkerValue::Extra) => { !extras.contains(l_string) } ( MarkerValue::MarkerEnvVersion(MarkerValueVersion::PythonVersion), operator, MarkerValue::QuotedString(r_string), ) => { // ignore all errors block (|| { // The right hand side is allowed to contain a star, e.g. `python_version == '3.*'` let (r_version, r_star) = Version::from_str_star(r_string).ok()?; let operator = operator.to_pep440_operator()?; // operator and right hand side make the specifier let specifier = VersionSpecifier::new(operator, r_version, r_star).ok()?; let compatible = python_versions .iter() .any(|l_version| specifier.contains(l_version)); Some(compatible) })() .unwrap_or(true) } ( MarkerValue::QuotedString(l_string), operator, MarkerValue::MarkerEnvVersion(MarkerValueVersion::PythonVersion), ) => { // ignore all errors block (|| { // Not star allowed here, `'3.*' == python_version` is not a valid PEP 440 // comparison let l_version = Version::from_str(l_string).ok()?; let operator = operator.to_pep440_operator()?; let compatible = python_versions.iter().any(|r_version| { // operator and right hand side make the specifier and in this case the // right hand is `python_version` so changes every iteration match VersionSpecifier::new(operator.clone(), r_version.clone(), false) { Ok(specifier) => specifier.contains(&l_version), Err(_) => true, } }); Some(compatible) })() .unwrap_or(true) } _ => true, } } /// Compare strings by PEP 508 logic, with warnings fn compare_strings( &self, l_string: &str, r_string: &str, reporter: &mut impl FnMut(MarkerWarningKind, String, &MarkerExpression), ) -> bool { match self.operator { MarkerOperator::Equal => l_string == r_string, MarkerOperator::NotEqual => l_string != r_string, MarkerOperator::GreaterThan => { reporter( MarkerWarningKind::LexicographicComparison, format!("Comparing {} and {} lexicographically", l_string, r_string), self, ); l_string > r_string } MarkerOperator::GreaterEqual => { reporter( MarkerWarningKind::LexicographicComparison, format!("Comparing {} and {} lexicographically", l_string, r_string), self, ); l_string >= r_string } MarkerOperator::LessThan => { reporter( MarkerWarningKind::LexicographicComparison, format!("Comparing {} and {} lexicographically", l_string, r_string), self, ); l_string < r_string } MarkerOperator::LessEqual => { reporter( MarkerWarningKind::LexicographicComparison, format!("Comparing {} and {} lexicographically", l_string, r_string), self, ); l_string <= r_string } MarkerOperator::TildeEqual => { reporter( MarkerWarningKind::LexicographicComparison, format!("Can't compare {} and {} with `~=`", l_string, r_string), self, ); false } MarkerOperator::In => r_string.contains(l_string), MarkerOperator::NotIn => !r_string.contains(l_string), } } // The `marker '...'` comparison fn marker_compare( &self, value: &str, extras: &[&str], reporter: &mut impl FnMut(MarkerWarningKind, String, &MarkerExpression), ) -> bool { match self.operator { // TODO: normalize extras MarkerOperator::Equal => extras.contains(&value), MarkerOperator::NotEqual => !extras.contains(&value), _ => { reporter(MarkerWarningKind::ExtraInvalidComparison, "Comparing extra with something other than equal (`==`) or unequal (`!=`) is wrong, evaluating to false".to_string(), self); false } } } } impl FromStr for MarkerExpression { type Err = Pep508Error; fn from_str(s: &str) -> Result { let mut chars = CharIter::new(s); let expression = parse_marker_key_op_value(&mut chars)?; chars.eat_whitespace(); if let Some((pos, unexpected)) = chars.next() { return Err(Pep508Error { message: Pep508ErrorSource::String(format!( "Unexpected character '{}', expected end of input", unexpected )), start: pos, len: chars.chars.clone().count(), input: chars.copy_chars(), }); } Ok(expression) } } impl Display for MarkerExpression { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{} {} {}", self.l_value, self.operator, self.r_value) } } /// Represents one of the nested marker expressions with and/or/parentheses #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub enum MarkerTree { /// A simple expression such as `python_version > "3.8"` Expression(MarkerExpression), /// An and between nested expressions, such as /// `python_version > "3.8" and implementation_name == 'cpython'` And(Vec), /// An or between nested expressions, such as /// `python_version > "3.8" or implementation_name == 'cpython'` Or(Vec), } impl FromStr for MarkerTree { type Err = Pep508Error; fn from_str(markers: &str) -> Result { parse_markers(markers) } } impl MarkerTree { /// Does this marker apply in the given environment? pub fn evaluate(&self, env: &MarkerEnvironment, extras: &[&str]) -> bool { let mut reporter = |_kind, message, _marker_expression: &MarkerExpression| { warn!("{}", message); }; self.report_deprecated_options(&mut reporter); match self { MarkerTree::Expression(expression) => expression.evaluate(env, extras, &mut reporter), MarkerTree::And(expressions) => expressions .iter() .all(|x| x.evaluate_reporter_impl(env, extras, &mut reporter)), MarkerTree::Or(expressions) => expressions .iter() .any(|x| x.evaluate_reporter_impl(env, extras, &mut reporter)), } } /// Same as [Self::evaluate], but instead of using logging to warn, you can pass your own /// handler for warnings pub fn evaluate_reporter( &self, env: &MarkerEnvironment, extras: &[&str], reporter: &mut impl FnMut(MarkerWarningKind, String, &MarkerExpression), ) -> bool { self.report_deprecated_options(reporter); self.evaluate_reporter_impl(env, extras, reporter) } fn evaluate_reporter_impl( &self, env: &MarkerEnvironment, extras: &[&str], reporter: &mut impl FnMut(MarkerWarningKind, String, &MarkerExpression), ) -> bool { match self { MarkerTree::Expression(expression) => expression.evaluate(env, extras, reporter), MarkerTree::And(expressions) => expressions .iter() .all(|x| x.evaluate_reporter_impl(env, extras, reporter)), MarkerTree::Or(expressions) => expressions .iter() .any(|x| x.evaluate_reporter_impl(env, extras, reporter)), } } /// Checks if the requirement should be activated with the given set of active extras and a set /// of possible python versions (from `requires-python`) without evaluating the remaining /// environment markers, i.e. if there is potentially an environment that could activate this /// requirement. /// /// Note that unlike [Self::evaluate] this does not perform any checks for bogus expressions but /// will simply return true. As caller you should separately perform a check with an environment /// and forward all warnings. pub fn evaluate_extras_and_python_version( &self, extras: &HashSet, python_versions: &[Version], ) -> bool { match self { MarkerTree::Expression(expression) => { expression.evaluate_extras_and_python_version(extras, python_versions) } MarkerTree::And(expressions) => expressions .iter() .all(|x| x.evaluate_extras_and_python_version(extras, python_versions)), MarkerTree::Or(expressions) => expressions .iter() .any(|x| x.evaluate_extras_and_python_version(extras, python_versions)), } } /// Same as [Self::evaluate], but instead of using logging to warn, you get a Vec with all /// warnings collected pub fn evaluate_collect_warnings( &self, env: &MarkerEnvironment, extras: &[&str], ) -> (bool, Vec<(MarkerWarningKind, String, String)>) { let mut warnings = Vec::new(); let mut reporter = |kind, warning, marker: &MarkerExpression| { warnings.push((kind, warning, marker.to_string())) }; self.report_deprecated_options(&mut reporter); let result = self.evaluate_reporter_impl(env, extras, &mut reporter); (result, warnings) } /// Report the deprecated marker from fn report_deprecated_options( &self, reporter: &mut impl FnMut(MarkerWarningKind, String, &MarkerExpression), ) { match self { MarkerTree::Expression(expression) => { for value in [&expression.l_value, &expression.r_value] { match value { MarkerValue::MarkerEnvString(MarkerValueString::OsNameDeprecated) => { reporter( MarkerWarningKind::DeprecatedMarkerName, "os.name is deprecated in favor of os_name".to_string(), expression, ); } MarkerValue::MarkerEnvString( MarkerValueString::PlatformMachineDeprecated, ) => { reporter( MarkerWarningKind::DeprecatedMarkerName, "platform.machine is deprecated in favor of platform_machine" .to_string(), expression, ); } MarkerValue::MarkerEnvString( MarkerValueString::PlatformPythonImplementationDeprecated, ) => { reporter( MarkerWarningKind::DeprecatedMarkerName, "platform.python_implementation is deprecated in favor of platform_python_implementation".to_string(), expression, ); } MarkerValue::MarkerEnvString( MarkerValueString::PlatformVersionDeprecated, ) => { reporter( MarkerWarningKind::DeprecatedMarkerName, "platform.version is deprecated in favor of platform_version" .to_string(), expression, ); } MarkerValue::MarkerEnvString(MarkerValueString::SysPlatformDeprecated) => { reporter( MarkerWarningKind::DeprecatedMarkerName, "sys.platform is deprecated in favor of sys_platform".to_string(), expression, ); } _ => {} } } } MarkerTree::And(expressions) => { for expression in expressions { expression.report_deprecated_options(reporter) } } MarkerTree::Or(expressions) => { for expression in expressions { expression.report_deprecated_options(reporter) } } } } } impl Display for MarkerTree { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { let format_inner = |expression: &MarkerTree| { if matches!(expression, MarkerTree::Expression(_)) { format!("{}", expression) } else { format!("({})", expression) } }; match self { MarkerTree::Expression(expression) => write!(f, "{}", expression), MarkerTree::And(and_list) => f.write_str( &and_list .iter() .map(format_inner) .collect::>() .join(" and "), ), MarkerTree::Or(or_list) => f.write_str( &or_list .iter() .map(format_inner) .collect::>() .join(" or "), ), } } } /// ```text /// version_cmp = wsp* <'<=' | '<' | '!=' | '==' | '>=' | '>' | '~=' | '==='> /// marker_op = version_cmp | (wsp* 'in') | (wsp* 'not' wsp+ 'in') /// ``` fn parse_marker_operator(chars: &mut CharIter) -> Result { let (operator, start, len) = chars.take_while(|char| !char.is_whitespace() && char != '\'' && char != '"'); if operator == "not" { // 'not' wsp+ 'in' match chars.next() { None => { return Err(Pep508Error { message: Pep508ErrorSource::String( "Expected whitespace after 'not', found end of input".to_string(), ), start: chars.get_pos(), len: 1, input: chars.copy_chars(), }) } Some((_, whitespace)) if whitespace.is_whitespace() => {} Some((pos, other)) => { return Err(Pep508Error { message: Pep508ErrorSource::String(format!( "Expected whitespace after 'not', found '{}'", other )), start: pos, len: 1, input: chars.copy_chars(), }) } }; chars.eat_whitespace(); chars.next_expect_char('i', chars.get_pos())?; chars.next_expect_char('n', chars.get_pos())?; return Ok(MarkerOperator::NotIn); } MarkerOperator::from_str(&operator).map_err(|_| Pep508Error { message: Pep508ErrorSource::String(format!( "Expected a valid marker operator (such as '>=' or 'not in'), found '{}'", operator )), start, len, input: chars.copy_chars(), }) } /// Either a single or double quoted string or one of 'python_version', 'python_full_version', /// 'os_name', 'sys_platform', 'platform_release', 'platform_system', 'platform_version', /// 'platform_machine', 'platform_python_implementation', 'implementation_name', /// 'implementation_version', 'extra' fn parse_marker_value(chars: &mut CharIter) -> Result { // > User supplied constants are always encoded as strings with either ' or " quote marks. Note // > that backslash escapes are not defined, but existing implementations do support them. They // > are not included in this specification because they add complexity and there is no observable // > need for them today. Similarly we do not define non-ASCII character support: all the runtime // > variables we are referencing are expected to be ASCII-only. match chars.peek() { None => Err(Pep508Error { message: Pep508ErrorSource::String( "Expected marker value, found end of dependency specification".to_string(), ), start: chars.get_pos(), len: 1, input: chars.copy_chars(), }), // It can be a string ... Some((start_pos, quotation_mark @ ('"' | '\''))) => { chars.next(); let (value, _, _) = chars.take_while(|c| c != quotation_mark); chars.next_expect_char(quotation_mark, start_pos)?; Ok(MarkerValue::string_value(value)) } // ... or it can be a keyword Some(_) => { let (key, start, len) = chars.take_while(|char| { !char.is_whitespace() && !['>', '=', '<', '!', '~', ')'].contains(&char) }); MarkerValue::from_str(&key).map_err(|_| Pep508Error { message: Pep508ErrorSource::String(format!( "Expected a valid marker name, found '{}'", key )), start, len, input: chars.copy_chars(), }) } } } /// ```text /// marker_var:l marker_op:o marker_var:r /// ``` fn parse_marker_key_op_value(chars: &mut CharIter) -> Result { chars.eat_whitespace(); let lvalue = parse_marker_value(chars)?; chars.eat_whitespace(); // "not in" and "in" must be preceded by whitespace. We must already have matched a whitespace // when we're here because other `parse_marker_key` would have pulled the characters in and // errored let operator = parse_marker_operator(chars)?; chars.eat_whitespace(); let rvalue = parse_marker_value(chars)?; Ok(MarkerExpression { l_value: lvalue, operator, r_value: rvalue, }) } /// ```text /// marker_expr = marker_var:l marker_op:o marker_var:r -> (o, l, r) /// | wsp* '(' marker:m wsp* ')' -> m /// ``` fn parse_marker_expr(chars: &mut CharIter) -> Result { chars.eat_whitespace(); if let Some(start_pos) = chars.eat('(') { let marker = parse_marker_or(chars)?; chars.next_expect_char(')', start_pos)?; Ok(marker) } else { Ok(MarkerTree::Expression(parse_marker_key_op_value(chars)?)) } } /// ```text /// marker_and = marker_expr:l wsp* 'and' marker_expr:r -> ('and', l, r) /// | marker_expr:m -> m /// ``` fn parse_marker_and(chars: &mut CharIter) -> Result { parse_marker_op(chars, "and", MarkerTree::And, parse_marker_expr) } /// ```text /// marker_or = marker_and:l wsp* 'or' marker_and:r -> ('or', l, r) /// | marker_and:m -> m /// ``` fn parse_marker_or(chars: &mut CharIter) -> Result { parse_marker_op(chars, "or", MarkerTree::Or, parse_marker_and) } /// Parses both `marker_and` and `marker_or` fn parse_marker_op( chars: &mut CharIter, op: &str, op_constructor: fn(Vec) -> MarkerTree, parse_inner: fn(&mut CharIter) -> Result, ) -> Result { // marker_and or marker_expr let first_element = parse_inner(chars)?; // wsp* chars.eat_whitespace(); // Check if we're done here instead of invoking the whole vec allocating loop if matches!(chars.peek_char(), None | Some(')')) { return Ok(first_element); } let mut expressions = Vec::with_capacity(1); expressions.push(first_element); loop { // wsp* chars.eat_whitespace(); // ('or' marker_and) or ('and' marker_or) let (maybe_op, _start, _len) = chars.peek_while(|c| !c.is_whitespace()); match maybe_op { value if value == op => { chars.take_while(|c| !c.is_whitespace()); let expression = parse_inner(chars)?; expressions.push(expression); } _ => { // Build minimal trees return if expressions.len() == 1 { Ok(expressions.remove(0)) } else { Ok(op_constructor(expressions)) }; } } } } /// ```text /// marker = marker_or /// ``` pub(crate) fn parse_markers_impl(chars: &mut CharIter) -> Result { let marker = parse_marker_or(chars)?; chars.eat_whitespace(); if let Some((pos, unexpected)) = chars.next() { // If we're here, both parse_marker_or and parse_marker_and returned because the next // character was neither "and" nor "or" return Err(Pep508Error { message: Pep508ErrorSource::String(format!( "Unexpected character '{}', expected 'and', 'or' or end of input", unexpected )), start: pos, len: chars.chars.clone().count(), input: chars.copy_chars(), }); }; Ok(marker) } /// Parses markers such as `python_version < '3.8'` or /// `python_version == "3.10" and (sys_platform == "win32" or (os_name == "linux" and implementation_name == 'cpython'))` fn parse_markers(markers: &str) -> Result { let mut chars = CharIter::new(markers); parse_markers_impl(&mut chars) } #[cfg(test)] mod test { use crate::marker::{MarkerEnvironment, StringVersion}; use crate::{MarkerExpression, MarkerOperator, MarkerTree, MarkerValue, MarkerValueString}; use indoc::indoc; use log::Level; use std::str::FromStr; fn assert_err(input: &str, error: &str) { assert_eq!(MarkerTree::from_str(input).unwrap_err().to_string(), error); } fn env37() -> MarkerEnvironment { let v37 = StringVersion::from_str("3.7").unwrap(); MarkerEnvironment { implementation_name: "".to_string(), implementation_version: v37.clone(), os_name: "linux".to_string(), platform_machine: "".to_string(), platform_python_implementation: "".to_string(), platform_release: "".to_string(), platform_system: "".to_string(), platform_version: "".to_string(), python_full_version: v37.clone(), python_version: v37, sys_platform: "linux".to_string(), } } /// Copied from #[test] fn test_marker_equivalence() { let values = [ (r#"python_version == '2.7'"#, r#"python_version == "2.7""#), (r#"python_version == "2.7""#, r#"python_version == "2.7""#), ( r#"python_version == "2.7" and os_name == "posix""#, r#"python_version == "2.7" and os_name == "posix""#, ), ( r#"python_version == "2.7" or os_name == "posix""#, r#"python_version == "2.7" or os_name == "posix""#, ), ( r#"python_version == "2.7" and os_name == "posix" or sys_platform == "win32""#, r#"python_version == "2.7" and os_name == "posix" or sys_platform == "win32""#, ), (r#"(python_version == "2.7")"#, r#"python_version == "2.7""#), ( r#"(python_version == "2.7" and sys_platform == "win32")"#, r#"python_version == "2.7" and sys_platform == "win32""#, ), ( r#"python_version == "2.7" and (sys_platform == "win32" or sys_platform == "linux")"#, r#"python_version == "2.7" and (sys_platform == "win32" or sys_platform == "linux")"#, ), ]; for (a, b) in values { assert_eq!( MarkerTree::from_str(a).unwrap(), MarkerTree::from_str(b).unwrap(), "{} {}", a, b ); } } #[test] fn test_marker_evaluation() { let v27 = StringVersion::from_str("2.7").unwrap(); let env27 = MarkerEnvironment { implementation_name: "".to_string(), implementation_version: v27.clone(), os_name: "linux".to_string(), platform_machine: "".to_string(), platform_python_implementation: "".to_string(), platform_release: "".to_string(), platform_system: "".to_string(), platform_version: "".to_string(), python_full_version: v27.clone(), python_version: v27, sys_platform: "linux".to_string(), }; let env37 = env37(); let marker1 = MarkerTree::from_str("python_version == '2.7'").unwrap(); let marker2 = MarkerTree::from_str( "os_name == \"linux\" or python_version == \"3.7\" and sys_platform == \"win32\"", ) .unwrap(); let marker3 = MarkerTree::from_str( "python_version == \"2.7\" and (sys_platform == \"win32\" or sys_platform == \"linux\")", ).unwrap(); assert!(marker1.evaluate(&env27, &[])); assert!(!marker1.evaluate(&env37, &[])); assert!(marker2.evaluate(&env27, &[])); assert!(marker2.evaluate(&env37, &[])); assert!(marker3.evaluate(&env27, &[])); assert!(!marker3.evaluate(&env37, &[])); } #[test] fn warnings() { let env37 = env37(); testing_logger::setup(); let compare_keys = MarkerTree::from_str("platform_version == sys_platform").unwrap(); compare_keys.evaluate(&env37, &[]); testing_logger::validate(|captured_logs| { assert_eq!( captured_logs[0].body, "Comparing two markers with each other doesn't make any sense, evaluating to false" ); assert_eq!(captured_logs[0].level, Level::Warn); assert_eq!(captured_logs.len(), 1); }); let non_pep440 = MarkerTree::from_str("python_version >= '3.9.'").unwrap(); non_pep440.evaluate(&env37, &[]); testing_logger::validate(|captured_logs| { assert_eq!( captured_logs[0].body, "Expected PEP 440 version to compare with python_version, found '3.9.', evaluating to false: Version `3.9.` doesn't match PEP 440 rules" ); assert_eq!(captured_logs[0].level, Level::Warn); assert_eq!(captured_logs.len(), 1); }); let string_string = MarkerTree::from_str("'b' >= 'a'").unwrap(); string_string.evaluate(&env37, &[]); testing_logger::validate(|captured_logs| { assert_eq!( captured_logs[0].body, "Comparing two quoted strings with each other doesn't make sense: 'b' >= 'a', evaluating to false" ); assert_eq!(captured_logs[0].level, Level::Warn); assert_eq!(captured_logs.len(), 1); }); let string_string = MarkerTree::from_str(r#"os.name == 'posix' and platform.machine == 'x86_64' and platform.python_implementation == 'CPython' and 'Ubuntu' in platform.version and sys.platform == 'linux'"#).unwrap(); string_string.evaluate(&env37, &[]); testing_logger::validate(|captured_logs| { let messages: Vec<_> = captured_logs .iter() .map(|message| { assert_eq!(message.level, Level::Warn); &message.body }) .collect(); let expected = [ "os.name is deprecated in favor of os_name", "platform.machine is deprecated in favor of platform_machine", "platform.python_implementation is deprecated in favor of platform_python_implementation", "platform.version is deprecated in favor of platform_version", "sys.platform is deprecated in favor of sys_platform" ]; assert_eq!(messages, &expected); }); } #[test] fn test_not_in() { MarkerTree::from_str("'posix' not in os_name").unwrap(); } #[test] fn test_marker_version_star() { let env37 = env37(); let (result, warnings) = MarkerTree::from_str("python_version == '3.7.*'") .unwrap() .evaluate_collect_warnings(&env37, &[]); assert_eq!(warnings, &[]); assert!(result); } #[test] fn test_tilde_equal() { let env37 = env37(); let (result, warnings) = MarkerTree::from_str("python_version ~= '3.7'") .unwrap() .evaluate_collect_warnings(&env37, &[]); assert_eq!(warnings, &[]); assert!(result); } #[test] fn test_closing_parentheses() { MarkerTree::from_str(r#"( "linux" in sys_platform) and extra == 'all'"#).unwrap(); } #[test] fn wrong_quotes_dot_star() { assert_err( r#"python_version == "3.8".* and python_version >= "3.8""#, indoc! {r#" Unexpected character '.', expected 'and', 'or' or end of input python_version == "3.8".* and python_version >= "3.8" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"# }, ); assert_err( r#"python_version == "3.8".*"#, indoc! {r#" Unexpected character '.', expected 'and', 'or' or end of input python_version == "3.8".* ^"# }, ); } #[test] fn test_marker_expression() { assert_eq!( MarkerExpression::from_str(r#"os_name == "nt""#).unwrap(), MarkerExpression { l_value: MarkerValue::MarkerEnvString(MarkerValueString::OsName), operator: MarkerOperator::Equal, r_value: MarkerValue::QuotedString("nt".to_string()), } ); } #[test] fn test_marker_expression_to_long() { assert_eq!( MarkerExpression::from_str(r#"os_name == "nt" and python_version >= "3.8""#) .unwrap_err() .to_string(), indoc! {r#" Unexpected character 'a', expected end of input os_name == "nt" and python_version >= "3.8" ^^^^^^^^^^^^^^^^^^^^^^^^^^"# }, ); } #[cfg(feature = "serde")] #[test] fn test_marker_environment_from_json() { let _env: MarkerEnvironment = serde_json::from_str( r##"{ "implementation_name": "cpython", "implementation_version": "3.7.13", "os_name": "posix", "platform_machine": "x86_64", "platform_python_implementation": "CPython", "platform_release": "5.4.188+", "platform_system": "Linux", "platform_version": "#1 SMP Sun Apr 24 10:03:06 PDT 2022", "python_full_version": "3.7.13", "python_version": "3.7", "sys_platform": "linux" }"##, ) .unwrap(); } } pep508_rs-0.2.3/src/modern.rs000064400000000000000000000344121046102023000140000ustar 00000000000000//! WIP Draft for a poetry/cargo like, modern dependency specification //! //! This still needs //! * Better VersionSpecifier (e.g. allowing `^1.19`) and it's sentry integration //! * PEP 440/PEP 508 translation //! * a json schema #![cfg(feature = "modern")] use crate::MarkerValue::QuotedString; use crate::{MarkerExpression, MarkerOperator, MarkerTree, MarkerValue, Requirement, VersionOrUrl}; use anyhow::{bail, format_err, Context}; use once_cell::sync::Lazy; use pep440_rs::{Operator, Pep440Error, Version, VersionSpecifier, VersionSpecifiers}; use regex::Regex; use serde::{de, Deserialize, Deserializer, Serialize}; use std::collections::HashMap; use std::str::FromStr; use url::Url; /// Shared fields for version/git/file/path/url dependencies (`optional`, `extras`, `markers`) #[derive(Eq, PartialEq, Debug, Clone, Deserialize, Serialize)] pub struct RequirementModernCommon { /// Whether this is an optional dependency. This is inverted from PEP 508 extras where the /// requirements has the extras attached, as here the extras has a table where each extra /// says which optional dependencies it activates #[serde(default)] pub optional: bool, /// The list of extras pub extras: Option>, /// The list of markers . /// Note that this will not accept extras. /// /// TODO: Deserialize into `MarkerTree` that does not accept the extras key pub markers: Option, } /// Instead of only PEP 440 specifier, you can also set a single version (exact) or TODO use /// the semver caret #[derive(Eq, PartialEq, Debug, Clone, Serialize)] pub enum VersionSpecifierModern { /// e.g. `4.12.1-beta.1` Version(Version), /// e.g. `== 4.12.1-beta.1` or `>=3.8,<4.0` VersionSpecifier(VersionSpecifiers), } impl VersionSpecifierModern { /// `4.12.1-beta.1` -> `== 4.12.1-beta.1` /// `== 4.12.1-beta.1` -> `== 4.12.1-beta.1` /// `>=3.8,<4.0` -> `>=3.8,<4.0` /// TODO: `^1.19` -> `>=1.19,<2.0` pub fn to_pep508_specifier(&self) -> VersionSpecifiers { match self { // unwrapping is safe here because we're using Operator::Equal VersionSpecifierModern::Version(version) => { [VersionSpecifier::new(Operator::Equal, version.clone(), false).unwrap()] .into_iter() .collect() } VersionSpecifierModern::VersionSpecifier(version_specifiers) => { version_specifiers.clone() } } } } impl FromStr for VersionSpecifierModern { /// TODO: Modern needs it's own error type type Err = Pep440Error; /// dispatching between just a version and a version specifier set fn from_str(s: &str) -> Result { // If it starts with if s.trim_start().starts_with(|x: char| x.is_ascii_digit()) { Ok(Self::Version(Version::from_str(s).map_err(|err| { // TODO: Fix this in pep440_rs Pep440Error { message: err, line: s.to_string(), start: 0, width: 1, } })?)) } else if s.starts_with('^') { todo!("TODO caret operator is not supported yet"); } else { Ok(Self::VersionSpecifier(VersionSpecifiers::from_str(s)?)) } } } /// https://github.com/serde-rs/serde/issues/908#issuecomment-298027413 impl<'de> Deserialize<'de> for VersionSpecifierModern { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { let s = String::deserialize(deserializer)?; FromStr::from_str(&s).map_err(de::Error::custom) } } /// WIP Draft for a poetry/cargo like, modern dependency specification #[derive(Eq, PartialEq, Debug, Clone, Deserialize, Serialize)] #[serde(untagged)] pub enum RequirementModern { /// e.g. `numpy = "1.24.1"` Dependency(VersionSpecifierModern), /// e.g. `numpy = { version = "1.24.1" }` or `django-anymail = { version = "1.24.1", extras = ["sendgrid"], optional = true }` LongDependency { /// e.g. `1.2.3.beta1` version: VersionSpecifierModern, #[serde(flatten)] #[allow(missing_docs)] common: RequirementModernCommon, }, /// e.g. `tqdm = { git = "https://github.com/tqdm/tqdm", rev = "0bb91857eca0d4aea08f66cf1c8949abe0cd6b7a" }` GitDependency { /// URL of the git repository e.g. `https://github.com/tqdm/tqdm` git: Url, /// The git branch to use branch: Option, /// The git revision to use. Can be the short revision (`0bb9185`) or the long revision /// (`0bb91857eca0d4aea08f66cf1c8949abe0cd6b7a`) rev: Option, #[serde(flatten)] #[allow(missing_docs)] common: RequirementModernCommon, }, /// e.g. `tqdm = { file = "tqdm-4.65.0-py3-none-any.whl" }` FileDependency { /// Path to a source distribution (e.g. `tqdm-4.65.0.tar.gz`) or wheel (e.g. `tqdm-4.65.0-py3-none-any.whl`) file: String, #[serde(flatten)] #[allow(missing_docs)] common: RequirementModernCommon, }, /// Path to a directory with source distributions and/or wheels e.g. /// `scilib_core = { path = "build_wheels/scilib_core/" }`. /// /// Use this option if you e.g. have multiple platform platform dependent wheels or want to /// have a fallback to a source distribution for you wheel. PathDependency { /// e.g. `dist/`, `target/wheels` or `vendored` path: String, #[serde(flatten)] #[allow(missing_docs)] common: RequirementModernCommon, }, /// e.g. `jax = { url = "https://storage.googleapis.com/jax-releases/cuda112/jaxlib-0.1.64+cuda112-cp39-none-manylinux2010_x86_64.whl" }` UrlDependency { /// URL to a source distribution or wheel. The file available there must be named /// appropriately for a source distribution or a wheel. url: Url, #[serde(flatten)] #[allow(missing_docs)] common: RequirementModernCommon, }, } /// Adopted from the grammar at static EXTRA_REGEX: Lazy = Lazy::new(|| Regex::new(r"^[a-zA-Z0-9]([-_.]*[a-zA-Z0-9])*$").unwrap()); impl RequirementModern { /// Check the things that serde doesn't check, namely that extra names are valid pub fn check(&self) -> anyhow::Result<()> { match self { Self::LongDependency { common, .. } | Self::GitDependency { common, .. } | Self::FileDependency { common, .. } | Self::PathDependency { common, .. } | Self::UrlDependency { common, .. } => { if let Some(extras) = &common.extras { for extra in extras { if !EXTRA_REGEX.is_match(extra) { bail!("Not a valid extra name: '{}'", extra) } } } } _ => {} } Ok(()) } /// WIP Converts the modern format to PEP 508 pub fn to_pep508( &self, name: &str, extras: &HashMap>, ) -> Result { let default = RequirementModernCommon { optional: false, extras: None, markers: None, }; let common = match self { RequirementModern::Dependency(..) => &default, RequirementModern::LongDependency { common, .. } | RequirementModern::GitDependency { common, .. } | RequirementModern::FileDependency { common, .. } | RequirementModern::PathDependency { common, .. } | RequirementModern::UrlDependency { common, .. } => common, }; let marker = if common.optional { // invert the extras table from the modern format // extra1 -> optional_dep1, optional_dep2, ... // to the PEP 508 format // optional_dep1; extra == "extra1" or extra == "extra2" let dep_markers = extras .iter() .filter(|(_marker, dependencies)| dependencies.contains(&name.to_string())) .map(|(marker, _dependencies)| { MarkerTree::Expression(MarkerExpression { l_value: MarkerValue::Extra, operator: MarkerOperator::Equal, r_value: QuotedString(marker.to_string()), }) }) .collect(); // any of these extras activates the dependency -> or clause let dep_markers = MarkerTree::Or(dep_markers); let joined_marker = if let Some(user_markers) = &common.markers { let user_markers = MarkerTree::from_str(user_markers) .context("TODO: parse this in serde already")?; // but the dependency needs to be activated and match the other markers // -> and clause MarkerTree::And(vec![user_markers, dep_markers]) } else { dep_markers }; Some(joined_marker) } else { None }; if let Some(extras) = &common.extras { debug_assert!(extras.iter().all(|extra| EXTRA_REGEX.is_match(extra))); } let version_or_url = match self { RequirementModern::Dependency(version) => { VersionOrUrl::VersionSpecifier(version.to_pep508_specifier()) } RequirementModern::LongDependency { version, .. } => { VersionOrUrl::VersionSpecifier(version.to_pep508_specifier()) } RequirementModern::GitDependency { git, branch, rev, .. } => { // TODO: Read https://peps.python.org/pep-0440/#direct-references properly // set_scheme doesn't like us adding `git+` to https, therefore this hack let mut url = Url::parse(&format!("git+{}", git)).expect("TODO: Better url validation"); match (branch, rev) { (Some(_branch), Some(_rev)) => { bail!("You can set both branch and rev (for {})", name) } (Some(branch), None) => url.set_path(&format!("{}@{}", url.path(), branch)), (None, Some(rev)) => url.set_path(&format!("{}@{}", url.path(), rev)), (None, None) => {} } VersionOrUrl::Url(url) } RequirementModern::FileDependency { file, .. } => VersionOrUrl::Url( Url::from_file_path(file) .map_err(|()| format_err!("File must be absolute (for {})", name))?, ), RequirementModern::PathDependency { path, .. } => VersionOrUrl::Url( Url::from_directory_path(path) .map_err(|()| format_err!("Path must be absolute (for {})", name))?, ), RequirementModern::UrlDependency { url, .. } => VersionOrUrl::Url(url.clone()), }; Ok(Requirement { name: name.to_string(), extras: common.extras.clone(), version_or_url: Some(version_or_url), marker, }) } } #[cfg(test)] mod test { use crate::modern::{RequirementModern, VersionSpecifierModern}; use crate::Requirement; use indoc::indoc; use pep440_rs::VersionSpecifiers; use serde::Deserialize; use std::collections::{BTreeMap, HashMap}; use std::str::FromStr; #[test] fn test_basic() { let deps: HashMap = toml::from_str(r#"numpy = "==1.19""#).unwrap(); assert_eq!( deps["numpy"], RequirementModern::Dependency(VersionSpecifierModern::VersionSpecifier( VersionSpecifiers::from_str("==1.19").unwrap() )) ); assert_eq!( deps["numpy"].to_pep508("numpy", &HashMap::new()).unwrap(), Requirement::from_str("numpy== 1.19").unwrap() ); } #[test] fn test_conversion() { #[derive(Deserialize)] struct PyprojectToml { // BTreeMap to keep the order #[serde(rename = "modern-dependencies")] modern_dependencies: BTreeMap, extras: HashMap>, } let pyproject_toml = indoc! {r#" [modern-dependencies] pydantic = "1.10.5" numpy = ">=1.24.2, <2.0.0" pandas = { version = ">=1.5.3, <2.0.0" } flask = { version = "2.2.3 ", extras = ["dotenv"], optional = true } tqdm = { git = "https://github.com/tqdm/tqdm", rev = "0bb91857eca0d4aea08f66cf1c8949abe0cd6b7a" } jax = { url = "https://storage.googleapis.com/jax-releases/cuda112/jaxlib-0.1.64+cuda112-cp39-none-manylinux2010_x86_64.whl" } zstandard = { file = "/home/ferris/wheels/zstandard/zstandard-0.20.0.tar.gz" } h5py = { path = "/home/ferris/wheels/h5py/" } [extras] internet = ["flask"] "# }; let deps: PyprojectToml = toml::from_str(pyproject_toml).unwrap(); let actual: Vec = deps .modern_dependencies .iter() .map(|(name, spec)| spec.to_pep508(name, &deps.extras).unwrap().to_string()) .collect(); let expected: Vec = vec![ "flask[dotenv] ==2.2.3 ; extra == 'internet'".to_string(), "h5py @ file:///home/ferris/wheels/h5py/".to_string(), "jax @ https://storage.googleapis.com/jax-releases/cuda112/jaxlib-0.1.64+cuda112-cp39-none-manylinux2010_x86_64.whl".to_string(), "numpy >=1.24.2, <2.0.0".to_string(), "pandas >=1.5.3, <2.0.0".to_string(), "pydantic ==1.10.5".to_string(), "tqdm @ git+https://github.com/tqdm/tqdm@0bb91857eca0d4aea08f66cf1c8949abe0cd6b7a".to_string(), "zstandard @ file:///home/ferris/wheels/zstandard/zstandard-0.20.0.tar.gz".to_string() ]; assert_eq!(actual, expected) } }