libcst-0.1.0/Cargo.lock0000644000000606770000000000100103050ustar # This file is automatically @generated by Cargo. # It is not intended for manual editing. version = 3 [[package]] name = "aho-corasick" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" dependencies = [ "memchr", ] [[package]] name = "anes" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "annotate-snippets" version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7021ce4924a3f25f802b2cccd1af585e39ea1a363a1aa2e72afe54b67a3a7a7" [[package]] name = "anstyle" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" [[package]] name = "bumpalo" version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "cast" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "libc", ] [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chic" version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5b5db619f3556839cb2223ae86ff3f9a09da2c5013be42bc9af08c9589bf70c" dependencies = [ "annotate-snippets", ] [[package]] name = "ciborium" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" dependencies = [ "ciborium-io", "ciborium-ll", "serde", ] [[package]] name = "ciborium-io" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" [[package]] name = "ciborium-ll" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" dependencies = [ "ciborium-io", "half", ] [[package]] name = "clap" version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d5f1946157a96594eb2d2c10eb7ad9a2b27518cb3000209dec700c35df9197d" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78116e32a042dd73c2901f0dc30790d20ff3447f3e3472fad359e8c3d282bcd6" dependencies = [ "anstyle", "clap_lex", ] [[package]] name = "clap_lex" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" [[package]] name = "criterion" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" dependencies = [ "anes", "cast", "ciborium", "clap", "criterion-plot", "is-terminal", "itertools 0.10.5", "num-traits", "once_cell", "oorandom", "plotters", "rayon", "regex", "serde", "serde_derive", "serde_json", "tinytemplate", "walkdir", ] [[package]] name = "criterion-plot" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", "itertools 0.10.5", ] [[package]] name = "crossbeam-channel" version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c02a4d71819009c192cf4872265391563fd6a84c81ff2c0f2a7026ca4c1d85c" dependencies = [ "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-deque" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" dependencies = [ "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07db9d94cbd326813772c968ccd25999e5f8ae22f4f8d1b11effa37ef6ce281d" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", "memoffset", "once_cell", "scopeguard", ] [[package]] name = "crossbeam-utils" version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d82ee10ce34d7bc12c2122495e7593a9c41347ecdd64185af4ecf72cb1a7f83" dependencies = [ "cfg-if", "once_cell", ] [[package]] name = "difference" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" [[package]] name = "either" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "errno" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" dependencies = [ "errno-dragonfly", "libc", "windows-sys", ] [[package]] name = "errno-dragonfly" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" dependencies = [ "cc", "libc", ] [[package]] name = "half" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" [[package]] name = "hermit-abi" version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" dependencies = [ "libc", ] [[package]] name = "hermit-abi" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "indoc" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05a0bd019339e5d968b37855180087b7b9d512c5046fbd244cf8c95687927d6e" [[package]] name = "instant" version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if", ] [[package]] name = "is-terminal" version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi 0.3.2", "rustix", "windows-sys", ] [[package]] name = "itertools" version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ "either", ] [[package]] name = "itertools" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" dependencies = [ "either", ] [[package]] name = "itoa" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" [[package]] name = "js-sys" version = "0.3.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27" dependencies = [ "wasm-bindgen", ] [[package]] name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libcst" version = "0.1.0" dependencies = [ "chic", "criterion", "difference", "itertools 0.11.0", "libcst_derive", "memchr", "paste", "peg", "pyo3", "rayon", "regex", "thiserror", ] [[package]] name = "libcst_derive" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "520197c50ba477f258cd7005ec5ed3a7393693ae6bec664990c7c8d9306a7c0d" dependencies = [ "quote", "syn", ] [[package]] name = "linux-raw-sys" version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" [[package]] name = "lock_api" version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ "autocfg", "scopeguard", ] [[package]] name = "log" version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", ] [[package]] name = "memchr" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" dependencies = [ "autocfg", ] [[package]] name = "num-traits" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] [[package]] name = "num_cpus" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" dependencies = [ "hermit-abi 0.1.19", "libc", ] [[package]] name = "once_cell" version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" [[package]] name = "oorandom" version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "parking_lot" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", "parking_lot_core", ] [[package]] name = "parking_lot_core" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" dependencies = [ "cfg-if", "instant", "libc", "redox_syscall", "smallvec", "winapi", ] [[package]] name = "paste" version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" [[package]] name = "peg" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a07f2cafdc3babeebc087e499118343442b742cc7c31b4d054682cc598508554" dependencies = [ "peg-macros", "peg-runtime", ] [[package]] name = "peg-macros" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a90084dc05cf0428428e3d12399f39faad19b0909f64fb9170c9fdd6d9cd49b" dependencies = [ "peg-runtime", "proc-macro2", "quote", ] [[package]] name = "peg-runtime" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa00462b37ead6d11a82c9d568b26682d78e0477dc02d1966c013af80969739" [[package]] name = "plotters" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a" dependencies = [ "num-traits", "plotters-backend", "plotters-svg", "wasm-bindgen", "web-sys", ] [[package]] name = "plotters-backend" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d88417318da0eaf0fdcdb51a0ee6c3bed624333bff8f946733049380be67ac1c" [[package]] name = "plotters-svg" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521fa9638fa597e1dc53e9412a4f9cefb01187ee1f7413076f9e6749e2885ba9" dependencies = [ "plotters-backend", ] [[package]] name = "proc-macro2" version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3" version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "268be0c73583c183f2b14052337465768c07726936a260f480f0857cb95ba543" dependencies = [ "cfg-if", "indoc", "libc", "memoffset", "parking_lot", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", "unindent", ] [[package]] name = "pyo3-build-config" version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28fcd1e73f06ec85bf3280c48c67e731d8290ad3d730f8be9dc07946923005c8" dependencies = [ "once_cell", "target-lexicon", ] [[package]] name = "pyo3-ffi" version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f6cb136e222e49115b3c51c32792886defbfb0adead26a688142b346a0b9ffc" dependencies = [ "libc", "pyo3-build-config", ] [[package]] name = "pyo3-macros" version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94144a1266e236b1c932682136dc35a9dee8d3589728f68130c7c3861ef96b28" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", "syn", ] [[package]] name = "pyo3-macros-backend" version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8df9be978a2d2f0cdebabb03206ed73b11314701a5bfe71b0d753b81997777f" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "quote" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" dependencies = [ "proc-macro2", ] [[package]] name = "rayon" version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" dependencies = [ "either", "rayon-core", ] [[package]] name = "rayon-core" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", "num_cpus", ] [[package]] name = "redox_syscall" version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "regex" version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" dependencies = [ "aho-corasick", "memchr", "regex-automata", "regex-syntax", ] [[package]] name = "regex-automata" version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] [[package]] name = "regex-syntax" version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "rustix" version = "0.38.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bfe0f2582b4931a45d1fa608f8a8722e8b3c7ac54dd6d5f3b3212791fedef49" dependencies = [ "bitflags 2.4.0", "errno", "libc", "linux-raw-sys", "windows-sys", ] [[package]] name = "ryu" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" [[package]] name = "same-file" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" dependencies = [ "winapi-util", ] [[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "serde" version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "serde_json" version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ "itoa", "ryu", "serde", ] [[package]] name = "smallvec" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc88c725d61fc6c3132893370cac4a0200e3fedf5da8331c570664b1987f5ca2" [[package]] name = "syn" version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] [[package]] name = "target-lexicon" version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c02424087780c9b71cc96799eaeddff35af2bc513278cda5c99fc1f5d026d3c1" [[package]] name = "thiserror" version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "tinytemplate" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" dependencies = [ "serde", "serde_json", ] [[package]] name = "unicode-ident" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c" [[package]] name = "unindent" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52fee519a3e570f7df377a06a1a7775cdbfb7aa460be7e08de2b1f0e69973a44" [[package]] name = "walkdir" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" dependencies = [ "same-file", "winapi", "winapi-util", ] [[package]] name = "wasm-bindgen" version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994" dependencies = [ "cfg-if", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" dependencies = [ "bumpalo", "lazy_static", "log", "proc-macro2", "quote", "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa" dependencies = [ "quote", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048" dependencies = [ "proc-macro2", "quote", "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be" [[package]] name = "web-sys" version = "0.3.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fed94beee57daf8dd7d51f2b15dc2bcde92d7a72304cdf662a4371008b71b90" dependencies = [ "js-sys", "wasm-bindgen", ] [[package]] name = "winapi" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu", ] [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ "winapi", ] [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ "windows-targets", ] [[package]] name = "windows-targets" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", "windows_i686_gnu", "windows_i686_msvc", "windows_x86_64_gnu", "windows_x86_64_gnullvm", "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" libcst-0.1.0/Cargo.toml0000644000000032600000000000100103110ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2018" rust-version = "1.70" name = "libcst" version = "0.1.0" authors = ["LibCST Developers"] description = "A Python parser and Concrete Syntax Tree library." homepage = "https://github.com/Instagram/LibCST" documentation = "https://libcst.rtfd.org" readme = "README.md" keywords = [ "python", "cst", "ast", ] categories = ["parser-implementations"] license-file = "LICENSE" [lib] name = "libcst_native" crate-type = [ "cdylib", "rlib", ] [[bin]] name = "parse" path = "src/bin.rs" [[bench]] name = "parser_benchmark" harness = false [dependencies.chic] version = "1.2.2" [dependencies.libcst_derive] version = "0.1.0" [dependencies.memchr] version = "2.5.0" [dependencies.paste] version = "1.0.9" [dependencies.peg] version = "0.8.1" [dependencies.pyo3] version = ">=0.17" optional = true [dependencies.regex] version = "1.9.3" [dependencies.thiserror] version = "1.0.37" [dev-dependencies.criterion] version = "0.5.1" features = ["html_reports"] [dev-dependencies.difference] version = "2.0.0" [dev-dependencies.itertools] version = "0.11.0" [dev-dependencies.rayon] version = "1.7.0" [features] default = ["py"] py = [ "pyo3", "pyo3/extension-module", ] trace = ["peg/trace"] libcst-0.1.0/Cargo.toml.orig000064400000000000000000000026320072674642500140240ustar 00000000000000# Copyright (c) Meta Platforms, Inc. and affiliates. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. [package] name = "libcst" version = "0.1.0" authors = ["LibCST Developers"] edition = "2018" rust-version = "1.70" description = "A Python parser and Concrete Syntax Tree library." license-file = "../../LICENSE" homepage = "https://github.com/Instagram/LibCST" documentation = "https://libcst.rtfd.org" keywords = ["python", "cst", "ast"] categories = ["parser-implementations"] [lib] name = "libcst_native" crate-type = ["cdylib", "rlib"] [[bin]] name = "parse" path = "src/bin.rs" [features] # This is a bit of a hack, since `cargo test` doesn't work with `extension-module`. # To run tests, use `cargo test --no-default-features`. # # Once https://github.com/PyO3/pyo3/pull/1123 lands, it may be better to use # `-Zextra-link-arg` for this instead. default = ["py"] py = ["pyo3", "pyo3/extension-module"] trace = ["peg/trace"] [dependencies] paste = "1.0.9" pyo3 = { version = ">=0.17", optional = true } thiserror = "1.0.37" peg = "0.8.1" chic = "1.2.2" regex = "1.9.3" memchr = "2.5.0" libcst_derive = { path = "../libcst_derive", version = "0.1.0" } [dev-dependencies] criterion = { version = "0.5.1", features = ["html_reports"] } difference = "2.0.0" rayon = "1.7.0" itertools = "0.11.0" [[bench]] name = "parser_benchmark" harness = false libcst-0.1.0/Grammar000064400000000000000000000772330072674642500124570ustar 00000000000000# PEG grammar for Python 3.9 @trailer ''' void * _PyPegen_parse(Parser *p) { // Initialize keywords p->keywords = reserved_keywords; p->n_keyword_lists = n_keyword_lists; // Run parser void *result = NULL; if (p->start_rule == Py_file_input) { result = file_rule(p); } else if (p->start_rule == Py_single_input) { result = interactive_rule(p); } else if (p->start_rule == Py_eval_input) { result = eval_rule(p); } else if (p->start_rule == Py_func_type_input) { result = func_type_rule(p); } else if (p->start_rule == Py_fstring_input) { result = fstring_rule(p); } return result; } // The end ''' file[mod_ty]: a=[statements] ENDMARKER { _PyPegen_make_module(p, a) } interactive[mod_ty]: a=statement_newline { Interactive(a, p->arena) } eval[mod_ty]: a=expressions NEWLINE* ENDMARKER { Expression(a, p->arena) } func_type[mod_ty]: '(' a=[type_expressions] ')' '->' b=expression NEWLINE* ENDMARKER { FunctionType(a, b, p->arena) } fstring[expr_ty]: star_expressions # type_expressions allow */** but ignore them type_expressions[asdl_seq*]: | a=','.expression+ ',' '*' b=expression ',' '**' c=expression { _PyPegen_seq_append_to_end(p, CHECK(_PyPegen_seq_append_to_end(p, a, b)), c) } | a=','.expression+ ',' '*' b=expression { _PyPegen_seq_append_to_end(p, a, b) } | a=','.expression+ ',' '**' b=expression { _PyPegen_seq_append_to_end(p, a, b) } | '*' a=expression ',' '**' b=expression { _PyPegen_seq_append_to_end(p, CHECK(_PyPegen_singleton_seq(p, a)), b) } | '*' a=expression { _PyPegen_singleton_seq(p, a) } | '**' a=expression { _PyPegen_singleton_seq(p, a) } | ','.expression+ statements[asdl_seq*]: a=statement+ { _PyPegen_seq_flatten(p, a) } statement[asdl_seq*]: a=compound_stmt { _PyPegen_singleton_seq(p, a) } | simple_stmt statement_newline[asdl_seq*]: | a=compound_stmt NEWLINE { _PyPegen_singleton_seq(p, a) } | simple_stmt | NEWLINE { _PyPegen_singleton_seq(p, CHECK(_Py_Pass(EXTRA))) } | ENDMARKER { _PyPegen_interactive_exit(p) } simple_stmt[asdl_seq*]: | a=small_stmt !';' NEWLINE { _PyPegen_singleton_seq(p, a) } # Not needed, there for speedup | a=';'.small_stmt+ [';'] NEWLINE { a } # NOTE: assignment MUST precede expression, else parsing a simple assignment # will throw a SyntaxError. small_stmt[stmt_ty] (memo): | assignment | e=star_expressions { _Py_Expr(e, EXTRA) } | &'return' return_stmt | &('import' | 'from') import_stmt | &'raise' raise_stmt | 'pass' { _Py_Pass(EXTRA) } | &'del' del_stmt | &'yield' yield_stmt | &'assert' assert_stmt | 'break' { _Py_Break(EXTRA) } | 'continue' { _Py_Continue(EXTRA) } | &'global' global_stmt | &'nonlocal' nonlocal_stmt compound_stmt[stmt_ty]: | &('def' | '@' | ASYNC) function_def | &'if' if_stmt | &('class' | '@') class_def | &('with' | ASYNC) with_stmt | &('for' | ASYNC) for_stmt | &'try' try_stmt | &'while' while_stmt # NOTE: annotated_rhs may start with 'yield'; yield_expr must start with 'yield' assignment[stmt_ty]: | a=NAME ':' b=expression c=['=' d=annotated_rhs { d }] { CHECK_VERSION( 6, "Variable annotation syntax is", _Py_AnnAssign(CHECK(_PyPegen_set_expr_context(p, a, Store)), b, c, 1, EXTRA) ) } | a=('(' b=single_target ')' { b } | single_subscript_attribute_target) ':' b=expression c=['=' d=annotated_rhs { d }] { CHECK_VERSION(6, "Variable annotations syntax is", _Py_AnnAssign(a, b, c, 0, EXTRA)) } | a=(z=star_targets '=' { z })+ b=(yield_expr | star_expressions) !'=' tc=[TYPE_COMMENT] { _Py_Assign(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } | a=single_target b=augassign ~ c=(yield_expr | star_expressions) { _Py_AugAssign(a, b->kind, c, EXTRA) } | invalid_assignment augassign[AugOperator*]: | '+=' { _PyPegen_augoperator(p, Add) } | '-=' { _PyPegen_augoperator(p, Sub) } | '*=' { _PyPegen_augoperator(p, Mult) } | '@=' { CHECK_VERSION(5, "The '@' operator is", _PyPegen_augoperator(p, MatMult)) } | '/=' { _PyPegen_augoperator(p, Div) } | '%=' { _PyPegen_augoperator(p, Mod) } | '&=' { _PyPegen_augoperator(p, BitAnd) } | '|=' { _PyPegen_augoperator(p, BitOr) } | '^=' { _PyPegen_augoperator(p, BitXor) } | '<<=' { _PyPegen_augoperator(p, LShift) } | '>>=' { _PyPegen_augoperator(p, RShift) } | '**=' { _PyPegen_augoperator(p, Pow) } | '//=' { _PyPegen_augoperator(p, FloorDiv) } global_stmt[stmt_ty]: 'global' a=','.NAME+ { _Py_Global(CHECK(_PyPegen_map_names_to_ids(p, a)), EXTRA) } nonlocal_stmt[stmt_ty]: 'nonlocal' a=','.NAME+ { _Py_Nonlocal(CHECK(_PyPegen_map_names_to_ids(p, a)), EXTRA) } yield_stmt[stmt_ty]: y=yield_expr { _Py_Expr(y, EXTRA) } assert_stmt[stmt_ty]: 'assert' a=expression b=[',' z=expression { z }] { _Py_Assert(a, b, EXTRA) } del_stmt[stmt_ty]: | 'del' a=del_targets &(';' | NEWLINE) { _Py_Delete(a, EXTRA) } | invalid_del_stmt import_stmt[stmt_ty]: import_name | import_from import_name[stmt_ty]: 'import' a=dotted_as_names { _Py_Import(a, EXTRA) } # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS import_from[stmt_ty]: | 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets { _Py_ImportFrom(b->v.Name.id, c, _PyPegen_seq_count_dots(a), EXTRA) } | 'from' a=('.' | '...')+ 'import' b=import_from_targets { _Py_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), EXTRA) } import_from_targets[asdl_seq*]: | '(' a=import_from_as_names [','] ')' { a } | import_from_as_names !',' | '*' { _PyPegen_singleton_seq(p, CHECK(_PyPegen_alias_for_star(p))) } | invalid_import_from_targets import_from_as_names[asdl_seq*]: | a=','.import_from_as_name+ { a } import_from_as_name[alias_ty]: | a=NAME b=['as' z=NAME { z }] { _Py_alias(a->v.Name.id, (b) ? ((expr_ty) b)->v.Name.id : NULL, p->arena) } dotted_as_names[asdl_seq*]: | a=','.dotted_as_name+ { a } dotted_as_name[alias_ty]: | a=dotted_name b=['as' z=NAME { z }] { _Py_alias(a->v.Name.id, (b) ? ((expr_ty) b)->v.Name.id : NULL, p->arena) } dotted_name[expr_ty]: | a=dotted_name '.' b=NAME { _PyPegen_join_names_with_dot(p, a, b) } | NAME if_stmt[stmt_ty]: | 'if' a=named_expression ':' b=block c=elif_stmt { _Py_If(a, b, CHECK(_PyPegen_singleton_seq(p, c)), EXTRA) } | 'if' a=named_expression ':' b=block c=[else_block] { _Py_If(a, b, c, EXTRA) } elif_stmt[stmt_ty]: | 'elif' a=named_expression ':' b=block c=elif_stmt { _Py_If(a, b, CHECK(_PyPegen_singleton_seq(p, c)), EXTRA) } | 'elif' a=named_expression ':' b=block c=[else_block] { _Py_If(a, b, c, EXTRA) } else_block[asdl_seq*]: 'else' ':' b=block { b } while_stmt[stmt_ty]: | 'while' a=named_expression ':' b=block c=[else_block] { _Py_While(a, b, c, EXTRA) } for_stmt[stmt_ty]: | 'for' t=star_targets 'in' ~ ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] { _Py_For(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA) } | ASYNC 'for' t=star_targets 'in' ~ ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] { CHECK_VERSION(5, "Async for loops are", _Py_AsyncFor(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA)) } | invalid_for_target with_stmt[stmt_ty]: | 'with' '(' a=','.with_item+ ','? ')' ':' b=block { _Py_With(a, b, NULL, EXTRA) } | 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { _Py_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } | ASYNC 'with' '(' a=','.with_item+ ','? ')' ':' b=block { CHECK_VERSION(5, "Async with statements are", _Py_AsyncWith(a, b, NULL, EXTRA)) } | ASYNC 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { CHECK_VERSION(5, "Async with statements are", _Py_AsyncWith(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA)) } with_item[withitem_ty]: | e=expression 'as' t=star_target &(',' | ')' | ':') { _Py_withitem(e, t, p->arena) } | invalid_with_item | e=expression { _Py_withitem(e, NULL, p->arena) } try_stmt[stmt_ty]: | 'try' ':' b=block f=finally_block { _Py_Try(b, NULL, NULL, f, EXTRA) } | 'try' ':' b=block ex=except_block+ el=[else_block] f=[finally_block] { _Py_Try(b, ex, el, f, EXTRA) } except_block[excepthandler_ty]: | 'except' e=expression t=['as' z=NAME { z }] ':' b=block { _Py_ExceptHandler(e, (t) ? ((expr_ty) t)->v.Name.id : NULL, b, EXTRA) } | 'except' ':' b=block { _Py_ExceptHandler(NULL, NULL, b, EXTRA) } finally_block[asdl_seq*]: 'finally' ':' a=block { a } return_stmt[stmt_ty]: | 'return' a=[star_expressions] { _Py_Return(a, EXTRA) } raise_stmt[stmt_ty]: | 'raise' a=expression b=['from' z=expression { z }] { _Py_Raise(a, b, EXTRA) } | 'raise' { _Py_Raise(NULL, NULL, EXTRA) } function_def[stmt_ty]: | d=decorators f=function_def_raw { _PyPegen_function_def_decorators(p, d, f) } | function_def_raw function_def_raw[stmt_ty]: | 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { _Py_FunctionDef(n->v.Name.id, (params) ? params : CHECK(_PyPegen_empty_arguments(p)), b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) } | ASYNC 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { CHECK_VERSION( 5, "Async functions are", _Py_AsyncFunctionDef(n->v.Name.id, (params) ? params : CHECK(_PyPegen_empty_arguments(p)), b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) ) } func_type_comment[Token*]: | NEWLINE t=TYPE_COMMENT &(NEWLINE INDENT) { t } # Must be followed by indented block | invalid_double_type_comments | TYPE_COMMENT params[arguments_ty]: | invalid_parameters | parameters parameters[arguments_ty]: | a=slash_no_default b=param_no_default* c=param_with_default* d=[star_etc] { _PyPegen_make_arguments(p, a, NULL, b, c, d) } | a=slash_with_default b=param_with_default* c=[star_etc] { _PyPegen_make_arguments(p, NULL, a, NULL, b, c) } | a=param_no_default+ b=param_with_default* c=[star_etc] { _PyPegen_make_arguments(p, NULL, NULL, a, b, c) } | a=param_with_default+ b=[star_etc] { _PyPegen_make_arguments(p, NULL, NULL, NULL, a, b)} | a=star_etc { _PyPegen_make_arguments(p, NULL, NULL, NULL, NULL, a) } # Some duplication here because we can't write (',' | &')'), # which is because we don't support empty alternatives (yet). # slash_no_default[asdl_seq*]: | a=param_no_default+ '/' ',' { a } | a=param_no_default+ '/' &')' { a } slash_with_default[SlashWithDefault*]: | a=param_no_default* b=param_with_default+ '/' ',' { _PyPegen_slash_with_default(p, a, b) } | a=param_no_default* b=param_with_default+ '/' &')' { _PyPegen_slash_with_default(p, a, b) } star_etc[StarEtc*]: | '*' a=param_no_default b=param_maybe_default* c=[kwds] { _PyPegen_star_etc(p, a, b, c) } | '*' ',' b=param_maybe_default+ c=[kwds] { _PyPegen_star_etc(p, NULL, b, c) } | a=kwds { _PyPegen_star_etc(p, NULL, NULL, a) } | invalid_star_etc kwds[arg_ty]: '**' a=param_no_default { a } # One parameter. This *includes* a following comma and type comment. # # There are three styles: # - No default # - With default # - Maybe with default # # There are two alternative forms of each, to deal with type comments: # - Ends in a comma followed by an optional type comment # - No comma, optional type comment, must be followed by close paren # The latter form is for a final parameter without trailing comma. # param_no_default[arg_ty]: | a=param ',' tc=TYPE_COMMENT? { _PyPegen_add_type_comment_to_arg(p, a, tc) } | a=param tc=TYPE_COMMENT? &')' { _PyPegen_add_type_comment_to_arg(p, a, tc) } param_with_default[NameDefaultPair*]: | a=param c=default ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) } | a=param c=default tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) } param_maybe_default[NameDefaultPair*]: | a=param c=default? ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) } | a=param c=default? tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) } param[arg_ty]: a=NAME b=annotation? { _Py_arg(a->v.Name.id, b, NULL, EXTRA) } annotation[expr_ty]: ':' a=expression { a } default[expr_ty]: '=' a=expression { a } decorators[asdl_seq*]: a=('@' f=named_expression NEWLINE { f })+ { a } class_def[stmt_ty]: | a=decorators b=class_def_raw { _PyPegen_class_def_decorators(p, a, b) } | class_def_raw class_def_raw[stmt_ty]: | 'class' a=NAME b=['(' z=[arguments] ')' { z }] ':' c=block { _Py_ClassDef(a->v.Name.id, (b) ? ((expr_ty) b)->v.Call.args : NULL, (b) ? ((expr_ty) b)->v.Call.keywords : NULL, c, NULL, EXTRA) } block[asdl_seq*] (memo): | NEWLINE INDENT a=statements DEDENT { a } | simple_stmt | invalid_block star_expressions[expr_ty]: | a=star_expression b=(',' c=star_expression { c })+ [','] { _Py_Tuple(CHECK(_PyPegen_seq_insert_in_front(p, a, b)), Load, EXTRA) } | a=star_expression ',' { _Py_Tuple(CHECK(_PyPegen_singleton_seq(p, a)), Load, EXTRA) } | star_expression star_expression[expr_ty] (memo): | '*' a=bitwise_or { _Py_Starred(a, Load, EXTRA) } | expression star_named_expressions[asdl_seq*]: a=','.star_named_expression+ [','] { a } star_named_expression[expr_ty]: | '*' a=bitwise_or { _Py_Starred(a, Load, EXTRA) } | named_expression named_expression[expr_ty]: | a=NAME ':=' ~ b=expression { _Py_NamedExpr(CHECK(_PyPegen_set_expr_context(p, a, Store)), b, EXTRA) } | expression !':=' | invalid_named_expression annotated_rhs[expr_ty]: yield_expr | star_expressions expressions[expr_ty]: | a=expression b=(',' c=expression { c })+ [','] { _Py_Tuple(CHECK(_PyPegen_seq_insert_in_front(p, a, b)), Load, EXTRA) } | a=expression ',' { _Py_Tuple(CHECK(_PyPegen_singleton_seq(p, a)), Load, EXTRA) } | expression expression[expr_ty] (memo): | a=disjunction 'if' b=disjunction 'else' c=expression { _Py_IfExp(b, a, c, EXTRA) } | disjunction | lambdef lambdef[expr_ty]: | 'lambda' a=[lambda_params] ':' b=expression { _Py_Lambda((a) ? a : CHECK(_PyPegen_empty_arguments(p)), b, EXTRA) } lambda_params[arguments_ty]: | invalid_lambda_parameters | lambda_parameters # lambda_parameters etc. duplicates parameters but without annotations # or type comments, and if there's no comma after a parameter, we expect # a colon, not a close parenthesis. (For more, see parameters above.) # lambda_parameters[arguments_ty]: | a=lambda_slash_no_default b=lambda_param_no_default* c=lambda_param_with_default* d=[lambda_star_etc] { _PyPegen_make_arguments(p, a, NULL, b, c, d) } | a=lambda_slash_with_default b=lambda_param_with_default* c=[lambda_star_etc] { _PyPegen_make_arguments(p, NULL, a, NULL, b, c) } | a=lambda_param_no_default+ b=lambda_param_with_default* c=[lambda_star_etc] { _PyPegen_make_arguments(p, NULL, NULL, a, b, c) } | a=lambda_param_with_default+ b=[lambda_star_etc] { _PyPegen_make_arguments(p, NULL, NULL, NULL, a, b)} | a=lambda_star_etc { _PyPegen_make_arguments(p, NULL, NULL, NULL, NULL, a) } lambda_slash_no_default[asdl_seq*]: | a=lambda_param_no_default+ '/' ',' { a } | a=lambda_param_no_default+ '/' &':' { a } lambda_slash_with_default[SlashWithDefault*]: | a=lambda_param_no_default* b=lambda_param_with_default+ '/' ',' { _PyPegen_slash_with_default(p, a, b) } | a=lambda_param_no_default* b=lambda_param_with_default+ '/' &':' { _PyPegen_slash_with_default(p, a, b) } lambda_star_etc[StarEtc*]: | '*' a=lambda_param_no_default b=lambda_param_maybe_default* c=[lambda_kwds] { _PyPegen_star_etc(p, a, b, c) } | '*' ',' b=lambda_param_maybe_default+ c=[lambda_kwds] { _PyPegen_star_etc(p, NULL, b, c) } | a=lambda_kwds { _PyPegen_star_etc(p, NULL, NULL, a) } | invalid_lambda_star_etc lambda_kwds[arg_ty]: '**' a=lambda_param_no_default { a } lambda_param_no_default[arg_ty]: | a=lambda_param ',' { a } | a=lambda_param &':' { a } lambda_param_with_default[NameDefaultPair*]: | a=lambda_param c=default ',' { _PyPegen_name_default_pair(p, a, c, NULL) } | a=lambda_param c=default &':' { _PyPegen_name_default_pair(p, a, c, NULL) } lambda_param_maybe_default[NameDefaultPair*]: | a=lambda_param c=default? ',' { _PyPegen_name_default_pair(p, a, c, NULL) } | a=lambda_param c=default? &':' { _PyPegen_name_default_pair(p, a, c, NULL) } lambda_param[arg_ty]: a=NAME { _Py_arg(a->v.Name.id, NULL, NULL, EXTRA) } disjunction[expr_ty] (memo): | a=conjunction b=('or' c=conjunction { c })+ { _Py_BoolOp( Or, CHECK(_PyPegen_seq_insert_in_front(p, a, b)), EXTRA) } | conjunction conjunction[expr_ty] (memo): | a=inversion b=('and' c=inversion { c })+ { _Py_BoolOp( And, CHECK(_PyPegen_seq_insert_in_front(p, a, b)), EXTRA) } | inversion inversion[expr_ty] (memo): | 'not' a=inversion { _Py_UnaryOp(Not, a, EXTRA) } | comparison comparison[expr_ty]: | a=bitwise_or b=compare_op_bitwise_or_pair+ { _Py_Compare(a, CHECK(_PyPegen_get_cmpops(p, b)), CHECK(_PyPegen_get_exprs(p, b)), EXTRA) } | bitwise_or compare_op_bitwise_or_pair[CmpopExprPair*]: | eq_bitwise_or | noteq_bitwise_or | lte_bitwise_or | lt_bitwise_or | gte_bitwise_or | gt_bitwise_or | notin_bitwise_or | in_bitwise_or | isnot_bitwise_or | is_bitwise_or eq_bitwise_or[CmpopExprPair*]: '==' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Eq, a) } noteq_bitwise_or[CmpopExprPair*]: | (tok='!=' { _PyPegen_check_barry_as_flufl(p, tok) ? NULL : tok}) a=bitwise_or {_PyPegen_cmpop_expr_pair(p, NotEq, a) } lte_bitwise_or[CmpopExprPair*]: '<=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, LtE, a) } lt_bitwise_or[CmpopExprPair*]: '<' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Lt, a) } gte_bitwise_or[CmpopExprPair*]: '>=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, GtE, a) } gt_bitwise_or[CmpopExprPair*]: '>' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Gt, a) } notin_bitwise_or[CmpopExprPair*]: 'not' 'in' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, NotIn, a) } in_bitwise_or[CmpopExprPair*]: 'in' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, In, a) } isnot_bitwise_or[CmpopExprPair*]: 'is' 'not' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, IsNot, a) } is_bitwise_or[CmpopExprPair*]: 'is' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Is, a) } bitwise_or[expr_ty]: | a=bitwise_or '|' b=bitwise_xor { _Py_BinOp(a, BitOr, b, EXTRA) } | bitwise_xor bitwise_xor[expr_ty]: | a=bitwise_xor '^' b=bitwise_and { _Py_BinOp(a, BitXor, b, EXTRA) } | bitwise_and bitwise_and[expr_ty]: | a=bitwise_and '&' b=shift_expr { _Py_BinOp(a, BitAnd, b, EXTRA) } | shift_expr shift_expr[expr_ty]: | a=shift_expr '<<' b=sum { _Py_BinOp(a, LShift, b, EXTRA) } | a=shift_expr '>>' b=sum { _Py_BinOp(a, RShift, b, EXTRA) } | sum sum[expr_ty]: | a=sum '+' b=term { _Py_BinOp(a, Add, b, EXTRA) } | a=sum '-' b=term { _Py_BinOp(a, Sub, b, EXTRA) } | term term[expr_ty]: | a=term '*' b=factor { _Py_BinOp(a, Mult, b, EXTRA) } | a=term '/' b=factor { _Py_BinOp(a, Div, b, EXTRA) } | a=term '//' b=factor { _Py_BinOp(a, FloorDiv, b, EXTRA) } | a=term '%' b=factor { _Py_BinOp(a, Mod, b, EXTRA) } | a=term '@' b=factor { CHECK_VERSION(5, "The '@' operator is", _Py_BinOp(a, MatMult, b, EXTRA)) } | factor factor[expr_ty] (memo): | '+' a=factor { _Py_UnaryOp(UAdd, a, EXTRA) } | '-' a=factor { _Py_UnaryOp(USub, a, EXTRA) } | '~' a=factor { _Py_UnaryOp(Invert, a, EXTRA) } | power power[expr_ty]: | a=await_primary '**' b=factor { _Py_BinOp(a, Pow, b, EXTRA) } | await_primary await_primary[expr_ty] (memo): | AWAIT a=primary { CHECK_VERSION(5, "Await expressions are", _Py_Await(a, EXTRA)) } | primary primary[expr_ty]: | invalid_primary # must be before 'primay genexp' because of invalid_genexp | a=primary '.' b=NAME { _Py_Attribute(a, b->v.Name.id, Load, EXTRA) } | a=primary b=genexp { _Py_Call(a, CHECK(_PyPegen_singleton_seq(p, b)), NULL, EXTRA) } | a=primary '(' b=[arguments] ')' { _Py_Call(a, (b) ? ((expr_ty) b)->v.Call.args : NULL, (b) ? ((expr_ty) b)->v.Call.keywords : NULL, EXTRA) } | a=primary '[' b=slices ']' { _Py_Subscript(a, b, Load, EXTRA) } | atom slices[expr_ty]: | a=slice !',' { a } | a=','.slice+ [','] { _Py_Tuple(a, Load, EXTRA) } slice[expr_ty]: | a=[expression] ':' b=[expression] c=[':' d=[expression] { d }] { _Py_Slice(a, b, c, EXTRA) } | a=expression { a } atom[expr_ty]: | NAME | 'True' { _Py_Constant(Py_True, NULL, EXTRA) } | 'False' { _Py_Constant(Py_False, NULL, EXTRA) } | 'None' { _Py_Constant(Py_None, NULL, EXTRA) } | '__peg_parser__' { RAISE_SYNTAX_ERROR("You found it!") } | &STRING strings | NUMBER | &'(' (tuple | group | genexp) | &'[' (list | listcomp) | &'{' (dict | set | dictcomp | setcomp) | '...' { _Py_Constant(Py_Ellipsis, NULL, EXTRA) } strings[expr_ty] (memo): a=STRING+ { _PyPegen_concatenate_strings(p, a) } list[expr_ty]: | '[' a=[star_named_expressions] ']' { _Py_List(a, Load, EXTRA) } listcomp[expr_ty]: | '[' a=named_expression ~ b=for_if_clauses ']' { _Py_ListComp(a, b, EXTRA) } | invalid_comprehension tuple[expr_ty]: | '(' a=[y=star_named_expression ',' z=[star_named_expressions] { _PyPegen_seq_insert_in_front(p, y, z) } ] ')' { _Py_Tuple(a, Load, EXTRA) } group[expr_ty]: | '(' a=(yield_expr | named_expression) ')' { a } | invalid_group genexp[expr_ty]: | '(' a=named_expression ~ b=for_if_clauses ')' { _Py_GeneratorExp(a, b, EXTRA) } | invalid_comprehension set[expr_ty]: '{' a=star_named_expressions '}' { _Py_Set(a, EXTRA) } setcomp[expr_ty]: | '{' a=named_expression ~ b=for_if_clauses '}' { _Py_SetComp(a, b, EXTRA) } | invalid_comprehension dict[expr_ty]: | '{' a=[double_starred_kvpairs] '}' { _Py_Dict(CHECK(_PyPegen_get_keys(p, a)), CHECK(_PyPegen_get_values(p, a)), EXTRA) } dictcomp[expr_ty]: | '{' a=kvpair b=for_if_clauses '}' { _Py_DictComp(a->key, a->value, b, EXTRA) } | invalid_dict_comprehension double_starred_kvpairs[asdl_seq*]: a=','.double_starred_kvpair+ [','] { a } double_starred_kvpair[KeyValuePair*]: | '**' a=bitwise_or { _PyPegen_key_value_pair(p, NULL, a) } | kvpair kvpair[KeyValuePair*]: a=expression ':' b=expression { _PyPegen_key_value_pair(p, a, b) } for_if_clauses[asdl_seq*]: | for_if_clause+ for_if_clause[comprehension_ty]: | ASYNC 'for' a=star_targets 'in' ~ b=disjunction c=('if' z=disjunction { z })* { CHECK_VERSION(6, "Async comprehensions are", _Py_comprehension(a, b, c, 1, p->arena)) } | 'for' a=star_targets 'in' ~ b=disjunction c=('if' z=disjunction { z })* { _Py_comprehension(a, b, c, 0, p->arena) } | invalid_for_target yield_expr[expr_ty]: | 'yield' 'from' a=expression { _Py_YieldFrom(a, EXTRA) } | 'yield' a=[star_expressions] { _Py_Yield(a, EXTRA) } arguments[expr_ty] (memo): | a=args [','] &')' { a } | invalid_arguments args[expr_ty]: | a=','.(starred_expression | named_expression !'=')+ b=[',' k=kwargs {k}] { _PyPegen_collect_call_seqs(p, a, b, EXTRA) } | a=kwargs { _Py_Call(_PyPegen_dummy_name(p), CHECK_NULL_ALLOWED(_PyPegen_seq_extract_starred_exprs(p, a)), CHECK_NULL_ALLOWED(_PyPegen_seq_delete_starred_exprs(p, a)), EXTRA) } kwargs[asdl_seq*]: | a=','.kwarg_or_starred+ ',' b=','.kwarg_or_double_starred+ { _PyPegen_join_sequences(p, a, b) } | ','.kwarg_or_starred+ | ','.kwarg_or_double_starred+ starred_expression[expr_ty]: | '*' a=expression { _Py_Starred(a, Load, EXTRA) } kwarg_or_starred[KeywordOrStarred*]: | a=NAME '=' b=expression { _PyPegen_keyword_or_starred(p, CHECK(_Py_keyword(a->v.Name.id, b, EXTRA)), 1) } | a=starred_expression { _PyPegen_keyword_or_starred(p, a, 0) } | invalid_kwarg kwarg_or_double_starred[KeywordOrStarred*]: | a=NAME '=' b=expression { _PyPegen_keyword_or_starred(p, CHECK(_Py_keyword(a->v.Name.id, b, EXTRA)), 1) } | '**' a=expression { _PyPegen_keyword_or_starred(p, CHECK(_Py_keyword(NULL, a, EXTRA)), 1) } | invalid_kwarg # NOTE: star_targets may contain *bitwise_or, targets may not. star_targets[expr_ty]: | a=star_target !',' { a } | a=star_target b=(',' c=star_target { c })* [','] { _Py_Tuple(CHECK(_PyPegen_seq_insert_in_front(p, a, b)), Store, EXTRA) } star_targets_list_seq[asdl_seq*]: a=','.star_target+ [','] { a } star_targets_tuple_seq[asdl_seq*]: | a=star_target b=(',' c=star_target { c })+ [','] { _PyPegen_seq_insert_in_front(p, a, b) } | a=star_target ',' { _PyPegen_singleton_seq(p, a) } star_target[expr_ty] (memo): | '*' a=(!'*' star_target) { _Py_Starred(CHECK(_PyPegen_set_expr_context(p, a, Store)), Store, EXTRA) } | target_with_star_atom target_with_star_atom[expr_ty] (memo): | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Store, EXTRA) } | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Store, EXTRA) } | star_atom star_atom[expr_ty]: | a=NAME { _PyPegen_set_expr_context(p, a, Store) } | '(' a=target_with_star_atom ')' { _PyPegen_set_expr_context(p, a, Store) } | '(' a=[star_targets_tuple_seq] ')' { _Py_Tuple(a, Store, EXTRA) } | '[' a=[star_targets_list_seq] ']' { _Py_List(a, Store, EXTRA) } single_target[expr_ty]: | single_subscript_attribute_target | a=NAME { _PyPegen_set_expr_context(p, a, Store) } | '(' a=single_target ')' { a } single_subscript_attribute_target[expr_ty]: | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Store, EXTRA) } | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Store, EXTRA) } del_targets[asdl_seq*]: a=','.del_target+ [','] { a } del_target[expr_ty] (memo): | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Del, EXTRA) } | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Del, EXTRA) } | del_t_atom del_t_atom[expr_ty]: | a=NAME { _PyPegen_set_expr_context(p, a, Del) } | '(' a=del_target ')' { _PyPegen_set_expr_context(p, a, Del) } | '(' a=[del_targets] ')' { _Py_Tuple(a, Del, EXTRA) } | '[' a=[del_targets] ']' { _Py_List(a, Del, EXTRA) } targets[asdl_seq*]: a=','.target+ [','] { a } target[expr_ty] (memo): | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Store, EXTRA) } | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Store, EXTRA) } | t_atom t_primary[expr_ty]: | a=t_primary '.' b=NAME &t_lookahead { _Py_Attribute(a, b->v.Name.id, Load, EXTRA) } | a=t_primary '[' b=slices ']' &t_lookahead { _Py_Subscript(a, b, Load, EXTRA) } | a=t_primary b=genexp &t_lookahead { _Py_Call(a, CHECK(_PyPegen_singleton_seq(p, b)), NULL, EXTRA) } | a=t_primary '(' b=[arguments] ')' &t_lookahead { _Py_Call(a, (b) ? ((expr_ty) b)->v.Call.args : NULL, (b) ? ((expr_ty) b)->v.Call.keywords : NULL, EXTRA) } | a=atom &t_lookahead { a } t_lookahead: '(' | '[' | '.' t_atom[expr_ty]: | a=NAME { _PyPegen_set_expr_context(p, a, Store) } | '(' a=target ')' { _PyPegen_set_expr_context(p, a, Store) } | '(' b=[targets] ')' { _Py_Tuple(b, Store, EXTRA) } | '[' b=[targets] ']' { _Py_List(b, Store, EXTRA) } # From here on, there are rules for invalid syntax with specialised error messages invalid_arguments: | args ',' '*' { RAISE_SYNTAX_ERROR("iterable argument unpacking follows keyword argument unpacking") } | a=expression for_if_clauses ',' [args | expression for_if_clauses] { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "Generator expression must be parenthesized") } | a=args for_if_clauses { _PyPegen_nonparen_genexp_in_call(p, a) } | args ',' a=expression for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "Generator expression must be parenthesized") } | a=args ',' args { _PyPegen_arguments_parsing_error(p, a) } invalid_kwarg: | !(NAME '=') a=expression b='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "expression cannot contain assignment, perhaps you meant \"==\"?") } invalid_named_expression: | a=expression ':=' expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "cannot use assignment expressions with %s", _PyPegen_get_expr_name(a)) } invalid_assignment: | a=invalid_ann_assign_target ':' expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "only single target (not %s) can be annotated", _PyPegen_get_expr_name(a) )} | a=star_named_expression ',' star_named_expressions* ':' expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "only single target (not tuple) can be annotated") } | a=expression ':' expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "illegal target for annotation") } | (star_targets '=')* a=star_expressions '=' { RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) } | (star_targets '=')* a=yield_expr '=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "assignment to yield expression not possible") } | a=star_expressions augassign (yield_expr | star_expressions) { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "'%s' is an illegal expression for augmented assignment", _PyPegen_get_expr_name(a) )} invalid_ann_assign_target[expr_ty]: | list | tuple | '(' a=invalid_ann_assign_target ')' { a } invalid_del_stmt: | 'del' a=star_expressions { RAISE_SYNTAX_ERROR_INVALID_TARGET(DEL_TARGETS, a) } invalid_block: | NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block") } invalid_primary: | primary a='{' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "invalid syntax") } invalid_comprehension: | ('[' | '(' | '{') a=starred_expression for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "iterable unpacking cannot be used in comprehension") } invalid_dict_comprehension: | '{' a='**' bitwise_or for_if_clauses '}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "dict unpacking cannot be used in dict comprehension") } invalid_parameters: | param_no_default* (slash_with_default | param_with_default+) param_no_default { RAISE_SYNTAX_ERROR("non-default argument follows default argument") } invalid_lambda_parameters: | lambda_param_no_default* (lambda_slash_with_default | lambda_param_with_default+) lambda_param_no_default { RAISE_SYNTAX_ERROR("non-default argument follows default argument") } invalid_star_etc: | '*' (')' | ',' (')' | '**')) { RAISE_SYNTAX_ERROR("named arguments must follow bare *") } | '*' ',' TYPE_COMMENT { RAISE_SYNTAX_ERROR("bare * has associated type comment") } invalid_lambda_star_etc: | '*' (':' | ',' (':' | '**')) { RAISE_SYNTAX_ERROR("named arguments must follow bare *") } invalid_double_type_comments: | TYPE_COMMENT NEWLINE TYPE_COMMENT NEWLINE INDENT { RAISE_SYNTAX_ERROR("Cannot have two type comments on def") } invalid_with_item: | expression 'as' a=expression { RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) } invalid_for_target: | ASYNC? 'for' a=star_expressions { RAISE_SYNTAX_ERROR_INVALID_TARGET(FOR_TARGETS, a) } invalid_group: | '(' a=starred_expression ')' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "can't use starred expression here") } invalid_import_from_targets: | import_from_as_names ',' { RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") }libcst-0.1.0/LICENSE000064400000000000000000000111320072674642500121350ustar 00000000000000All contributions towards LibCST are MIT licensed. Some Python files have been derived from the standard library and are therefore PSF licensed. Modifications on these files are dual licensed (both MIT and PSF). These files are: - libcst/_parser/base_parser.py - libcst/_parser/parso/utils.py - libcst/_parser/parso/pgen2/generator.py - libcst/_parser/parso/pgen2/grammar_parser.py - libcst/_parser/parso/python/py_token.py - libcst/_parser/parso/python/tokenize.py - libcst/_parser/parso/tests/test_fstring.py - libcst/_parser/parso/tests/test_tokenize.py - libcst/_parser/parso/tests/test_utils.py - libcst_native/src/tokenize/core/mod.rs - libcst_native/src/tokenize/core/string_types.rs Some Python files have been taken from dataclasses and are therefore Apache licensed. Modifications on these files are licensed under Apache 2.0 license. These files are: - libcst/_add_slots.py ------------------------------------------------------------------------------- MIT License Copyright (c) Meta Platforms, Inc. and affiliates. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------- PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated documentation. 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Python Software Foundation; All Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee. 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to Python. 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions. 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade name in a trademark sense to endorse or promote products or services of Licensee, or any third party. 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this License Agreement. ------------------------------------------------------------------------------- APACHE LICENSE, VERSION 2.0 http://www.apache.org/licenses/LICENSE-2.0 libcst-0.1.0/README.md000064400000000000000000000143710072674642500124170ustar 00000000000000# libcst/native A native extension to enable parsing of new Python grammar in LibCST. The extension is written in Rust, and exposed to Python using [PyO3](https://pyo3.rs/). This is packaged together with libcst, and can be imported from `libcst.native`. By default the LibCST APIs use this module for all parsing. Later on, the parser library might be packaged separately as [a Rust crate](https://crates.io). Pull requests towards this are much appreciated. ## Goals 1. Adopt the CPython grammar definition as closely as possible to reduce maintenance burden. This means using a PEG parser. 2. Feature-parity with the pure-python LibCST parser: the API should be easy to use from Python, support parsing with a target version, bytes and strings as inputs, etc. 3. [future] Performance. The aspirational goal is to be within 2x CPython performance, which would enable LibCST to be used in interactive use cases (think IDEs). 4. [future] Error recovery. The parser should be able to handle partially complete documents, returning a CST for the syntactically correct parts, and a list of errors found. ## Structure The extension is organized into two rust crates: `libcst_derive` contains some macros to facilitate various features of CST nodes, and `libcst` contains the `parser` itself (including the Python grammar), a `tokenizer` implementation by @bgw, and a very basic representation of CST `nodes`. Parsing is done by 1. **tokenizing** the input utf-8 string (bytes are not supported at the Rust layer, they are converted to utf-8 strings by the python wrapper) 2. running the **PEG parser** on the tokenized input, which also captures certain anchor tokens in the resulting syntax tree 3. using the anchor tokens to **inflate** the syntax tree into a proper CST These steps are wrapped into a high-level `parse_module` API [here](https://github.com/Instagram/LibCST/blob/main/native/libcst/src/lib.rs#L43), along with `parse_statement` and `parse_expression` functions which all just accept the input string and an optional encoding. These Rust functions are exposed to Python [here](https://github.com/Instagram/LibCST/blob/main/native/libcst/src/py.rs) using the excellent [PyO3](https://pyo3.rs/) library, plus an `IntoPy` trait which is mostly implemented via a macro in `libcst_derive`. ## Hacking ### Nodes All CST nodes are marked with the `#[cst_node]` proc macro, which duplicates the node types; for a node named `Foo`, there's: - `DeflatedFoo`, which is the output of the parsing phase and isn't exposed through the API of the crate. - it has two lifetime parameters: `'r` (or `'input` in the grammar) is the lifetime of `Token` references, and `'a` is the lifetime of `str` slices from the original input - `TokenRef` fields are contained here, while whitespace fields aren't - if there aren't any fields that refer to other CST nodes or `TokenRef`s, there's an extra (private) `_phantom` field that "contains" the two lifetime parameters (this is to make the type parameters of all `DeflatedFoo` types uniform) - it implements the `Inflate` trait, which converts `DeflatedFoo` into `Foo` - `Foo`, which is what's publicly exposed in the crate and is the output of `Inflate`ing `DeflatedFoo`. - it only retains the second (`'a`) lifetime parameter of `DeflatedFoo` to refer back to slices of the original input string - whitespace fields are contained here, but `TokenRef`s aren't - `IntoPy` is implemented for it (assuming the `py` crate feature is enabled), which contains code to translate `Foo` back into a Python object; hence, the fields on `Foo` match the Python CST node implementations (barring fields marked with `#[skip_py]`) ### Grammar The grammar is mostly a straightforward translation from the [CPython grammar](https://github.com/python/cpython/blob/main/Grammar/python.gram), with some exceptions: * The output of grammar rules are deflated CST nodes that capture the AST plus additional anchor token references used for whitespace parsing later on. * Rules in the grammar must be strongly typed, as enforced by the Rust compiler. The CPython grammar rules are a bit more loosely-typed in comparison. * Some features in the CPython peg parser are not supported by rust-peg: keywords, mutually recursive rules, special `invalid_` rules, the `~` operator, terminating the parser early. The PEG parser is run on a `Vec` of `Token`s (more precisely `&'input Vec>`), and tries its best to avoid allocating any strings, working only with references. As such, the output nodes don't own any strings, but refer to slices of the original input (hence the `'input, 'a` lifetime parameters on almost all nodes). ### Whitespace parsing The `Inflate` trait is responsible for taking a "deflated", skeleton CST node, and parsing out the relevant whitespace from the anchor tokens to produce an "inflated" (normal) CST node. In addition to the deflated node, inflation requires a whitespace config object which contains global information required for certain aspects of whitespace parsing, like the default indentation. Inflation consumes the deflated node, while mutating the tokens referenced by it. This is important to make sure whitespace is only ever assigned to at most one CST node. The `Inflate` trait implementation needs to ensure that all whitespace is assigned to a CST node; this is generally verified using roundtrip tests (i.e. parsing code and then generating it back to then assert the original and generated are byte-by-byte equal). The general convention is that the top-most possible node owns a certain piece of whitespace, which should be straightforward to achieve in a top-down parser like `Inflate`. In cases where whitespace is shared between sibling nodes, usually the leftmost node owns the whitespace except in the case of trailing commas and closing parentheses, where the latter owns the whitespace (for backwards compatibility with the pure python parser). See the implementation of `inflate_element` for how this is done. ### Tests In addition to running the python test suite, you can run some tests written in rust with ``` cd native cargo test ``` These include unit and roundtrip tests. Additionally, some benchmarks can be run on x86-based architectures using `cargo bench`. ### Code Formatting Use `cargo fmt` to format your code. libcst-0.1.0/benches/parser_benchmark.rs000064400000000000000000000123760072674642500164260ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree use std::{ path::{Component, PathBuf}, time::Duration, }; use criterion::{ black_box, criterion_group, criterion_main, measurement::Measurement, BatchSize, BenchmarkId, Criterion, Throughput, }; use itertools::Itertools; use rayon::prelude::*; use libcst_native::{ parse_module, parse_tokens_without_whitespace, tokenize, Codegen, Config, Inflate, }; #[cfg(not(windows))] const NEWLINE: &str = "\n"; #[cfg(windows)] const NEWLINE: &str = "\r\n"; fn load_all_fixtures_vec() -> Vec { let mut path = PathBuf::from(file!()); path.pop(); path.pop(); path = path .components() .skip(1) .chain( vec!["tests".as_ref(), "fixtures".as_ref()] .into_iter() .map(Component::Normal), ) .collect(); path.read_dir() .expect("read_dir") .into_iter() .map(|file| { let path = file.unwrap().path(); std::fs::read_to_string(&path).expect("reading_file") }) .collect() } fn load_all_fixtures() -> String { load_all_fixtures_vec().join(NEWLINE) } pub fn inflate_benchmarks(c: &mut Criterion) { let fixture = load_all_fixtures(); let tokens = tokenize(fixture.as_str()).expect("tokenize failed"); let tokvec = tokens.clone().into(); let mut group = c.benchmark_group("inflate"); group.bench_function("all", |b| { b.iter_batched( || { let conf = Config::new(fixture.as_str(), &tokens); let m = parse_tokens_without_whitespace(&tokvec, fixture.as_str(), None) .expect("parse failed"); (conf, m) }, |(conf, m)| black_box(m.inflate(&conf)), BatchSize::SmallInput, ) }); group.finish(); } pub fn parser_benchmarks(c: &mut Criterion) { let fixture = load_all_fixtures(); let mut group = c.benchmark_group("parse"); group.measurement_time(Duration::from_secs(15)); group.bench_function("all", |b| { b.iter_batched( || tokenize(fixture.as_str()).expect("tokenize failed").into(), |tokens| { black_box(drop(parse_tokens_without_whitespace( &tokens, fixture.as_str(), None, ))) }, BatchSize::SmallInput, ) }); group.finish(); } pub fn codegen_benchmarks(c: &mut Criterion) { let input = load_all_fixtures(); let m = parse_module(input.as_str(), None).expect("parse failed"); let mut group = c.benchmark_group("codegen"); group.bench_function("all", |b| { b.iter(|| { let mut state = Default::default(); #[allow(clippy::unit_arg)] black_box(m.codegen(&mut state)); }) }); group.finish(); } pub fn tokenize_benchmarks(c: &mut Criterion) { let input = load_all_fixtures(); let mut group = c.benchmark_group("tokenize"); group.measurement_time(Duration::from_secs(15)); group.bench_function("all", |b| b.iter(|| black_box(tokenize(input.as_str())))); group.finish(); } pub fn parse_into_cst_benchmarks(c: &mut Criterion) { let fixture = load_all_fixtures(); let mut group = c.benchmark_group("parse_into_cst"); group.measurement_time(Duration::from_secs(15)); group.bench_function("all", |b| { b.iter(|| black_box(parse_module(&fixture, None))) }); group.finish(); } pub fn parse_into_cst_multithreaded_benchmarks( c: &mut Criterion, ) where ::Value: Send, { let fixtures = load_all_fixtures_vec(); let mut group = c.benchmark_group("parse_into_cst_parallel"); group.measurement_time(Duration::from_secs(15)); group.warm_up_time(Duration::from_secs(5)); for thread_count in 1..10 { let expanded_fixtures = (0..thread_count) .flat_map(|_| fixtures.clone()) .collect_vec(); group.throughput(Throughput::Elements(expanded_fixtures.len() as u64)); group.bench_with_input( BenchmarkId::from_parameter(thread_count), &thread_count, |b, thread_count| { let thread_pool = rayon::ThreadPoolBuilder::new() .num_threads(*thread_count) .build() .unwrap(); thread_pool.install(|| { b.iter_with_large_drop(|| { expanded_fixtures .par_iter() .map(|contents| black_box(parse_module(&contents, None))) .collect::>() }); }); }, ); } group.finish(); } criterion_group!( name=benches; config=Criterion::default(); targets=parser_benchmarks, codegen_benchmarks, inflate_benchmarks, tokenize_benchmarks, parse_into_cst_benchmarks, parse_into_cst_multithreaded_benchmarks ); criterion_main!(benches); libcst-0.1.0/src/bin.rs000064400000000000000000000016440072674642500130440ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree use libcst_native::*; use std::{ env, io::{self, Read}, process::exit, }; pub fn main() { let mut str = std::string::String::new(); io::stdin().read_to_string(&mut str).unwrap(); match parse_module(str.as_ref(), None) { Err(e) => { eprintln!("{}", prettify_error(e, "stdin")); exit(1); } Ok(m) => { let first_arg = env::args().nth(1).unwrap_or_else(|| "".to_string()); if first_arg == "-d" { println!("{:#?}", m); } if first_arg != "-n" { let mut state = Default::default(); m.codegen(&mut state); print!("{}", state.to_string()); } } }; } libcst-0.1.0/src/lib.rs000064400000000000000000000130660072674642500130430ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. use std::cmp::{max, min}; mod tokenizer; pub use tokenizer::whitespace_parser::Config; use tokenizer::{whitespace_parser, TokConfig, Token, TokenIterator}; mod nodes; use nodes::deflated::Module as DeflatedModule; pub use nodes::*; mod parser; use parser::{ParserError, Result, TokVec}; #[cfg(feature = "py")] pub mod py; pub fn tokenize(text: &str) -> Result> { let iter = TokenIterator::new( text, &TokConfig { async_hacks: false, split_fstring: true, }, ); iter.collect::, _>>() .map_err(|err| ParserError::TokenizerError(err, text)) } pub fn parse_module<'a>( mut module_text: &'a str, encoding: Option<&str>, ) -> Result<'a, Module<'a>> { // Strip UTF-8 BOM if let Some(stripped) = module_text.strip_prefix('\u{feff}') { module_text = stripped; } let tokens = tokenize(module_text)?; let conf = whitespace_parser::Config::new(module_text, &tokens); let tokvec = tokens.into(); let m = parse_tokens_without_whitespace(&tokvec, module_text, encoding)?; Ok(m.inflate(&conf)?) } pub fn parse_tokens_without_whitespace<'r, 'a>( tokens: &'r TokVec<'a>, module_text: &'a str, encoding: Option<&str>, ) -> Result<'a, DeflatedModule<'r, 'a>> { let m = parser::python::file(tokens, module_text, encoding) .map_err(|err| ParserError::ParserError(err, module_text))?; Ok(m) } pub fn parse_statement(text: &str) -> Result { let tokens = tokenize(text)?; let conf = whitespace_parser::Config::new(text, &tokens); let tokvec = tokens.into(); let stm = parser::python::statement_input(&tokvec, text) .map_err(|err| ParserError::ParserError(err, text))?; Ok(stm.inflate(&conf)?) } pub fn parse_expression(text: &str) -> Result { let tokens = tokenize(text)?; let conf = whitespace_parser::Config::new(text, &tokens); let tokvec = tokens.into(); let expr = parser::python::expression_input(&tokvec, text) .map_err(|err| ParserError::ParserError(err, text))?; Ok(expr.inflate(&conf)?) } // n starts from 1 fn bol_offset(source: &str, n: i32) -> usize { if n <= 1 { return 0; } source .match_indices('\n') .nth((n - 2) as usize) .map(|(index, _)| index + 1) .unwrap_or_else(|| source.len()) } pub fn prettify_error(err: ParserError, label: &str) -> std::string::String { match err { ParserError::ParserError(e, module_text) => { let loc = e.location; let context = 1; let start_offset = bol_offset(module_text, loc.start_pos.line as i32 - context); let end_offset = bol_offset(module_text, loc.end_pos.line as i32 + context + 1); let source = &module_text[start_offset..end_offset]; let start = loc.start_pos.offset - start_offset; let end = loc.end_pos.offset - start_offset; chic::Error::new(label) .error( max( 1, loc.start_pos .line .checked_sub(context as usize) .unwrap_or(1), ), start, if start == end { min(end + 1, end_offset - start_offset + 1) } else { end }, source, format!( "expected {} {} -> {}", e.expected, loc.start_pos, loc.end_pos ), ) .to_string() } e => format!("Parse error for {}: {}", label, e), } } #[cfg(test)] mod test { use super::*; use tokenizer::TokError; #[test] fn test_simple() { let n = parse_module("1_", None); assert_eq!( n.err().unwrap(), ParserError::TokenizerError(TokError::BadDecimal, "1_") ); } #[test] fn test_bare_minimum_funcdef() { parse_module("def f(): ...", None).expect("parse error"); } #[test] fn test_funcdef_params() { parse_module("def g(a, b): ...", None).expect("parse error"); } #[test] fn test_single_statement_with_no_newline() { for src in &[ "(\n \\\n)", "(\n \\\n)", "(\n '''\n''')", "del _", "if _:\n '''\n)'''", "if _:\n ('''\n''')", "if _:\n '''\n '''", "if _:\n '''\n ''' ", ] { parse_module(src, None).unwrap_or_else(|e| panic!("'{}' doesn't parse: {}", src, e)); } } #[test] fn bol_offset_first_line() { assert_eq!(0, bol_offset("hello", 1)); assert_eq!(0, bol_offset("hello", 0)); assert_eq!(0, bol_offset("hello\nhello", 1)); assert_eq!(0, bol_offset("hello\nhello", 0)); } #[test] fn bol_offset_second_line() { assert_eq!(5, bol_offset("hello", 2)); assert_eq!(6, bol_offset("hello\nhello", 2)); assert_eq!(6, bol_offset("hello\nhello\nhello", 2)); } #[test] fn bol_offset_last_line() { assert_eq!(5, bol_offset("hello", 3)); assert_eq!(11, bol_offset("hello\nhello", 3)); assert_eq!(12, bol_offset("hello\nhello\nhello", 3)); } } libcst-0.1.0/src/nodes/codegen.rs000064400000000000000000000030320072674642500150010ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. use std::fmt; #[derive(Debug)] pub struct CodegenState<'a> { pub tokens: String, pub indent_tokens: Vec<&'a str>, pub default_newline: &'a str, pub default_indent: &'a str, } impl<'a> CodegenState<'a> { pub fn indent(&mut self, v: &'a str) { self.indent_tokens.push(v); } pub fn dedent(&mut self) { self.indent_tokens.pop(); } pub fn add_indent(&mut self) { self.tokens.extend(self.indent_tokens.iter().cloned()); } pub fn add_token(&mut self, tok: &'a str) { self.tokens.push_str(tok); } } impl<'a> fmt::Display for CodegenState<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.tokens) } } pub trait Codegen<'a> { fn codegen(&self, state: &mut CodegenState<'a>); } impl<'a, T> Codegen<'a> for Option where T: Codegen<'a>, { fn codegen(&self, state: &mut CodegenState<'a>) { if let Some(s) = &self { s.codegen(state); } } } #[cfg(windows)] const LINE_ENDING: &str = "\r\n"; #[cfg(not(windows))] const LINE_ENDING: &str = "\n"; impl<'a> Default for CodegenState<'a> { fn default() -> Self { Self { default_newline: LINE_ENDING, default_indent: " ", indent_tokens: Default::default(), tokens: Default::default(), } } } libcst-0.1.0/src/nodes/expression.rs000064400000000000000000002343750072674642500156140ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. use std::mem::swap; use crate::{ inflate_helpers::adjust_parameters_trailing_whitespace, nodes::{ op::*, statement::*, traits::{Inflate, ParenthesizedDeflatedNode, ParenthesizedNode, Result, WithComma}, whitespace::ParenthesizableWhitespace, Annotation, AssignEqual, AssignTargetExpression, BinaryOp, BooleanOp, Codegen, CodegenState, Colon, Comma, CompOp, Dot, UnaryOp, }, tokenizer::{ whitespace_parser::{parse_parenthesizable_whitespace, Config}, Token, }, }; #[cfg(feature = "py")] use libcst_derive::TryIntoPy; use libcst_derive::{cst_node, Codegen, Inflate, ParenthesizedDeflatedNode, ParenthesizedNode}; type TokenRef<'r, 'a> = &'r Token<'a>; #[cst_node(Default)] pub struct Parameters<'a> { pub params: Vec>, pub star_arg: Option>, pub kwonly_params: Vec>, pub star_kwarg: Option>, pub posonly_params: Vec>, pub posonly_ind: Option>, } impl<'a> Parameters<'a> { pub fn is_empty(&self) -> bool { self.params.is_empty() && self.star_arg.is_none() && self.kwonly_params.is_empty() && self.star_kwarg.is_none() && self.posonly_params.is_empty() && self.posonly_ind.is_none() } } impl<'r, 'a> DeflatedParameters<'r, 'a> { pub fn is_empty(&self) -> bool { self.params.is_empty() && self.star_arg.is_none() && self.kwonly_params.is_empty() && self.star_kwarg.is_none() && self.posonly_params.is_empty() && self.posonly_ind.is_none() } } impl<'r, 'a> Inflate<'a> for DeflatedParameters<'r, 'a> { type Inflated = Parameters<'a>; fn inflate(self, config: &Config<'a>) -> Result { let posonly_params = self.posonly_params.inflate(config)?; let posonly_ind = self.posonly_ind.inflate(config)?; let params = self.params.inflate(config)?; let star_arg = self.star_arg.inflate(config)?; let kwonly_params = self.kwonly_params.inflate(config)?; let star_kwarg = self.star_kwarg.inflate(config)?; Ok(Self::Inflated { params, star_arg, kwonly_params, star_kwarg, posonly_params, posonly_ind, }) } } #[cst_node(Inflate)] pub enum StarArg<'a> { Star(Box>), Param(Box>), } impl<'a> Codegen<'a> for Parameters<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { let params_after_kwonly = self.star_kwarg.is_some(); let params_after_regular = !self.kwonly_params.is_empty() || params_after_kwonly; let params_after_posonly = !self.params.is_empty() || params_after_regular; let star_included = self.star_arg.is_some() || !self.kwonly_params.is_empty(); for p in &self.posonly_params { p.codegen(state, None, true); } match &self.posonly_ind { Some(ind) => ind.codegen(state, params_after_posonly), _ => { if !self.posonly_params.is_empty() { if params_after_posonly { state.add_token("/, "); } else { state.add_token("/"); } } } } let param_size = self.params.len(); for (i, p) in self.params.iter().enumerate() { p.codegen(state, None, params_after_regular || i < param_size - 1); } let kwonly_size = self.kwonly_params.len(); match &self.star_arg { None => { if star_included { state.add_token("*, ") } } Some(StarArg::Param(p)) => p.codegen( state, Some("*"), kwonly_size > 0 || self.star_kwarg.is_some(), ), Some(StarArg::Star(s)) => s.codegen(state), } for (i, p) in self.kwonly_params.iter().enumerate() { p.codegen(state, None, params_after_kwonly || i < kwonly_size - 1); } if let Some(star) = &self.star_kwarg { star.codegen(state, Some("**"), false) } } } #[cst_node] pub struct ParamSlash<'a> { pub comma: Option>, pub whitespace_after: ParenthesizableWhitespace<'a>, pub(crate) tok: TokenRef<'a>, } impl<'a> ParamSlash<'a> { fn codegen(&self, state: &mut CodegenState<'a>, default_comma: bool) { state.add_token("/"); self.whitespace_after.codegen(state); match (&self.comma, default_comma) { (Some(comma), _) => comma.codegen(state), (None, true) => state.add_token(", "), _ => {} } } } impl<'r, 'a> Inflate<'a> for DeflatedParamSlash<'r, 'a> { type Inflated = ParamSlash<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_after = parse_parenthesizable_whitespace(config, &mut self.tok.whitespace_after.borrow_mut())?; let comma = self.comma.inflate(config)?; Ok(Self::Inflated { comma, whitespace_after, }) } } #[cst_node] pub struct ParamStar<'a> { pub comma: Comma<'a>, } impl<'a> Codegen<'a> for ParamStar<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("*"); self.comma.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedParamStar<'r, 'a> { type Inflated = ParamStar<'a>; fn inflate(self, config: &Config<'a>) -> Result { let comma = self.comma.inflate(config)?; Ok(Self::Inflated { comma }) } } #[cst_node(ParenthesizedNode, Default)] pub struct Name<'a> { pub value: &'a str, pub lpar: Vec>, pub rpar: Vec>, } impl<'r, 'a> Inflate<'a> for DeflatedName<'r, 'a> { type Inflated = Name<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { value: self.value, lpar, rpar, }) } } impl<'a> Codegen<'a> for Name<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { state.add_token(self.value); }); } } #[cst_node] pub struct Param<'a> { pub name: Name<'a>, pub annotation: Option>, pub equal: Option>, pub default: Option>, pub comma: Option>, pub star: Option<&'a str>, pub whitespace_after_star: ParenthesizableWhitespace<'a>, pub whitespace_after_param: ParenthesizableWhitespace<'a>, pub(crate) star_tok: Option>, } impl<'r, 'a> Inflate<'a> for DeflatedParam<'r, 'a> { type Inflated = Param<'a>; fn inflate(mut self, config: &Config<'a>) -> Result { let name = self.name.inflate(config)?; let annotation = self.annotation.inflate(config)?; let equal = self.equal.inflate(config)?; let default = self.default.inflate(config)?; let comma = self.comma.inflate(config)?; let whitespace_after_star = if let Some(star_tok) = self.star_tok.as_mut() { parse_parenthesizable_whitespace(config, &mut star_tok.whitespace_after.borrow_mut())? } else { Default::default() }; let whitespace_after_param = Default::default(); // TODO Ok(Self::Inflated { name, annotation, equal, default, comma, star: self.star, whitespace_after_star, whitespace_after_param, }) } } impl<'r, 'a> Default for DeflatedParam<'r, 'a> { fn default() -> Self { Self { name: Default::default(), annotation: None, equal: None, default: None, comma: None, star: Some(""), // Note: this preserves a quirk of the pure python parser star_tok: None, } } } impl<'a> Param<'a> { fn codegen( &self, state: &mut CodegenState<'a>, default_star: Option<&'a str>, default_comma: bool, ) { match (self.star, default_star) { (Some(star), _) => state.add_token(star), (None, Some(star)) => state.add_token(star), _ => {} } self.whitespace_after_star.codegen(state); self.name.codegen(state); if let Some(ann) = &self.annotation { ann.codegen(state, ":"); } match (&self.equal, &self.default) { (Some(equal), Some(def)) => { equal.codegen(state); def.codegen(state); } (None, Some(def)) => { state.add_token(" = "); def.codegen(state); } _ => {} } match &self.comma { Some(comma) => comma.codegen(state), None if default_comma => state.add_token(", "), _ => {} } self.whitespace_after_param.codegen(state); } } #[cst_node] pub struct Arg<'a> { pub value: Expression<'a>, pub keyword: Option>, pub equal: Option>, pub comma: Option>, pub star: &'a str, pub whitespace_after_star: ParenthesizableWhitespace<'a>, pub whitespace_after_arg: ParenthesizableWhitespace<'a>, pub(crate) star_tok: Option>, } impl<'r, 'a> Inflate<'a> for DeflatedArg<'r, 'a> { type Inflated = Arg<'a>; fn inflate(mut self, config: &Config<'a>) -> Result { let whitespace_after_star = if let Some(star_tok) = self.star_tok.as_mut() { parse_parenthesizable_whitespace(config, &mut star_tok.whitespace_after.borrow_mut())? } else { Default::default() }; let keyword = self.keyword.inflate(config)?; let equal = self.equal.inflate(config)?; let value = self.value.inflate(config)?; let comma = self.comma.inflate(config)?; // whitespace_after_arg is handled in Call let whitespace_after_arg = Default::default(); Ok(Self::Inflated { value, keyword, equal, comma, star: self.star, whitespace_after_star, whitespace_after_arg, }) } } impl<'a> Arg<'a> { pub fn codegen(&self, state: &mut CodegenState<'a>, default_comma: bool) { state.add_token(self.star); self.whitespace_after_star.codegen(state); if let Some(kw) = &self.keyword { kw.codegen(state); } if let Some(eq) = &self.equal { eq.codegen(state); } else if self.keyword.is_some() { state.add_token(" = "); } self.value.codegen(state); if let Some(comma) = &self.comma { comma.codegen(state); } else if default_comma { state.add_token(", "); } self.whitespace_after_arg.codegen(state); } } impl<'r, 'a> WithComma<'r, 'a> for DeflatedArg<'r, 'a> { fn with_comma(self, c: DeflatedComma<'r, 'a>) -> Self { Self { comma: Some(c), ..self } } } #[cst_node] #[derive(Default)] pub struct LeftParen<'a> { /// Any space that appears directly after this left parenthesis. pub whitespace_after: ParenthesizableWhitespace<'a>, pub(crate) lpar_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for LeftParen<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("("); self.whitespace_after.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedLeftParen<'r, 'a> { type Inflated = LeftParen<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*self.lpar_tok).whitespace_after.borrow_mut(), )?; Ok(Self::Inflated { whitespace_after }) } } #[cst_node] #[derive(Default)] pub struct RightParen<'a> { /// Any space that appears directly before this right parenthesis. pub whitespace_before: ParenthesizableWhitespace<'a>, pub(crate) rpar_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for RightParen<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.whitespace_before.codegen(state); state.add_token(")"); } } impl<'r, 'a> Inflate<'a> for DeflatedRightParen<'r, 'a> { type Inflated = RightParen<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*self.rpar_tok).whitespace_before.borrow_mut(), )?; Ok(Self::Inflated { whitespace_before }) } } #[cst_node(ParenthesizedNode, Codegen, Inflate)] pub enum Expression<'a> { Name(Box>), Ellipsis(Box>), Integer(Box>), Float(Box>), Imaginary(Box>), Comparison(Box>), UnaryOperation(Box>), BinaryOperation(Box>), BooleanOperation(Box>), Attribute(Box>), Tuple(Box>), Call(Box>), GeneratorExp(Box>), ListComp(Box>), SetComp(Box>), DictComp(Box>), List(Box>), Set(Box>), Dict(Box>), Subscript(Box>), StarredElement(Box>), IfExp(Box>), Lambda(Box>), Yield(Box>), Await(Box>), SimpleString(Box>), ConcatenatedString(Box>), FormattedString(Box>), NamedExpr(Box>), } #[cst_node(ParenthesizedNode)] pub struct Ellipsis<'a> { pub lpar: Vec>, pub rpar: Vec>, } impl<'a> Codegen<'a> for Ellipsis<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { state.add_token("..."); }) } } impl<'r, 'a> Inflate<'a> for DeflatedEllipsis<'r, 'a> { type Inflated = Ellipsis<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { lpar, rpar }) } } #[cst_node(ParenthesizedNode)] pub struct Integer<'a> { /// A string representation of the integer, such as ``"100000"`` or /// ``"100_000"``. pub value: &'a str, pub lpar: Vec>, pub rpar: Vec>, } impl<'a> Codegen<'a> for Integer<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { state.add_token(self.value); }) } } impl<'r, 'a> Inflate<'a> for DeflatedInteger<'r, 'a> { type Inflated = Integer<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { value: self.value, lpar, rpar, }) } } #[cst_node(ParenthesizedNode)] pub struct Float<'a> { /// A string representation of the floating point number, such as ```"0.05"``, /// ``".050"``, or ``"5e-2"``. pub value: &'a str, pub lpar: Vec>, pub rpar: Vec>, } impl<'a> Codegen<'a> for Float<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { state.add_token(self.value); }) } } impl<'r, 'a> Inflate<'a> for DeflatedFloat<'r, 'a> { type Inflated = Float<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { value: self.value, lpar, rpar, }) } } #[cst_node(ParenthesizedNode)] pub struct Imaginary<'a> { /// A string representation of the complex number, such as ``"2j"`` pub value: &'a str, pub lpar: Vec>, pub rpar: Vec>, } impl<'a> Codegen<'a> for Imaginary<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { state.add_token(self.value); }) } } impl<'r, 'a> Inflate<'a> for DeflatedImaginary<'r, 'a> { type Inflated = Imaginary<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { value: self.value, lpar, rpar, }) } } #[cst_node(ParenthesizedNode)] pub struct Comparison<'a> { pub left: Box>, pub comparisons: Vec>, pub lpar: Vec>, pub rpar: Vec>, } impl<'a> Codegen<'a> for Comparison<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.left.codegen(state); for comp in &self.comparisons { comp.codegen(state); } }) } } impl<'r, 'a> Inflate<'a> for DeflatedComparison<'r, 'a> { type Inflated = Comparison<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let left = self.left.inflate(config)?; let comparisons = self.comparisons.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { left, comparisons, lpar, rpar, }) } } #[cst_node(ParenthesizedNode)] pub struct UnaryOperation<'a> { pub operator: UnaryOp<'a>, pub expression: Box>, pub lpar: Vec>, pub rpar: Vec>, } impl<'a> Codegen<'a> for UnaryOperation<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.operator.codegen(state); self.expression.codegen(state); }) } } impl<'r, 'a> Inflate<'a> for DeflatedUnaryOperation<'r, 'a> { type Inflated = UnaryOperation<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let operator = self.operator.inflate(config)?; let expression = self.expression.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { operator, expression, lpar, rpar, }) } } #[cst_node(ParenthesizedNode)] pub struct BinaryOperation<'a> { pub left: Box>, pub operator: BinaryOp<'a>, pub right: Box>, pub lpar: Vec>, pub rpar: Vec>, } impl<'a> Codegen<'a> for BinaryOperation<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.left.codegen(state); self.operator.codegen(state); self.right.codegen(state); }) } } impl<'r, 'a> Inflate<'a> for DeflatedBinaryOperation<'r, 'a> { type Inflated = BinaryOperation<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let left = self.left.inflate(config)?; let operator = self.operator.inflate(config)?; let right = self.right.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { left, operator, right, lpar, rpar, }) } } #[cst_node(ParenthesizedNode)] pub struct BooleanOperation<'a> { pub left: Box>, pub operator: BooleanOp<'a>, pub right: Box>, pub lpar: Vec>, pub rpar: Vec>, } impl<'a> Codegen<'a> for BooleanOperation<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.left.codegen(state); self.operator.codegen(state); self.right.codegen(state); }) } } impl<'r, 'a> Inflate<'a> for DeflatedBooleanOperation<'r, 'a> { type Inflated = BooleanOperation<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let left = self.left.inflate(config)?; let operator = self.operator.inflate(config)?; let right = self.right.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { left, operator, right, lpar, rpar, }) } } #[cst_node(ParenthesizedNode)] pub struct Call<'a> { pub func: Box>, pub args: Vec>, pub lpar: Vec>, pub rpar: Vec>, pub whitespace_after_func: ParenthesizableWhitespace<'a>, pub whitespace_before_args: ParenthesizableWhitespace<'a>, pub(crate) lpar_tok: TokenRef<'a>, pub(crate) rpar_tok: TokenRef<'a>, } impl<'r, 'a> Inflate<'a> for DeflatedCall<'r, 'a> { type Inflated = Call<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let func = self.func.inflate(config)?; let whitespace_after_func = parse_parenthesizable_whitespace( config, &mut (*self.lpar_tok).whitespace_before.borrow_mut(), )?; let whitespace_before_args = parse_parenthesizable_whitespace( config, &mut (*self.lpar_tok).whitespace_after.borrow_mut(), )?; let mut args = self.args.inflate(config)?; if let Some(arg) = args.last_mut() { if arg.comma.is_none() { arg.whitespace_after_arg = parse_parenthesizable_whitespace( config, &mut (*self.rpar_tok).whitespace_before.borrow_mut(), )?; } } let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { func, args, lpar, rpar, whitespace_after_func, whitespace_before_args, }) } } impl<'a> Codegen<'a> for Call<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.func.codegen(state); self.whitespace_after_func.codegen(state); state.add_token("("); self.whitespace_before_args.codegen(state); let arg_len = self.args.len(); for (i, arg) in self.args.iter().enumerate() { arg.codegen(state, i + 1 < arg_len); } state.add_token(")"); }) } } #[cst_node(ParenthesizedNode)] pub struct Attribute<'a> { pub value: Box>, pub attr: Name<'a>, pub dot: Dot<'a>, pub lpar: Vec>, pub rpar: Vec>, } impl<'r, 'a> Inflate<'a> for DeflatedAttribute<'r, 'a> { type Inflated = Attribute<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let value = self.value.inflate(config)?; let dot = self.dot.inflate(config)?; let attr = self.attr.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { value, attr, dot, lpar, rpar, }) } } impl<'a> Codegen<'a> for Attribute<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.value.codegen(state); self.dot.codegen(state); self.attr.codegen(state); }) } } #[cst_node(Codegen, Inflate)] pub enum NameOrAttribute<'a> { N(Box>), A(Box>), } impl<'r, 'a> std::convert::From> for DeflatedExpression<'r, 'a> { fn from(x: DeflatedNameOrAttribute<'r, 'a>) -> Self { match x { DeflatedNameOrAttribute::N(n) => Self::Name(n), DeflatedNameOrAttribute::A(a) => Self::Attribute(a), } } } #[cst_node] pub struct ComparisonTarget<'a> { pub operator: CompOp<'a>, pub comparator: Expression<'a>, } impl<'a> Codegen<'a> for ComparisonTarget<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.operator.codegen(state); self.comparator.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedComparisonTarget<'r, 'a> { type Inflated = ComparisonTarget<'a>; fn inflate(self, config: &Config<'a>) -> Result { let operator = self.operator.inflate(config)?; let comparator = self.comparator.inflate(config)?; Ok(Self::Inflated { operator, comparator, }) } } #[cst_node(ParenthesizedNode)] pub struct StarredElement<'a> { pub value: Box>, pub comma: Option>, pub lpar: Vec>, pub rpar: Vec>, pub whitespace_before_value: ParenthesizableWhitespace<'a>, pub(crate) star_tok: TokenRef<'a>, } impl<'r, 'a> DeflatedStarredElement<'r, 'a> { pub fn inflate_element(self, config: &Config<'a>, is_last: bool) -> Result> { let lpar = self.lpar.inflate(config)?; let whitespace_before_value = parse_parenthesizable_whitespace( config, &mut (*self.star_tok).whitespace_after.borrow_mut(), )?; let value = self.value.inflate(config)?; let rpar = self.rpar.inflate(config)?; let comma = if is_last { self.comma.map(|c| c.inflate_before(config)).transpose() } else { self.comma.inflate(config) }?; Ok(StarredElement { value, comma, lpar, rpar, whitespace_before_value, }) } } impl<'r, 'a> Inflate<'a> for DeflatedStarredElement<'r, 'a> { type Inflated = StarredElement<'a>; fn inflate(self, config: &Config<'a>) -> Result { self.inflate_element(config, false) } } impl<'a> Codegen<'a> for StarredElement<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { state.add_token("*"); self.whitespace_before_value.codegen(state); self.value.codegen(state); }); if let Some(comma) = &self.comma { comma.codegen(state); } } } #[allow(clippy::large_enum_variant)] #[cst_node(NoIntoPy)] pub enum Element<'a> { Simple { value: Expression<'a>, comma: Option>, }, Starred(Box>), } impl<'a> Element<'a> { fn codegen( &self, state: &mut CodegenState<'a>, default_comma: bool, default_comma_whitespace: bool, ) { match self { Self::Simple { value, comma } => { value.codegen(state); if let Some(comma) = comma { comma.codegen(state) } } Self::Starred(s) => s.codegen(state), } let maybe_comma = match self { Self::Simple { comma, .. } => comma, Self::Starred(s) => &s.comma, }; if maybe_comma.is_none() && default_comma { state.add_token(if default_comma_whitespace { ", " } else { "," }); } } } impl<'r, 'a> DeflatedElement<'r, 'a> { pub fn inflate_element(self, config: &Config<'a>, is_last: bool) -> Result> { Ok(match self { Self::Starred(s) => Element::Starred(Box::new(s.inflate_element(config, is_last)?)), Self::Simple { value, comma } => Element::Simple { value: value.inflate(config)?, comma: if is_last { comma.map(|c| c.inflate_before(config)).transpose()? } else { comma.inflate(config)? }, }, }) } } impl<'r, 'a> WithComma<'r, 'a> for DeflatedElement<'r, 'a> { fn with_comma(self, comma: DeflatedComma<'r, 'a>) -> Self { let comma = Some(comma); match self { Self::Simple { value, .. } => Self::Simple { comma, value }, Self::Starred(mut s) => { s.comma = comma; Self::Starred(s) } } } } impl<'r, 'a> std::convert::From> for DeflatedElement<'r, 'a> { fn from(e: DeflatedExpression<'r, 'a>) -> Self { match e { DeflatedExpression::StarredElement(e) => Self::Starred(e), value => Self::Simple { value, comma: None }, } } } #[cst_node(ParenthesizedNode, Default)] pub struct Tuple<'a> { pub elements: Vec>, pub lpar: Vec>, pub rpar: Vec>, } impl<'r, 'a> Inflate<'a> for DeflatedTuple<'r, 'a> { type Inflated = Tuple<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let len = self.elements.len(); let elements = self .elements .into_iter() .enumerate() .map(|(idx, el)| el.inflate_element(config, idx + 1 == len)) .collect::>>()?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { elements, lpar, rpar, }) } } impl<'a> Codegen<'a> for Tuple<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { let len = self.elements.len(); if len == 1 { self.elements.first().unwrap().codegen(state, true, false); } else { for (idx, el) in self.elements.iter().enumerate() { el.codegen(state, idx < len - 1, true); } } }); } } #[cst_node(ParenthesizedNode)] pub struct GeneratorExp<'a> { pub elt: Box>, pub for_in: Box>, pub lpar: Vec>, pub rpar: Vec>, } impl<'a> Codegen<'a> for GeneratorExp<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.elt.codegen(state); self.for_in.codegen(state); }) } } impl<'r, 'a> Inflate<'a> for DeflatedGeneratorExp<'r, 'a> { type Inflated = GeneratorExp<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let elt = self.elt.inflate(config)?; let for_in = self.for_in.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { elt, for_in, lpar, rpar, }) } } #[cst_node(ParenthesizedNode)] pub struct ListComp<'a> { pub elt: Box>, pub for_in: Box>, pub lbracket: LeftSquareBracket<'a>, pub rbracket: RightSquareBracket<'a>, pub lpar: Vec>, pub rpar: Vec>, } impl<'a> Codegen<'a> for ListComp<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.lbracket.codegen(state); self.elt.codegen(state); self.for_in.codegen(state); self.rbracket.codegen(state); }) } } impl<'r, 'a> Inflate<'a> for DeflatedListComp<'r, 'a> { type Inflated = ListComp<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let lbracket = self.lbracket.inflate(config)?; let elt = self.elt.inflate(config)?; let for_in = self.for_in.inflate(config)?; let rbracket = self.rbracket.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { elt, for_in, lbracket, rbracket, lpar, rpar, }) } } #[cst_node] #[derive(Default)] pub struct LeftSquareBracket<'a> { pub whitespace_after: ParenthesizableWhitespace<'a>, pub(crate) tok: TokenRef<'a>, } impl<'a> Codegen<'a> for LeftSquareBracket<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("["); self.whitespace_after.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedLeftSquareBracket<'r, 'a> { type Inflated = LeftSquareBracket<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_after.borrow_mut(), )?; Ok(Self::Inflated { whitespace_after }) } } #[cst_node] #[derive(Default)] pub struct RightSquareBracket<'a> { pub whitespace_before: ParenthesizableWhitespace<'a>, pub(crate) tok: TokenRef<'a>, } impl<'a> Codegen<'a> for RightSquareBracket<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.whitespace_before.codegen(state); state.add_token("]"); } } impl<'r, 'a> Inflate<'a> for DeflatedRightSquareBracket<'r, 'a> { type Inflated = RightSquareBracket<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_before.borrow_mut(), )?; Ok(Self::Inflated { whitespace_before }) } } #[cst_node(ParenthesizedNode)] pub struct SetComp<'a> { pub elt: Box>, pub for_in: Box>, pub lbrace: LeftCurlyBrace<'a>, pub rbrace: RightCurlyBrace<'a>, pub lpar: Vec>, pub rpar: Vec>, } impl<'r, 'a> Inflate<'a> for DeflatedSetComp<'r, 'a> { type Inflated = SetComp<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let lbrace = self.lbrace.inflate(config)?; let elt = self.elt.inflate(config)?; let for_in = self.for_in.inflate(config)?; let rbrace = self.rbrace.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { elt, for_in, lbrace, rbrace, lpar, rpar, }) } } impl<'a> Codegen<'a> for SetComp<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.lbrace.codegen(state); self.elt.codegen(state); self.for_in.codegen(state); self.rbrace.codegen(state); }) } } #[cst_node(ParenthesizedNode)] pub struct DictComp<'a> { pub key: Box>, pub value: Box>, pub for_in: Box>, pub lbrace: LeftCurlyBrace<'a>, pub rbrace: RightCurlyBrace<'a>, pub lpar: Vec>, pub rpar: Vec>, pub whitespace_before_colon: ParenthesizableWhitespace<'a>, pub whitespace_after_colon: ParenthesizableWhitespace<'a>, pub(crate) colon_tok: TokenRef<'a>, } impl<'r, 'a> Inflate<'a> for DeflatedDictComp<'r, 'a> { type Inflated = DictComp<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let lbrace = self.lbrace.inflate(config)?; let key = self.key.inflate(config)?; let whitespace_before_colon = parse_parenthesizable_whitespace( config, &mut (*self.colon_tok).whitespace_before.borrow_mut(), )?; let whitespace_after_colon = parse_parenthesizable_whitespace( config, &mut (*self.colon_tok).whitespace_after.borrow_mut(), )?; let value = self.value.inflate(config)?; let for_in = self.for_in.inflate(config)?; let rbrace = self.rbrace.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { key, value, for_in, lbrace, rbrace, lpar, rpar, whitespace_before_colon, whitespace_after_colon, }) } } impl<'a> Codegen<'a> for DictComp<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.lbrace.codegen(state); self.key.codegen(state); self.whitespace_before_colon.codegen(state); state.add_token(":"); self.whitespace_after_colon.codegen(state); self.value.codegen(state); self.for_in.codegen(state); self.rbrace.codegen(state); }) } } #[cst_node] pub struct LeftCurlyBrace<'a> { pub whitespace_after: ParenthesizableWhitespace<'a>, pub(crate) tok: TokenRef<'a>, } impl<'a> Default for LeftCurlyBrace<'a> { fn default() -> Self { Self { whitespace_after: Default::default(), } } } impl<'r, 'a> Inflate<'a> for DeflatedLeftCurlyBrace<'r, 'a> { type Inflated = LeftCurlyBrace<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_after.borrow_mut(), )?; Ok(Self::Inflated { whitespace_after }) } } impl<'a> Codegen<'a> for LeftCurlyBrace<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("{"); self.whitespace_after.codegen(state); } } #[cst_node] pub struct RightCurlyBrace<'a> { pub whitespace_before: ParenthesizableWhitespace<'a>, pub(crate) tok: TokenRef<'a>, } impl<'a> Default for RightCurlyBrace<'a> { fn default() -> Self { Self { whitespace_before: Default::default(), } } } impl<'r, 'a> Inflate<'a> for DeflatedRightCurlyBrace<'r, 'a> { type Inflated = RightCurlyBrace<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_before.borrow_mut(), )?; Ok(Self::Inflated { whitespace_before }) } } impl<'a> Codegen<'a> for RightCurlyBrace<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.whitespace_before.codegen(state); state.add_token("}"); } } #[cst_node] pub struct CompFor<'a> { pub target: AssignTargetExpression<'a>, pub iter: Expression<'a>, pub ifs: Vec>, pub inner_for_in: Option>>, pub asynchronous: Option>, pub whitespace_before: ParenthesizableWhitespace<'a>, pub whitespace_after_for: ParenthesizableWhitespace<'a>, pub whitespace_before_in: ParenthesizableWhitespace<'a>, pub whitespace_after_in: ParenthesizableWhitespace<'a>, pub(crate) async_tok: Option>, pub(crate) for_tok: TokenRef<'a>, pub(crate) in_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for CompFor<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.whitespace_before.codegen(state); if let Some(asynchronous) = &self.asynchronous { asynchronous.codegen(state); } state.add_token("for"); self.whitespace_after_for.codegen(state); self.target.codegen(state); self.whitespace_before_in.codegen(state); state.add_token("in"); self.whitespace_after_in.codegen(state); self.iter.codegen(state); for if_ in &self.ifs { if_.codegen(state); } if let Some(inner) = &self.inner_for_in { inner.codegen(state); } } } impl<'r, 'a> Inflate<'a> for DeflatedCompFor<'r, 'a> { type Inflated = CompFor<'a>; fn inflate(mut self, config: &Config<'a>) -> Result { let mut whitespace_before = parse_parenthesizable_whitespace( config, &mut (*self.for_tok).whitespace_before.borrow_mut(), )?; let asynchronous = if let Some(asy_tok) = self.async_tok.as_mut() { // If there is an async keyword, the start of the CompFor expression is // considered to be this keyword, so whitespace_before needs to adjust but // Asynchronous will own the whitespace before the for token. let mut asy_whitespace_after = parse_parenthesizable_whitespace( config, &mut asy_tok.whitespace_before.borrow_mut(), )?; swap(&mut asy_whitespace_after, &mut whitespace_before); Some(Asynchronous { whitespace_after: asy_whitespace_after, }) } else { None }; let whitespace_after_for = parse_parenthesizable_whitespace( config, &mut (*self.for_tok).whitespace_after.borrow_mut(), )?; let target = self.target.inflate(config)?; let whitespace_before_in = parse_parenthesizable_whitespace( config, &mut (*self.in_tok).whitespace_before.borrow_mut(), )?; let whitespace_after_in = parse_parenthesizable_whitespace( config, &mut (*self.in_tok).whitespace_after.borrow_mut(), )?; let iter = self.iter.inflate(config)?; let ifs = self.ifs.inflate(config)?; let inner_for_in = self.inner_for_in.inflate(config)?; Ok(Self::Inflated { target, iter, ifs, inner_for_in, asynchronous, whitespace_before, whitespace_after_for, whitespace_before_in, whitespace_after_in, }) } } #[cst_node] pub struct Asynchronous<'a> { pub whitespace_after: ParenthesizableWhitespace<'a>, } impl<'a> Codegen<'a> for Asynchronous<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("async"); self.whitespace_after.codegen(state); } } pub(crate) fn make_async<'r, 'a>() -> DeflatedAsynchronous<'r, 'a> { DeflatedAsynchronous { _phantom: Default::default(), } } #[cst_node] pub struct CompIf<'a> { pub test: Expression<'a>, pub whitespace_before: ParenthesizableWhitespace<'a>, pub whitespace_before_test: ParenthesizableWhitespace<'a>, pub(crate) if_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for CompIf<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.whitespace_before.codegen(state); state.add_token("if"); self.whitespace_before_test.codegen(state); self.test.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedCompIf<'r, 'a> { type Inflated = CompIf<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*self.if_tok).whitespace_before.borrow_mut(), )?; let whitespace_before_test = parse_parenthesizable_whitespace( config, &mut (*self.if_tok).whitespace_after.borrow_mut(), )?; let test = self.test.inflate(config)?; Ok(Self::Inflated { test, whitespace_before, whitespace_before_test, }) } } #[cst_node(ParenthesizedNode)] pub struct List<'a> { pub elements: Vec>, pub lbracket: LeftSquareBracket<'a>, pub rbracket: RightSquareBracket<'a>, pub lpar: Vec>, pub rpar: Vec>, } impl<'r, 'a> Inflate<'a> for DeflatedList<'r, 'a> { type Inflated = List<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let lbracket = self.lbracket.inflate(config)?; let len = self.elements.len(); let elements = self .elements .into_iter() .enumerate() .map(|(idx, el)| el.inflate_element(config, idx + 1 == len)) .collect::>>()?; let rbracket = if !elements.is_empty() { // lbracket owns all the whitespace if there are no elements self.rbracket.inflate(config)? } else { Default::default() }; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { elements, lbracket, rbracket, lpar, rpar, }) } } impl<'a> Codegen<'a> for List<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.lbracket.codegen(state); let len = self.elements.len(); for (idx, el) in self.elements.iter().enumerate() { el.codegen(state, idx < len - 1, true); } self.rbracket.codegen(state); }) } } #[cst_node(ParenthesizedNode)] pub struct Set<'a> { pub elements: Vec>, pub lbrace: LeftCurlyBrace<'a>, pub rbrace: RightCurlyBrace<'a>, pub lpar: Vec>, pub rpar: Vec>, } impl<'r, 'a> Inflate<'a> for DeflatedSet<'r, 'a> { type Inflated = Set<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let lbrace = self.lbrace.inflate(config)?; let len = self.elements.len(); let elements = self .elements .into_iter() .enumerate() .map(|(idx, el)| el.inflate_element(config, idx + 1 == len)) .collect::>>()?; let rbrace = if !elements.is_empty() { self.rbrace.inflate(config)? } else { Default::default() }; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { elements, lbrace, rbrace, lpar, rpar, }) } } impl<'a> Codegen<'a> for Set<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.lbrace.codegen(state); let len = self.elements.len(); for (idx, el) in self.elements.iter().enumerate() { el.codegen(state, idx < len - 1, true); } self.rbrace.codegen(state); }) } } #[cst_node(ParenthesizedNode)] pub struct Dict<'a> { pub elements: Vec>, pub lbrace: LeftCurlyBrace<'a>, pub rbrace: RightCurlyBrace<'a>, pub lpar: Vec>, pub rpar: Vec>, } impl<'r, 'a> Inflate<'a> for DeflatedDict<'r, 'a> { type Inflated = Dict<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let lbrace = self.lbrace.inflate(config)?; let len = self.elements.len(); let elements = self .elements .into_iter() .enumerate() .map(|(idx, el)| el.inflate_element(config, idx + 1 == len)) .collect::>>()?; let rbrace = if !elements.is_empty() { self.rbrace.inflate(config)? } else { Default::default() }; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { elements, lbrace, rbrace, lpar, rpar, }) } } impl<'a> Codegen<'a> for Dict<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.lbrace.codegen(state); let len = self.elements.len(); for (idx, el) in self.elements.iter().enumerate() { el.codegen(state, idx < len - 1, true); } self.rbrace.codegen(state); }) } } #[cst_node(NoIntoPy)] pub enum DictElement<'a> { Simple { key: Expression<'a>, value: Expression<'a>, comma: Option>, whitespace_before_colon: ParenthesizableWhitespace<'a>, whitespace_after_colon: ParenthesizableWhitespace<'a>, colon_tok: TokenRef<'a>, }, Starred(StarredDictElement<'a>), } impl<'r, 'a> DeflatedDictElement<'r, 'a> { pub fn inflate_element( self, config: &Config<'a>, last_element: bool, ) -> Result> { Ok(match self { Self::Starred(s) => DictElement::Starred(s.inflate_element(config, last_element)?), Self::Simple { key, value, comma, colon_tok, .. } => { let whitespace_before_colon = parse_parenthesizable_whitespace( config, &mut colon_tok.whitespace_before.borrow_mut(), )?; let whitespace_after_colon = parse_parenthesizable_whitespace( config, &mut colon_tok.whitespace_after.borrow_mut(), )?; DictElement::Simple { key: key.inflate(config)?, whitespace_before_colon, whitespace_after_colon, value: value.inflate(config)?, comma: if last_element { comma.map(|c| c.inflate_before(config)).transpose() } else { comma.inflate(config) }?, } } }) } } impl<'a> DictElement<'a> { fn codegen( &self, state: &mut CodegenState<'a>, default_comma: bool, default_comma_whitespace: bool, ) { match self { Self::Simple { key, value, comma, whitespace_before_colon, whitespace_after_colon, .. } => { key.codegen(state); whitespace_before_colon.codegen(state); state.add_token(":"); whitespace_after_colon.codegen(state); value.codegen(state); if let Some(comma) = comma { comma.codegen(state) } } Self::Starred(s) => s.codegen(state), } let maybe_comma = match self { Self::Simple { comma, .. } => comma, Self::Starred(s) => &s.comma, }; if maybe_comma.is_none() && default_comma { state.add_token(if default_comma_whitespace { ", " } else { "," }); } } } impl<'r, 'a> WithComma<'r, 'a> for DeflatedDictElement<'r, 'a> { fn with_comma(self, comma: DeflatedComma<'r, 'a>) -> Self { let comma = Some(comma); match self { Self::Starred(s) => Self::Starred(DeflatedStarredDictElement { comma, ..s }), Self::Simple { key, value, colon_tok, .. } => Self::Simple { comma, key, value, colon_tok, }, } } } #[cst_node] pub struct StarredDictElement<'a> { pub value: Expression<'a>, pub comma: Option>, pub whitespace_before_value: ParenthesizableWhitespace<'a>, pub(crate) star_tok: TokenRef<'a>, } impl<'r, 'a> DeflatedStarredDictElement<'r, 'a> { fn inflate_element( self, config: &Config<'a>, last_element: bool, ) -> Result> { let whitespace_before_value = parse_parenthesizable_whitespace( config, &mut (*self.star_tok).whitespace_after.borrow_mut(), )?; let value = self.value.inflate(config)?; let comma = if last_element { self.comma.map(|c| c.inflate_before(config)).transpose() } else { self.comma.inflate(config) }?; Ok(StarredDictElement { value, comma, whitespace_before_value, }) } } impl<'a> Codegen<'a> for StarredDictElement<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("**"); self.whitespace_before_value.codegen(state); self.value.codegen(state); if let Some(comma) = &self.comma { comma.codegen(state); } } } #[cst_node(Codegen, Inflate)] pub enum BaseSlice<'a> { Index(Box>), Slice(Box>), } #[cst_node] pub struct Index<'a> { pub value: Expression<'a>, pub star: Option<&'a str>, pub whitespace_after_star: Option>, pub(crate) star_tok: Option>, } impl<'r, 'a> Inflate<'a> for DeflatedIndex<'r, 'a> { type Inflated = Index<'a>; fn inflate(mut self, config: &Config<'a>) -> Result { let (star, whitespace_after_star) = if let Some(star_tok) = self.star_tok.as_mut() { ( Some(star_tok.string), Some(parse_parenthesizable_whitespace( config, &mut star_tok.whitespace_after.borrow_mut(), )?), ) } else { (None, None) }; let value = self.value.inflate(config)?; Ok(Self::Inflated { value, star, whitespace_after_star, }) } } impl<'a> Codegen<'a> for Index<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { if let Some(star) = self.star { state.add_token(star); } self.whitespace_after_star.codegen(state); self.value.codegen(state); } } #[cst_node] pub struct Slice<'a> { #[cfg_attr(feature = "py", no_py_default)] pub lower: Option>, #[cfg_attr(feature = "py", no_py_default)] pub upper: Option>, pub step: Option>, pub first_colon: Colon<'a>, pub second_colon: Option>, } impl<'r, 'a> Inflate<'a> for DeflatedSlice<'r, 'a> { type Inflated = Slice<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lower = self.lower.inflate(config)?; let first_colon = self.first_colon.inflate(config)?; let upper = self.upper.inflate(config)?; let second_colon = self.second_colon.inflate(config)?; let step = self.step.inflate(config)?; Ok(Self::Inflated { lower, upper, step, first_colon, second_colon, }) } } impl<'a> Codegen<'a> for Slice<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { if let Some(lower) = &self.lower { lower.codegen(state); } self.first_colon.codegen(state); if let Some(upper) = &self.upper { upper.codegen(state); } if let Some(second_colon) = &self.second_colon { second_colon.codegen(state); } else if self.step.is_some() { state.add_token(";"); } if let Some(step) = &self.step { step.codegen(state); } } } #[cst_node] pub struct SubscriptElement<'a> { pub slice: BaseSlice<'a>, pub comma: Option>, } impl<'r, 'a> Inflate<'a> for DeflatedSubscriptElement<'r, 'a> { type Inflated = SubscriptElement<'a>; fn inflate(self, config: &Config<'a>) -> Result { let slice = self.slice.inflate(config)?; let comma = self.comma.inflate(config)?; Ok(Self::Inflated { slice, comma }) } } impl<'a> Codegen<'a> for SubscriptElement<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.slice.codegen(state); if let Some(comma) = &self.comma { comma.codegen(state); } } } #[cst_node(ParenthesizedNode)] pub struct Subscript<'a> { pub value: Box>, pub slice: Vec>, pub lbracket: LeftSquareBracket<'a>, pub rbracket: RightSquareBracket<'a>, pub lpar: Vec>, pub rpar: Vec>, pub whitespace_after_value: ParenthesizableWhitespace<'a>, } impl<'r, 'a> Inflate<'a> for DeflatedSubscript<'r, 'a> { type Inflated = Subscript<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let value = self.value.inflate(config)?; let whitespace_after_value = parse_parenthesizable_whitespace( config, &mut self.lbracket.tok.whitespace_before.borrow_mut(), )?; let lbracket = self.lbracket.inflate(config)?; let slice = self.slice.inflate(config)?; let rbracket = self.rbracket.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { value, slice, lbracket, rbracket, lpar, rpar, whitespace_after_value, }) } } impl<'a> Codegen<'a> for Subscript<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.value.codegen(state); self.whitespace_after_value.codegen(state); self.lbracket.codegen(state); let len = self.slice.len(); for (i, slice) in self.slice.iter().enumerate() { slice.codegen(state); if slice.comma.is_none() && i + 1 < len { state.add_token(", ") } } self.rbracket.codegen(state); }) } } #[cst_node(ParenthesizedNode)] pub struct IfExp<'a> { pub test: Box>, pub body: Box>, pub orelse: Box>, pub lpar: Vec>, pub rpar: Vec>, pub whitespace_before_if: ParenthesizableWhitespace<'a>, pub whitespace_after_if: ParenthesizableWhitespace<'a>, pub whitespace_before_else: ParenthesizableWhitespace<'a>, pub whitespace_after_else: ParenthesizableWhitespace<'a>, pub(crate) if_tok: TokenRef<'a>, pub(crate) else_tok: TokenRef<'a>, } impl<'r, 'a> Inflate<'a> for DeflatedIfExp<'r, 'a> { type Inflated = IfExp<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let body = self.body.inflate(config)?; let whitespace_before_if = parse_parenthesizable_whitespace( config, &mut (*self.if_tok).whitespace_before.borrow_mut(), )?; let whitespace_after_if = parse_parenthesizable_whitespace( config, &mut (*self.if_tok).whitespace_after.borrow_mut(), )?; let test = self.test.inflate(config)?; let whitespace_before_else = parse_parenthesizable_whitespace( config, &mut (*self.else_tok).whitespace_before.borrow_mut(), )?; let whitespace_after_else = parse_parenthesizable_whitespace( config, &mut (*self.else_tok).whitespace_after.borrow_mut(), )?; let orelse = self.orelse.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { test, body, orelse, lpar, rpar, whitespace_before_if, whitespace_after_if, whitespace_before_else, whitespace_after_else, }) } } impl<'a> Codegen<'a> for IfExp<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.body.codegen(state); self.whitespace_before_if.codegen(state); state.add_token("if"); self.whitespace_after_if.codegen(state); self.test.codegen(state); self.whitespace_before_else.codegen(state); state.add_token("else"); self.whitespace_after_else.codegen(state); self.orelse.codegen(state); }) } } #[cst_node(ParenthesizedNode)] pub struct Lambda<'a> { pub params: Box>, pub body: Box>, pub colon: Colon<'a>, pub lpar: Vec>, pub rpar: Vec>, pub whitespace_after_lambda: Option>, pub(crate) lambda_tok: TokenRef<'a>, } impl<'r, 'a> Inflate<'a> for DeflatedLambda<'r, 'a> { type Inflated = Lambda<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let whitespace_after_lambda = if !self.params.is_empty() { Some(parse_parenthesizable_whitespace( config, &mut (*self.lambda_tok).whitespace_after.borrow_mut(), )?) } else { Default::default() }; let mut params = self.params.inflate(config)?; adjust_parameters_trailing_whitespace(config, &mut params, &self.colon.tok)?; let colon = self.colon.inflate(config)?; let body = self.body.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { params, body, colon, lpar, rpar, whitespace_after_lambda, }) } } impl<'a> Codegen<'a> for Lambda<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { state.add_token("lambda"); if let Some(ws) = &self.whitespace_after_lambda { ws.codegen(state); } else if !self.params.is_empty() { // there's one or more params, add a space state.add_token(" ") } self.params.codegen(state); self.colon.codegen(state); self.body.codegen(state); }) } } #[cst_node] pub struct From<'a> { pub item: Expression<'a>, pub whitespace_before_from: Option>, pub whitespace_after_from: ParenthesizableWhitespace<'a>, pub(crate) tok: TokenRef<'a>, } impl<'a> From<'a> { pub fn codegen(&self, state: &mut CodegenState<'a>, default_space: &'a str) { if let Some(ws) = &self.whitespace_before_from { ws.codegen(state); } else { state.add_token(default_space); } state.add_token("from"); self.whitespace_after_from.codegen(state); self.item.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedFrom<'r, 'a> { type Inflated = From<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_before_from = Some(parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_before.borrow_mut(), )?); let whitespace_after_from = parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_after.borrow_mut(), )?; let item = self.item.inflate(config)?; Ok(Self::Inflated { item, whitespace_before_from, whitespace_after_from, }) } } #[cst_node] pub enum YieldValue<'a> { Expression(Box>), From(Box>), } impl<'r, 'a> Inflate<'a> for DeflatedYieldValue<'r, 'a> { type Inflated = YieldValue<'a>; fn inflate(self, config: &Config<'a>) -> Result { Ok(match self { Self::Expression(e) => Self::Inflated::Expression(e.inflate(config)?), Self::From(e) => { let mut e = e.inflate(config)?; e.whitespace_before_from = None; Self::Inflated::From(e) } }) } } impl<'a> YieldValue<'a> { fn codegen(&self, state: &mut CodegenState<'a>, default_space: &'a str) { match self { Self::Expression(e) => e.codegen(state), Self::From(f) => f.codegen(state, default_space), } } } #[cst_node(ParenthesizedNode)] pub struct Yield<'a> { pub value: Option>>, pub lpar: Vec>, pub rpar: Vec>, pub whitespace_after_yield: Option>, pub(crate) yield_tok: TokenRef<'a>, } impl<'r, 'a> Inflate<'a> for DeflatedYield<'r, 'a> { type Inflated = Yield<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let whitespace_after_yield = if self.value.is_some() { Some(parse_parenthesizable_whitespace( config, &mut (*self.yield_tok).whitespace_after.borrow_mut(), )?) } else { Default::default() }; let value = self.value.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { value, lpar, rpar, whitespace_after_yield, }) } } impl<'a> Codegen<'a> for Yield<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { state.add_token("yield"); if let Some(ws) = &self.whitespace_after_yield { ws.codegen(state); } else if self.value.is_some() { state.add_token(" "); } if let Some(val) = &self.value { val.codegen(state, "") } }) } } #[cst_node(ParenthesizedNode)] pub struct Await<'a> { pub expression: Box>, pub lpar: Vec>, pub rpar: Vec>, pub whitespace_after_await: ParenthesizableWhitespace<'a>, pub(crate) await_tok: TokenRef<'a>, } impl<'r, 'a> Inflate<'a> for DeflatedAwait<'r, 'a> { type Inflated = Await<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let whitespace_after_await = parse_parenthesizable_whitespace( config, &mut (*self.await_tok).whitespace_after.borrow_mut(), )?; let expression = self.expression.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { expression, lpar, rpar, whitespace_after_await, }) } } impl<'a> Codegen<'a> for Await<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { state.add_token("await"); self.whitespace_after_await.codegen(state); self.expression.codegen(state); }) } } #[cst_node(Codegen, Inflate)] pub enum String<'a> { Simple(SimpleString<'a>), Concatenated(ConcatenatedString<'a>), Formatted(FormattedString<'a>), } impl<'r, 'a> std::convert::From> for DeflatedExpression<'r, 'a> { fn from(s: DeflatedString<'r, 'a>) -> Self { match s { DeflatedString::Simple(s) => Self::SimpleString(Box::new(s)), DeflatedString::Concatenated(s) => Self::ConcatenatedString(Box::new(s)), DeflatedString::Formatted(s) => Self::FormattedString(Box::new(s)), } } } #[cst_node(ParenthesizedNode)] pub struct ConcatenatedString<'a> { pub left: Box>, pub right: Box>, pub lpar: Vec>, pub rpar: Vec>, pub whitespace_between: ParenthesizableWhitespace<'a>, // we capture the next token after each string piece so Inflate can extract the // whitespace between individual pieces pub(crate) right_tok: TokenRef<'a>, } impl<'r, 'a> Inflate<'a> for DeflatedConcatenatedString<'r, 'a> { type Inflated = ConcatenatedString<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let left = self.left.inflate(config)?; let whitespace_between = parse_parenthesizable_whitespace( config, &mut (*self.right_tok).whitespace_before.borrow_mut(), )?; let right = self.right.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { left, right, lpar, rpar, whitespace_between, }) } } impl<'a> Codegen<'a> for ConcatenatedString<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.left.codegen(state); self.whitespace_between.codegen(state); self.right.codegen(state); }) } } #[cst_node(ParenthesizedNode, Default)] pub struct SimpleString<'a> { /// The texual representation of the string, including quotes, prefix /// characters, and any escape characters present in the original source code, /// such as ``r"my string\n"``. pub value: &'a str, pub lpar: Vec>, pub rpar: Vec>, } impl<'r, 'a> Inflate<'a> for DeflatedSimpleString<'r, 'a> { type Inflated = SimpleString<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { value: self.value, lpar, rpar, }) } } impl<'a> Codegen<'a> for SimpleString<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| state.add_token(self.value)) } } #[cst_node] pub struct FormattedStringText<'a> { pub value: &'a str, } impl<'r, 'a> Inflate<'a> for DeflatedFormattedStringText<'r, 'a> { type Inflated = FormattedStringText<'a>; fn inflate(self, _config: &Config<'a>) -> Result { Ok(Self::Inflated { value: self.value }) } } impl<'a> Codegen<'a> for FormattedStringText<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token(self.value); } } pub(crate) fn make_fstringtext<'r, 'a>(value: &'a str) -> DeflatedFormattedStringText<'r, 'a> { DeflatedFormattedStringText { value, _phantom: Default::default(), } } #[cst_node] pub struct FormattedStringExpression<'a> { pub expression: Expression<'a>, pub conversion: Option<&'a str>, pub format_spec: Option>>, pub whitespace_before_expression: ParenthesizableWhitespace<'a>, pub whitespace_after_expression: ParenthesizableWhitespace<'a>, pub equal: Option>, pub(crate) lbrace_tok: TokenRef<'a>, // This is None if there's an equal sign, otherwise it's the first token of // (conversion, format spec, right brace) in that order pub(crate) after_expr_tok: Option>, } impl<'r, 'a> Inflate<'a> for DeflatedFormattedStringExpression<'r, 'a> { type Inflated = FormattedStringExpression<'a>; fn inflate(mut self, config: &Config<'a>) -> Result { let whitespace_before_expression = parse_parenthesizable_whitespace( config, &mut (*self.lbrace_tok).whitespace_after.borrow_mut(), )?; let expression = self.expression.inflate(config)?; let equal = self.equal.inflate(config)?; let whitespace_after_expression = if let Some(after_expr_tok) = self.after_expr_tok.as_mut() { parse_parenthesizable_whitespace( config, &mut after_expr_tok.whitespace_before.borrow_mut(), )? } else { Default::default() }; let format_spec = self.format_spec.inflate(config)?; Ok(Self::Inflated { expression, conversion: self.conversion, format_spec, whitespace_before_expression, whitespace_after_expression, equal, }) } } impl<'a> Codegen<'a> for FormattedStringExpression<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("{"); self.whitespace_before_expression.codegen(state); self.expression.codegen(state); if let Some(eq) = &self.equal { eq.codegen(state); } self.whitespace_after_expression.codegen(state); if let Some(conv) = &self.conversion { state.add_token("!"); state.add_token(conv); } if let Some(specs) = &self.format_spec { state.add_token(":"); for spec in specs { spec.codegen(state); } } state.add_token("}"); } } #[cst_node(Codegen, Inflate)] pub enum FormattedStringContent<'a> { Text(FormattedStringText<'a>), Expression(Box>), } #[cst_node(ParenthesizedNode)] pub struct FormattedString<'a> { pub parts: Vec>, pub start: &'a str, pub end: &'a str, pub lpar: Vec>, pub rpar: Vec>, } impl<'r, 'a> Inflate<'a> for DeflatedFormattedString<'r, 'a> { type Inflated = FormattedString<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let parts = self.parts.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { parts, start: self.start, end: self.end, lpar, rpar, }) } } impl<'a> Codegen<'a> for FormattedString<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { state.add_token(self.start); for part in &self.parts { part.codegen(state); } state.add_token(self.end); }) } } #[cst_node(ParenthesizedNode)] pub struct NamedExpr<'a> { pub target: Box>, pub value: Box>, pub lpar: Vec>, pub rpar: Vec>, pub whitespace_before_walrus: ParenthesizableWhitespace<'a>, pub whitespace_after_walrus: ParenthesizableWhitespace<'a>, pub(crate) walrus_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for NamedExpr<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.target.codegen(state); self.whitespace_before_walrus.codegen(state); state.add_token(":="); self.whitespace_after_walrus.codegen(state); self.value.codegen(state); }) } } impl<'r, 'a> Inflate<'a> for DeflatedNamedExpr<'r, 'a> { type Inflated = NamedExpr<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let target = self.target.inflate(config)?; let whitespace_before_walrus = parse_parenthesizable_whitespace( config, &mut self.walrus_tok.whitespace_before.borrow_mut(), )?; let whitespace_after_walrus = parse_parenthesizable_whitespace( config, &mut self.walrus_tok.whitespace_after.borrow_mut(), )?; let value = self.value.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { target, value, lpar, rpar, whitespace_before_walrus, whitespace_after_walrus, }) } } #[cfg(feature = "py")] mod py { use pyo3::types::PyModule; use super::*; use crate::nodes::traits::py::TryIntoPy; // TODO: this could be a derive helper attribute to override the python class name impl<'a> TryIntoPy for Element<'a> { fn try_into_py(self, py: pyo3::Python) -> pyo3::PyResult { match self { Self::Starred(s) => s.try_into_py(py), Self::Simple { value, comma } => { let libcst = PyModule::import(py, "libcst")?; let kwargs = [ Some(("value", value.try_into_py(py)?)), comma .map(|x| x.try_into_py(py)) .transpose()? .map(|x| ("comma", x)), ] .iter() .filter(|x| x.is_some()) .map(|x| x.as_ref().unwrap()) .collect::>() .into_py_dict(py); Ok(libcst .getattr("Element") .expect("no Element found in libcst") .call((), Some(kwargs))? .into()) } } } } // TODO: this could be a derive helper attribute to override the python class name impl<'a> TryIntoPy for DictElement<'a> { fn try_into_py(self, py: pyo3::Python) -> pyo3::PyResult { match self { Self::Starred(s) => s.try_into_py(py), Self::Simple { key, value, comma, whitespace_after_colon, whitespace_before_colon, .. } => { let libcst = PyModule::import(py, "libcst")?; let kwargs = [ Some(("key", key.try_into_py(py)?)), Some(("value", value.try_into_py(py)?)), Some(( "whitespace_before_colon", whitespace_before_colon.try_into_py(py)?, )), Some(( "whitespace_after_colon", whitespace_after_colon.try_into_py(py)?, )), comma .map(|x| x.try_into_py(py)) .transpose()? .map(|x| ("comma", x)), ] .iter() .filter(|x| x.is_some()) .map(|x| x.as_ref().unwrap()) .collect::>() .into_py_dict(py); Ok(libcst .getattr("DictElement") .expect("no Element found in libcst") .call((), Some(kwargs))? .into()) } } } } } libcst-0.1.0/src/nodes/inflate_helpers.rs000064400000000000000000000023360072674642500165470ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree use crate::{ nodes::traits::Result, tokenizer::{ whitespace_parser::{parse_parenthesizable_whitespace, Config}, Token, }, Param, Parameters, StarArg, }; pub(crate) fn adjust_parameters_trailing_whitespace<'a>( config: &Config<'a>, parameters: &mut Parameters<'a>, next_tok: &Token<'a>, ) -> Result<()> { let do_adjust = |param: &mut Param<'a>| -> Result<()> { let whitespace_after = parse_parenthesizable_whitespace(config, &mut next_tok.whitespace_before.borrow_mut())?; if param.comma.is_none() { param.whitespace_after_param = whitespace_after; } Ok(()) }; if let Some(param) = &mut parameters.star_kwarg { do_adjust(param)?; } else if let Some(param) = parameters.kwonly_params.last_mut() { do_adjust(param)?; } else if let Some(StarArg::Param(param)) = parameters.star_arg.as_mut() { do_adjust(param)?; } else if let Some(param) = parameters.params.last_mut() { do_adjust(param)?; } Ok(()) } libcst-0.1.0/src/nodes/macros.rs000064400000000000000000000026500072674642500146660ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. /// Generates a function that lazily imports and caches a module's member. This will hold a /// permanent reference to the imported member. Python's module cache is rarely purged though, so /// it typically won't matter. /// /// This cache is cheaper than looking up the module in python's module cache inspecting the /// module's `__dict__` each time you want access to the member. /// /// If you have multiple imports from the same module, we'll call `py.import` once for each member /// of the module. #[macro_export] macro_rules! py_import { ( $module_name:expr, $member_name:expr, $getter_fn:ident ) => { paste::paste! { static [] : pyo3::once_cell::GILOnceCell> = pyo3::once_cell::GILOnceCell::new(); fn $getter_fn<'py>(py: pyo3::Python<'py>) -> pyo3::PyResult<&'py pyo3::PyAny> { Ok([].get_or_init(py, || { Ok(py.import($module_name)?.get($member_name)?.to_object(py)) }) .as_ref() .map_err(|err| err.clone_ref(py))? .as_ref(py)) } } }; } libcst-0.1.0/src/nodes/mod.rs000064400000000000000000000172700072674642500141650ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree pub(crate) mod whitespace; pub use whitespace::{ Comment, EmptyLine, Fakeness, Newline, ParenthesizableWhitespace, ParenthesizedWhitespace, SimpleWhitespace, TrailingWhitespace, }; pub(crate) mod statement; pub use statement::{ AnnAssign, Annotation, AsName, Assert, Assign, AssignTarget, AssignTargetExpression, AugAssign, Break, ClassDef, CompoundStatement, Continue, Decorator, Del, DelTargetExpression, Else, ExceptHandler, ExceptStarHandler, Expr, Finally, For, FunctionDef, Global, If, Import, ImportAlias, ImportFrom, ImportNames, IndentedBlock, Match, MatchAs, MatchCase, MatchClass, MatchKeywordElement, MatchList, MatchMapping, MatchMappingElement, MatchOr, MatchOrElement, MatchPattern, MatchSequence, MatchSequenceElement, MatchSingleton, MatchStar, MatchTuple, MatchValue, NameItem, Nonlocal, OrElse, Pass, Raise, Return, SimpleStatementLine, SimpleStatementSuite, SmallStatement, StarrableMatchSequenceElement, Statement, Suite, Try, TryStar, While, With, WithItem, }; pub(crate) mod expression; pub use expression::{ Arg, Asynchronous, Attribute, Await, BaseSlice, BinaryOperation, BooleanOperation, Call, CompFor, CompIf, Comparison, ComparisonTarget, ConcatenatedString, Dict, DictComp, DictElement, Element, Ellipsis, Expression, Float, FormattedString, FormattedStringContent, FormattedStringExpression, FormattedStringText, From, GeneratorExp, IfExp, Imaginary, Index, Integer, Lambda, LeftCurlyBrace, LeftParen, LeftSquareBracket, List, ListComp, Name, NameOrAttribute, NamedExpr, Param, ParamSlash, ParamStar, Parameters, RightCurlyBrace, RightParen, RightSquareBracket, Set, SetComp, SimpleString, Slice, StarArg, StarredDictElement, StarredElement, String, Subscript, SubscriptElement, Tuple, UnaryOperation, Yield, YieldValue, }; pub(crate) mod op; pub use op::{ AssignEqual, AugOp, BinaryOp, BitOr, BooleanOp, Colon, Comma, CompOp, Dot, ImportStar, Semicolon, UnaryOp, }; pub(crate) mod module; pub use module::Module; mod codegen; pub use codegen::{Codegen, CodegenState}; pub(crate) mod traits; pub use traits::{Inflate, ParenthesizedNode, WithComma, WithLeadingLines}; pub(crate) mod inflate_helpers; pub(crate) mod deflated { pub use super::expression::{ DeflatedArg as Arg, DeflatedAsynchronous as Asynchronous, DeflatedAttribute as Attribute, DeflatedAwait as Await, DeflatedBaseSlice as BaseSlice, DeflatedBinaryOperation as BinaryOperation, DeflatedBooleanOperation as BooleanOperation, DeflatedCall as Call, DeflatedCompFor as CompFor, DeflatedCompIf as CompIf, DeflatedComparison as Comparison, DeflatedComparisonTarget as ComparisonTarget, DeflatedConcatenatedString as ConcatenatedString, DeflatedDict as Dict, DeflatedDictComp as DictComp, DeflatedDictElement as DictElement, DeflatedElement as Element, DeflatedEllipsis as Ellipsis, DeflatedExpression as Expression, DeflatedFloat as Float, DeflatedFormattedString as FormattedString, DeflatedFormattedStringContent as FormattedStringContent, DeflatedFormattedStringExpression as FormattedStringExpression, DeflatedFormattedStringText as FormattedStringText, DeflatedFrom as From, DeflatedGeneratorExp as GeneratorExp, DeflatedIfExp as IfExp, DeflatedImaginary as Imaginary, DeflatedIndex as Index, DeflatedInteger as Integer, DeflatedLambda as Lambda, DeflatedLeftCurlyBrace as LeftCurlyBrace, DeflatedLeftParen as LeftParen, DeflatedLeftSquareBracket as LeftSquareBracket, DeflatedList as List, DeflatedListComp as ListComp, DeflatedName as Name, DeflatedNameOrAttribute as NameOrAttribute, DeflatedNamedExpr as NamedExpr, DeflatedParam as Param, DeflatedParamSlash as ParamSlash, DeflatedParamStar as ParamStar, DeflatedParameters as Parameters, DeflatedRightCurlyBrace as RightCurlyBrace, DeflatedRightParen as RightParen, DeflatedRightSquareBracket as RightSquareBracket, DeflatedSet as Set, DeflatedSetComp as SetComp, DeflatedSimpleString as SimpleString, DeflatedSlice as Slice, DeflatedStarArg as StarArg, DeflatedStarredDictElement as StarredDictElement, DeflatedStarredElement as StarredElement, DeflatedString as String, DeflatedSubscript as Subscript, DeflatedSubscriptElement as SubscriptElement, DeflatedTuple as Tuple, DeflatedUnaryOperation as UnaryOperation, DeflatedYield as Yield, DeflatedYieldValue as YieldValue, }; pub use super::module::DeflatedModule as Module; pub use super::op::{ DeflatedAssignEqual as AssignEqual, DeflatedAugOp as AugOp, DeflatedBinaryOp as BinaryOp, DeflatedBitOr as BitOr, DeflatedBooleanOp as BooleanOp, DeflatedColon as Colon, DeflatedComma as Comma, DeflatedCompOp as CompOp, DeflatedDot as Dot, DeflatedImportStar as ImportStar, DeflatedSemicolon as Semicolon, DeflatedUnaryOp as UnaryOp, }; pub use super::statement::{ DeflatedAnnAssign as AnnAssign, DeflatedAnnotation as Annotation, DeflatedAsName as AsName, DeflatedAssert as Assert, DeflatedAssign as Assign, DeflatedAssignTarget as AssignTarget, DeflatedAssignTargetExpression as AssignTargetExpression, DeflatedAugAssign as AugAssign, DeflatedBreak as Break, DeflatedClassDef as ClassDef, DeflatedCompoundStatement as CompoundStatement, DeflatedContinue as Continue, DeflatedDecorator as Decorator, DeflatedDel as Del, DeflatedDelTargetExpression as DelTargetExpression, DeflatedElse as Else, DeflatedExceptHandler as ExceptHandler, DeflatedExceptStarHandler as ExceptStarHandler, DeflatedExpr as Expr, DeflatedFinally as Finally, DeflatedFor as For, DeflatedFunctionDef as FunctionDef, DeflatedGlobal as Global, DeflatedIf as If, DeflatedImport as Import, DeflatedImportAlias as ImportAlias, DeflatedImportFrom as ImportFrom, DeflatedImportNames as ImportNames, DeflatedIndentedBlock as IndentedBlock, DeflatedMatch as Match, DeflatedMatchAs as MatchAs, DeflatedMatchCase as MatchCase, DeflatedMatchClass as MatchClass, DeflatedMatchKeywordElement as MatchKeywordElement, DeflatedMatchList as MatchList, DeflatedMatchMapping as MatchMapping, DeflatedMatchMappingElement as MatchMappingElement, DeflatedMatchOr as MatchOr, DeflatedMatchOrElement as MatchOrElement, DeflatedMatchPattern as MatchPattern, DeflatedMatchSequence as MatchSequence, DeflatedMatchSequenceElement as MatchSequenceElement, DeflatedMatchSingleton as MatchSingleton, DeflatedMatchStar as MatchStar, DeflatedMatchTuple as MatchTuple, DeflatedMatchValue as MatchValue, DeflatedNameItem as NameItem, DeflatedNonlocal as Nonlocal, DeflatedOrElse as OrElse, DeflatedParamSpec as ParamSpec, DeflatedPass as Pass, DeflatedRaise as Raise, DeflatedReturn as Return, DeflatedSimpleStatementLine as SimpleStatementLine, DeflatedSimpleStatementSuite as SimpleStatementSuite, DeflatedSmallStatement as SmallStatement, DeflatedStarrableMatchSequenceElement as StarrableMatchSequenceElement, DeflatedStatement as Statement, DeflatedSuite as Suite, DeflatedTry as Try, DeflatedTryStar as TryStar, DeflatedTypeAlias as TypeAlias, DeflatedTypeParam as TypeParam, DeflatedTypeParameters as TypeParameters, DeflatedTypeVar as TypeVar, DeflatedTypeVarLike as TypeVarLike, DeflatedTypeVarTuple as TypeVarTuple, DeflatedWhile as While, DeflatedWith as With, DeflatedWithItem as WithItem, }; } libcst-0.1.0/src/nodes/module.rs000064400000000000000000000054410072674642500146700ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. use std::mem::swap; use crate::tokenizer::whitespace_parser::parse_empty_lines; use crate::tokenizer::Token; use crate::{ nodes::{ codegen::{Codegen, CodegenState}, statement::*, whitespace::EmptyLine, }, tokenizer::whitespace_parser::Config, }; use libcst_derive::cst_node; #[cfg(feature = "py")] use libcst_derive::TryIntoPy; use super::traits::{Inflate, Result, WithLeadingLines}; type TokenRef<'r, 'a> = &'r Token<'a>; #[cst_node] pub struct Module<'a> { pub body: Vec>, pub header: Vec>, pub footer: Vec>, pub default_indent: &'a str, pub default_newline: &'a str, pub has_trailing_newline: bool, pub encoding: String, pub(crate) eof_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for Module<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for h in &self.header { h.codegen(state); } for s in &self.body { s.codegen(state); } for nl in &self.footer { nl.codegen(state); } } } impl<'r, 'a> Inflate<'a> for DeflatedModule<'r, 'a> { type Inflated = Module<'a>; fn inflate(self, config: &Config<'a>) -> Result { let default_indent = config.default_indent; let default_newline = config.default_newline; let has_trailing_newline = config.has_trailing_newline(); let mut body = self.body.inflate(config)?; let mut footer = parse_empty_lines( config, &mut (*self.eof_tok).whitespace_before.borrow_mut(), Some(""), )?; let mut header = vec![]; if let Some(stmt) = body.first_mut() { swap(stmt.leading_lines(), &mut header); let mut last_indented = None; for (num, line) in footer.iter().enumerate() { if !line.whitespace.0.is_empty() { last_indented = Some(num); } else if line.comment.is_some() { // This is a non-indented comment. Everything from here should belong in the // footer. break; } } if let Some(num) = last_indented { let (_, rest) = footer.split_at(num); footer = rest.to_vec(); } } else { swap(&mut header, &mut footer); } Ok(Self::Inflated { body, header, footer, default_indent, default_newline, has_trailing_newline, encoding: self.encoding, }) } } libcst-0.1.0/src/nodes/op.rs000064400000000000000000001430530072674642500140230ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. use super::{whitespace::ParenthesizableWhitespace, Codegen, CodegenState}; use crate::{ nodes::traits::{Inflate, Result}, tokenizer::{ whitespace_parser::{parse_parenthesizable_whitespace, parse_simple_whitespace, Config}, Token, }, }; use libcst_derive::cst_node; #[cfg(feature = "py")] use libcst_derive::TryIntoPy; type TokenRef<'r, 'a> = &'r Token<'a>; #[cst_node] pub struct Semicolon<'a> { /// Any space that appears directly before this semicolon. pub whitespace_before: ParenthesizableWhitespace<'a>, /// Any space that appears directly after this semicolon. pub whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] pub(crate) tok: TokenRef<'a>, } impl<'a> Codegen<'a> for Semicolon<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.whitespace_before.codegen(state); state.add_token(";"); self.whitespace_after.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedSemicolon<'r, 'a> { type Inflated = Semicolon<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_before = ParenthesizableWhitespace::SimpleWhitespace( parse_simple_whitespace(config, &mut (*self.tok).whitespace_before.borrow_mut())?, ); let whitespace_after = ParenthesizableWhitespace::SimpleWhitespace( parse_simple_whitespace(config, &mut (*self.tok).whitespace_after.borrow_mut())?, ); Ok(Self::Inflated { whitespace_before, whitespace_after, }) } } #[cst_node] pub struct Comma<'a> { /// Any space that appears directly before this comma. pub whitespace_before: ParenthesizableWhitespace<'a>, /// Any space that appears directly after this comma. pub whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] pub(crate) tok: TokenRef<'a>, } impl<'a> Codegen<'a> for Comma<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.whitespace_before.codegen(state); state.add_token(","); self.whitespace_after.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedComma<'r, 'a> { type Inflated = Comma<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_after.borrow_mut(), )?; Ok(Self::Inflated { whitespace_before, whitespace_after, }) } } impl<'r, 'a> DeflatedComma<'r, 'a> { pub fn inflate_before(self, config: &Config<'a>) -> Result> { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_before.borrow_mut(), )?; let whitespace_after = Default::default(); Ok(Comma { whitespace_before, whitespace_after, }) } } #[cst_node] pub struct AssignEqual<'a> { /// Any space that appears directly before this equal sign. pub whitespace_before: ParenthesizableWhitespace<'a>, /// Any space that appears directly after this equal sign. pub whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] pub(crate) tok: TokenRef<'a>, } impl<'a> Codegen<'a> for AssignEqual<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.whitespace_before.codegen(state); state.add_token("="); self.whitespace_after.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedAssignEqual<'r, 'a> { type Inflated = AssignEqual<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_after.borrow_mut(), )?; Ok(Self::Inflated { whitespace_before, whitespace_after, }) } } #[cst_node] pub struct Dot<'a> { /// Any space that appears directly before this dot. pub whitespace_before: ParenthesizableWhitespace<'a>, /// Any space that appears directly after this dot. pub whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] pub(crate) tok: TokenRef<'a>, } impl<'a> Codegen<'a> for Dot<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.whitespace_before.codegen(state); state.add_token("."); self.whitespace_after.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedDot<'r, 'a> { type Inflated = Dot<'a>; fn inflate(mut self, config: &Config<'a>) -> Result { let whitespace_before = self.inflate_before(config)?; let whitespace_after = self.inflate_after(config)?; Ok(Self::Inflated { whitespace_before, whitespace_after, }) } } impl<'r, 'a> DeflatedDot<'r, 'a> { fn inflate_before(&mut self, config: &Config<'a>) -> Result> { parse_parenthesizable_whitespace(config, &mut (*self.tok).whitespace_before.borrow_mut()) } fn inflate_after(&mut self, config: &Config<'a>) -> Result> { parse_parenthesizable_whitespace(config, &mut (*self.tok).whitespace_after.borrow_mut()) } } #[cst_node] pub struct ImportStar {} pub(crate) fn make_importstar<'r, 'a>() -> DeflatedImportStar<'r, 'a> { DeflatedImportStar { _phantom: Default::default(), } } impl<'a> Codegen<'a> for ImportStar { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("*"); } } impl<'r, 'a> Inflate<'a> for DeflatedImportStar<'r, 'a> { type Inflated = ImportStar; fn inflate(self, _config: &Config<'a>) -> Result { Ok(ImportStar {}) } } #[cst_node] pub enum UnaryOp<'a> { Plus { whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, Minus { whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, BitInvert { whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, Not { whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, } impl<'a> Codegen<'a> for UnaryOp<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { let (tok, whitespace_after) = match self { Self::Plus { whitespace_after, .. } => ("+", whitespace_after), Self::Minus { whitespace_after, .. } => ("-", whitespace_after), Self::BitInvert { whitespace_after, .. } => ("~", whitespace_after), Self::Not { whitespace_after, .. } => ("not", whitespace_after), }; state.add_token(tok); whitespace_after.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedUnaryOp<'r, 'a> { type Inflated = UnaryOp<'a>; fn inflate(self, config: &Config<'a>) -> Result { Ok(match self { Self::Plus { tok, .. } => { let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::Plus { whitespace_after } } Self::Minus { tok, .. } => { let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::Minus { whitespace_after } } Self::BitInvert { tok, .. } => { let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::BitInvert { whitespace_after } } Self::Not { tok, .. } => { let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::Not { whitespace_after } } }) } } #[cst_node] pub enum BooleanOp<'a> { And { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, Or { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, } impl<'a> Codegen<'a> for BooleanOp<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { let (tok, ws_bef, ws_aft) = match self { Self::And { whitespace_after, whitespace_before, .. } => ("and", whitespace_before, whitespace_after), Self::Or { whitespace_after, whitespace_before, .. } => ("or", whitespace_before, whitespace_after), }; ws_bef.codegen(state); state.add_token(tok); ws_aft.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedBooleanOp<'r, 'a> { type Inflated = BooleanOp<'a>; fn inflate(self, config: &Config<'a>) -> Result { Ok(match self { Self::And { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::And { whitespace_before, whitespace_after, } } Self::Or { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::Or { whitespace_before, whitespace_after, } } }) } } #[cst_node] pub enum BinaryOp<'a> { Add { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, Subtract { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, Multiply { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, Divide { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, FloorDivide { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, Modulo { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, Power { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, LeftShift { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, RightShift { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, BitOr { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, BitAnd { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, BitXor { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, MatrixMultiply { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, } impl<'a> Codegen<'a> for BinaryOp<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { let (whitespace_before, whitespace_after) = match self { Self::Add { whitespace_before, whitespace_after, } | Self::Subtract { whitespace_before, whitespace_after, } | Self::Multiply { whitespace_before, whitespace_after, } | Self::Divide { whitespace_before, whitespace_after, } | Self::FloorDivide { whitespace_before, whitespace_after, } | Self::Modulo { whitespace_before, whitespace_after, } | Self::Power { whitespace_before, whitespace_after, } | Self::LeftShift { whitespace_before, whitespace_after, } | Self::RightShift { whitespace_before, whitespace_after, } | Self::BitOr { whitespace_before, whitespace_after, } | Self::BitAnd { whitespace_before, whitespace_after, } | Self::BitXor { whitespace_before, whitespace_after, } | Self::MatrixMultiply { whitespace_before, whitespace_after, } => (whitespace_before, whitespace_after), }; let tok = match self { BinaryOp::Add { .. } => "+", BinaryOp::Subtract { .. } => "-", BinaryOp::Multiply { .. } => "*", BinaryOp::Divide { .. } => "/", BinaryOp::FloorDivide { .. } => "//", BinaryOp::Modulo { .. } => "%", BinaryOp::Power { .. } => "**", BinaryOp::LeftShift { .. } => "<<", BinaryOp::RightShift { .. } => ">>", BinaryOp::BitOr { .. } => "|", BinaryOp::BitAnd { .. } => "&", BinaryOp::BitXor { .. } => "^", BinaryOp::MatrixMultiply { .. } => "@", }; whitespace_before.codegen(state); state.add_token(tok); whitespace_after.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedBinaryOp<'r, 'a> { type Inflated = BinaryOp<'a>; fn inflate(self, config: &Config<'a>) -> Result { Ok(match self { Self::Add { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::Add { whitespace_before, whitespace_after, } } Self::Subtract { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::Subtract { whitespace_before, whitespace_after, } } Self::Multiply { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::Multiply { whitespace_before, whitespace_after, } } Self::Divide { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::Divide { whitespace_before, whitespace_after, } } Self::FloorDivide { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::FloorDivide { whitespace_before, whitespace_after, } } Self::Modulo { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::Modulo { whitespace_before, whitespace_after, } } Self::Power { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::Power { whitespace_before, whitespace_after, } } Self::LeftShift { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::LeftShift { whitespace_before, whitespace_after, } } Self::RightShift { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::RightShift { whitespace_before, whitespace_after, } } Self::BitOr { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::BitOr { whitespace_before, whitespace_after, } } Self::BitAnd { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::BitAnd { whitespace_before, whitespace_after, } } Self::BitXor { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::BitXor { whitespace_before, whitespace_after, } } Self::MatrixMultiply { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::MatrixMultiply { whitespace_before, whitespace_after, } } }) } } #[cst_node] pub enum CompOp<'a> { LessThan { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, GreaterThan { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, LessThanEqual { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, GreaterThanEqual { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, Equal { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, NotEqual { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, In { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, NotIn { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_between: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] not_tok: TokenRef<'a>, #[cfg_attr(feature = "py", skip_py)] in_tok: TokenRef<'a>, }, Is { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, IsNot { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_between: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] is_tok: TokenRef<'a>, #[cfg_attr(feature = "py", skip_py)] not_tok: TokenRef<'a>, }, } impl<'a> Codegen<'a> for CompOp<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { let (bef, aft, between) = match self { Self::LessThan { whitespace_before, whitespace_after, } | Self::GreaterThan { whitespace_before, whitespace_after, } | Self::LessThanEqual { whitespace_before, whitespace_after, } | Self::GreaterThanEqual { whitespace_before, whitespace_after, } | Self::Equal { whitespace_before, whitespace_after, } | Self::NotEqual { whitespace_before, whitespace_after, } | Self::In { whitespace_before, whitespace_after, } | Self::Is { whitespace_before, whitespace_after, } => (whitespace_before, whitespace_after, None), Self::IsNot { whitespace_before, whitespace_between, whitespace_after, } => ( whitespace_before, whitespace_after, Some(whitespace_between), ), Self::NotIn { whitespace_before, whitespace_between, whitespace_after, } => ( whitespace_before, whitespace_after, Some(whitespace_between), ), }; let (first_tok, second_tok) = match self { CompOp::LessThan { .. } => ("<", None), CompOp::GreaterThan { .. } => (">", None), CompOp::LessThanEqual { .. } => ("<=", None), CompOp::GreaterThanEqual { .. } => (">=", None), CompOp::Equal { .. } => ("==", None), CompOp::NotEqual { .. } => ("!=", None), CompOp::In { .. } => ("in", None), CompOp::NotIn { .. } => ("not", Some("in")), CompOp::Is { .. } => ("is", None), CompOp::IsNot { .. } => ("is", Some("not")), }; bef.codegen(state); state.add_token(first_tok); if let (Some(btw), Some(second_tok)) = (between, second_tok) { btw.codegen(state); state.add_token(second_tok); } aft.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedCompOp<'r, 'a> { type Inflated = CompOp<'a>; fn inflate(self, config: &Config<'a>) -> Result { Ok(match self { Self::LessThan { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::LessThan { whitespace_before, whitespace_after, } } Self::GreaterThan { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::GreaterThan { whitespace_before, whitespace_after, } } Self::LessThanEqual { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::LessThanEqual { whitespace_before, whitespace_after, } } Self::GreaterThanEqual { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::GreaterThanEqual { whitespace_before, whitespace_after, } } Self::Equal { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::Equal { whitespace_before, whitespace_after, } } Self::NotEqual { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::NotEqual { whitespace_before, whitespace_after, } } Self::In { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::In { whitespace_before, whitespace_after, } } Self::Is { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::Is { whitespace_before, whitespace_after, } } Self::IsNot { is_tok, not_tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*is_tok).whitespace_before.borrow_mut(), )?; let whitespace_between = parse_parenthesizable_whitespace( config, &mut (*is_tok).whitespace_after.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*not_tok).whitespace_after.borrow_mut(), )?; Self::Inflated::IsNot { whitespace_before, whitespace_between, whitespace_after, } } Self::NotIn { not_tok, in_tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*not_tok).whitespace_before.borrow_mut(), )?; let whitespace_between = parse_parenthesizable_whitespace( config, &mut (*not_tok).whitespace_after.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*in_tok).whitespace_after.borrow_mut(), )?; Self::Inflated::NotIn { whitespace_before, whitespace_between, whitespace_after, } } }) } } #[cst_node] pub struct Colon<'a> { pub whitespace_before: ParenthesizableWhitespace<'a>, pub whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] pub(crate) tok: TokenRef<'a>, } impl<'r, 'a> Inflate<'a> for DeflatedColon<'r, 'a> { type Inflated = Colon<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_after.borrow_mut(), )?; Ok(Self::Inflated { whitespace_before, whitespace_after, }) } } impl<'a> Codegen<'a> for Colon<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.whitespace_before.codegen(state); state.add_token(":"); self.whitespace_after.codegen(state); } } #[cst_node] pub enum AugOp<'a> { AddAssign { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, SubtractAssign { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, MultiplyAssign { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, MatrixMultiplyAssign { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, DivideAssign { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, ModuloAssign { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, BitAndAssign { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, BitOrAssign { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, BitXorAssign { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, LeftShiftAssign { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, RightShiftAssign { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, PowerAssign { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, FloorDivideAssign { whitespace_before: ParenthesizableWhitespace<'a>, whitespace_after: ParenthesizableWhitespace<'a>, #[cfg_attr(feature = "py", skip_py)] tok: TokenRef<'a>, }, } impl<'r, 'a> Inflate<'a> for DeflatedAugOp<'r, 'a> { type Inflated = AugOp<'a>; fn inflate(self, config: &Config<'a>) -> Result { Ok(match self { Self::AddAssign { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::AddAssign { whitespace_before, whitespace_after, } } Self::SubtractAssign { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::SubtractAssign { whitespace_before, whitespace_after, } } Self::MultiplyAssign { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::MultiplyAssign { whitespace_before, whitespace_after, } } Self::MatrixMultiplyAssign { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::MatrixMultiplyAssign { whitespace_before, whitespace_after, } } Self::DivideAssign { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::DivideAssign { whitespace_before, whitespace_after, } } Self::ModuloAssign { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::ModuloAssign { whitespace_before, whitespace_after, } } Self::BitAndAssign { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::BitAndAssign { whitespace_before, whitespace_after, } } Self::BitOrAssign { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::BitOrAssign { whitespace_before, whitespace_after, } } Self::BitXorAssign { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::BitXorAssign { whitespace_before, whitespace_after, } } Self::LeftShiftAssign { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::LeftShiftAssign { whitespace_before, whitespace_after, } } Self::RightShiftAssign { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::RightShiftAssign { whitespace_before, whitespace_after, } } Self::PowerAssign { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::PowerAssign { whitespace_before, whitespace_after, } } Self::FloorDivideAssign { tok, .. } => { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*tok).whitespace_after.borrow_mut(), )?; Self::Inflated::FloorDivideAssign { whitespace_before, whitespace_after, } } }) } } impl<'a> Codegen<'a> for AugOp<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { let (tok, bef, aft) = match self { Self::AddAssign { whitespace_before, whitespace_after, .. } => ("+=", whitespace_before, whitespace_after), Self::SubtractAssign { whitespace_before, whitespace_after, .. } => ("-=", whitespace_before, whitespace_after), Self::MultiplyAssign { whitespace_before, whitespace_after, .. } => ("*=", whitespace_before, whitespace_after), Self::MatrixMultiplyAssign { whitespace_before, whitespace_after, .. } => ("@=", whitespace_before, whitespace_after), Self::DivideAssign { whitespace_before, whitespace_after, .. } => ("/=", whitespace_before, whitespace_after), Self::ModuloAssign { whitespace_before, whitespace_after, .. } => ("%=", whitespace_before, whitespace_after), Self::BitAndAssign { whitespace_before, whitespace_after, .. } => ("&=", whitespace_before, whitespace_after), Self::BitOrAssign { whitespace_before, whitespace_after, .. } => ("|=", whitespace_before, whitespace_after), Self::BitXorAssign { whitespace_before, whitespace_after, .. } => ("^=", whitespace_before, whitespace_after), Self::LeftShiftAssign { whitespace_before, whitespace_after, .. } => ("<<=", whitespace_before, whitespace_after), Self::RightShiftAssign { whitespace_before, whitespace_after, .. } => (">>=", whitespace_before, whitespace_after), Self::PowerAssign { whitespace_before, whitespace_after, .. } => ("**=", whitespace_before, whitespace_after), Self::FloorDivideAssign { whitespace_before, whitespace_after, .. } => ("//=", whitespace_before, whitespace_after), }; bef.codegen(state); state.add_token(tok); aft.codegen(state); } } #[cst_node] pub struct BitOr<'a> { pub whitespace_before: ParenthesizableWhitespace<'a>, pub whitespace_after: ParenthesizableWhitespace<'a>, pub(crate) tok: TokenRef<'a>, } impl<'r, 'a> Inflate<'a> for DeflatedBitOr<'r, 'a> { type Inflated = BitOr<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_before = parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_before.borrow_mut(), )?; let whitespace_after = parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_after.borrow_mut(), )?; Ok(Self::Inflated { whitespace_before, whitespace_after, }) } } impl<'a> Codegen<'a> for BitOr<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.whitespace_before.codegen(state); state.add_token("|"); self.whitespace_after.codegen(state); } } libcst-0.1.0/src/nodes/parser_config.rs000064400000000000000000000106570072674642500162310ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. use pyo3::exceptions::PyIndexError; use pyo3::prelude::*; use pyo3::types::{IntoPyDict, PyDict, PySequence, PyString}; use pyo3::wrap_pyfunction; use crate::py_cached::PyCached; #[pyclass(subclass, module = "libcst_native.parser_config")] #[text_signature = "(*, lines, default_newline)"] pub struct BaseWhitespaceParserConfig { pub lines: PyCached>, pub default_newline: PyCached, } #[pymethods] impl BaseWhitespaceParserConfig { #[new] fn new(lines: &PySequence, default_newline: &PyString) -> PyResult { // These fields will get initialized when ParserConfig.__init__ (our subclass) runs Ok(Self { lines: lines.extract()?, default_newline: default_newline.extract()?, }) } #[getter] fn get_lines(&self, py: Python) -> PyObject { self.lines.to_object(py) } #[getter] fn get_default_newline(&self, py: Python) -> PyObject { self.default_newline.to_object(py) } } impl BaseWhitespaceParserConfig { /// Equivalent to `config.lines.unwrap()[line_number - 1]`, but it return a PyErr when we get /// an index that's out of range, instead of panicing. pub fn get_line(&self, line_number: usize) -> PyResult<&str> { let err_fn = || PyIndexError::new_err(format!("line number of {} is out of range", line_number)); self.lines .get(line_number.checked_sub(1).ok_or_else(err_fn)?) .map(|l| &l[..]) .ok_or_else(err_fn) } /// Equivalent to `config.get_line(line_number)[column_index..]`, but it return a PyErr when /// we get an column index that's out of range, instead of panicing. pub fn get_line_after_column(&self, line_number: usize, column_index: usize) -> PyResult<&str> { self.get_line(line_number)? .get(column_index..) .ok_or_else(|| { PyIndexError::new_err(format!("column index of {} is out of range", column_index)) }) } } // These fields are private and PyObject, since we don't currently care about using them from // within rust. #[pyclass(extends=BaseWhitespaceParserConfig, module="libcst_native.parser_config")] #[text_signature = "(*, lines, encoding, default_indent, default_newline, has_trailing_newline, version, future_imports)"] pub struct ParserConfig { // lines is inherited #[pyo3(get)] encoding: PyObject, #[pyo3(get)] default_indent: PyObject, // default_newline is inherited #[pyo3(get)] has_trailing_newline: PyObject, #[pyo3(get)] version: PyObject, #[pyo3(get)] future_imports: PyObject, } #[pymethods] impl ParserConfig { #[new] fn new( lines: &PySequence, encoding: PyObject, default_indent: PyObject, default_newline: &PyString, has_trailing_newline: PyObject, version: PyObject, future_imports: PyObject, ) -> PyResult<(Self, BaseWhitespaceParserConfig)> { Ok(( Self { encoding, default_indent, has_trailing_newline, version, future_imports, }, BaseWhitespaceParserConfig::new(lines, default_newline)?, )) } } /// An internal helper function used by python unit tests to compare configs. #[pyfunction] fn parser_config_asdict<'py>(py: Python<'py>, config: PyRef<'py, ParserConfig>) -> &'py PyDict { let super_config: &BaseWhitespaceParserConfig = config.as_ref(); vec![ ("lines", super_config.lines.to_object(py)), ("encoding", config.encoding.clone_ref(py)), ("default_indent", config.default_indent.clone_ref(py)), ( "default_newline", super_config.default_newline.to_object(py), ), ( "has_trailing_newline", config.has_trailing_newline.clone_ref(py), ), ("version", config.version.clone_ref(py)), ("future_imports", config.future_imports.clone_ref(py)), ] .into_py_dict(py) } pub fn init_module(_py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_function(wrap_pyfunction!(parser_config_asdict, m)?) .unwrap(); Ok(self) } libcst-0.1.0/src/nodes/py_cached.rs000064400000000000000000000030640072674642500153210ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. use pyo3::prelude::*; use std::convert::AsRef; use std::ops::Deref; /// An immutable wrapper around a rust type T and it's PyObject equivalent. Caches the conversion /// to and from the PyObject. pub struct PyCached { native: T, py_object: PyObject, } impl PyCached where T: ToPyObject, { pub fn new(py: Python, native: T) -> Self { Self { py_object: native.to_object(py), native, } } } impl<'source, T> FromPyObject<'source> for PyCached where T: FromPyObject<'source>, { fn extract(ob: &'source PyAny) -> PyResult { Python::with_gil(|py| { Ok(PyCached { native: ob.extract()?, py_object: ob.to_object(py), }) }) } } impl IntoPy for PyCached { fn into_py(self, _py: Python) -> PyObject { self.py_object } } impl ToPyObject for PyCached { fn to_object(&self, py: Python) -> PyObject { self.py_object.clone_ref(py) } } impl AsRef for PyCached { fn as_ref(&self) -> &T { &self.native } } impl Deref for PyCached { type Target = T; fn deref(&self) -> &Self::Target { &self.native } } impl From for PyCached where T: ToPyObject, { fn from(val: T) -> Self { Python::with_gil(|py| Self::new(py, val)) } } libcst-0.1.0/src/nodes/statement.rs000064400000000000000000003363230072674642500154150ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. use std::mem::swap; use super::{ inflate_helpers::adjust_parameters_trailing_whitespace, Attribute, Codegen, CodegenState, Comma, Dot, EmptyLine, Expression, From, ImportStar, LeftParen, List, Name, NameOrAttribute, Parameters, ParenthesizableWhitespace, RightParen, Semicolon, SimpleWhitespace, StarredElement, Subscript, TrailingWhitespace, Tuple, }; use crate::{ nodes::{ expression::*, op::*, traits::{ Inflate, ParenthesizedDeflatedNode, ParenthesizedNode, Result, WithComma, WithLeadingLines, }, }, tokenizer::{ whitespace_parser::{ parse_empty_lines, parse_parenthesizable_whitespace, parse_simple_whitespace, parse_trailing_whitespace, Config, }, Token, }, LeftCurlyBrace, LeftSquareBracket, RightCurlyBrace, RightSquareBracket, }; #[cfg(feature = "py")] use libcst_derive::TryIntoPy; use libcst_derive::{cst_node, Codegen, Inflate, ParenthesizedDeflatedNode, ParenthesizedNode}; type TokenRef<'r, 'a> = &'r Token<'a>; #[allow(clippy::large_enum_variant)] #[cst_node(Inflate, Codegen)] pub enum Statement<'a> { Simple(SimpleStatementLine<'a>), Compound(CompoundStatement<'a>), } impl<'a> WithLeadingLines<'a> for Statement<'a> { fn leading_lines(&mut self) -> &mut Vec> { match self { Self::Simple(s) => &mut s.leading_lines, Self::Compound(c) => c.leading_lines(), } } } #[allow(clippy::large_enum_variant)] #[cst_node(Inflate, Codegen)] pub enum CompoundStatement<'a> { FunctionDef(FunctionDef<'a>), If(If<'a>), For(For<'a>), While(While<'a>), ClassDef(ClassDef<'a>), Try(Try<'a>), TryStar(TryStar<'a>), With(With<'a>), Match(Match<'a>), } impl<'a> WithLeadingLines<'a> for CompoundStatement<'a> { fn leading_lines(&mut self) -> &mut Vec> { match self { Self::FunctionDef(f) => &mut f.leading_lines, Self::If(f) => &mut f.leading_lines, Self::For(f) => &mut f.leading_lines, Self::While(f) => &mut f.leading_lines, Self::ClassDef(c) => &mut c.leading_lines, Self::Try(t) => &mut t.leading_lines, Self::TryStar(t) => &mut t.leading_lines, Self::With(w) => &mut w.leading_lines, Self::Match(m) => &mut m.leading_lines, } } } #[cst_node(Inflate, Codegen)] pub enum Suite<'a> { IndentedBlock(IndentedBlock<'a>), SimpleStatementSuite(SimpleStatementSuite<'a>), } #[cst_node] pub struct IndentedBlock<'a> { /// Sequence of statements belonging to this indented block. pub body: Vec>, /// Any optional trailing comment and the final ``NEWLINE`` at the end of the line. pub header: TrailingWhitespace<'a>, /// A string represents a specific indentation. A ``None`` value uses the modules's /// default indentation. This is included because indentation is allowed to be /// inconsistent across a file, just not ambiguously. pub indent: Option<&'a str>, /// Any trailing comments or lines after the dedent that are owned by this indented /// block. Statements own preceeding and same-line trailing comments, but not /// trailing lines, so it falls on :class:`IndentedBlock` to own it. In the case /// that a statement follows an :class:`IndentedBlock`, that statement will own the /// comments and lines that are at the same indent as the statement, and this /// :class:`IndentedBlock` will own the comments and lines that are indented /// further. pub footer: Vec>, pub(crate) newline_tok: TokenRef<'a>, pub(crate) indent_tok: TokenRef<'a>, pub(crate) dedent_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for IndentedBlock<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.header.codegen(state); let indent = match self.indent { Some(i) => i, None => state.default_indent, }; state.indent(indent); if self.body.is_empty() { // Empty indented blocks are not syntactically valid in Python unless they // contain a 'pass' statement, so add one here. state.add_indent(); state.add_token("pass"); state.add_token(state.default_newline); } else { for stmt in &self.body { // IndentedBlock is responsible for adjusting the current indentation // level, but its children are responsible for actually adding that // indentation to the token list. stmt.codegen(state); } } for f in &self.footer { f.codegen(state); } state.dedent(); } } impl<'r, 'a> Inflate<'a> for DeflatedIndentedBlock<'r, 'a> { type Inflated = IndentedBlock<'a>; fn inflate(self, config: &Config<'a>) -> Result { let body = self.body.inflate(config)?; // We want to be able to only keep comments in the footer that are actually for // this IndentedBlock. We do so by assuming that lines which are indented to the // same level as the block itself are comments that go at the footer of the // block. Comments that are indented to less than this indent are assumed to // belong to the next line of code. We override the indent here because the // dedent node's absolute indent is the resulting indentation after the dedent // is performed. Its this way because the whitespace state for both the dedent's // whitespace_after and the next BaseCompoundStatement's whitespace_before is // shared. This allows us to partially parse here and parse the rest of the // whitespace and comments on the next line, effectively making sure that // comments are attached to the correct node. let footer = parse_empty_lines( config, &mut (*self.dedent_tok).whitespace_after.borrow_mut(), Some(self.indent_tok.whitespace_before.borrow().absolute_indent), )?; let header = parse_trailing_whitespace( config, &mut (*self.newline_tok).whitespace_before.borrow_mut(), )?; let mut indent = self.indent_tok.relative_indent; if indent == Some(config.default_indent) { indent = None; } Ok(Self::Inflated { body, header, indent, footer, }) } } #[cst_node] pub struct SimpleStatementSuite<'a> { /// Sequence of small statements. All but the last statement are required to have /// a semicolon. pub body: Vec>, /// The whitespace between the colon in the parent statement and the body. pub leading_whitespace: SimpleWhitespace<'a>, /// Any optional trailing comment and the final ``NEWLINE`` at the end of the line. pub trailing_whitespace: TrailingWhitespace<'a>, pub(crate) first_tok: TokenRef<'a>, pub(crate) newline_tok: TokenRef<'a>, } impl<'r, 'a> Inflate<'a> for DeflatedSimpleStatementSuite<'r, 'a> { type Inflated = SimpleStatementSuite<'a>; fn inflate(self, config: &Config<'a>) -> Result { let leading_whitespace = parse_simple_whitespace( config, &mut (*self.first_tok).whitespace_before.borrow_mut(), )?; let body = self.body.inflate(config)?; let trailing_whitespace = parse_trailing_whitespace( config, &mut (*self.newline_tok).whitespace_before.borrow_mut(), )?; Ok(Self::Inflated { body, leading_whitespace, trailing_whitespace, }) } } fn _simple_statement_codegen<'a>( body: &[SmallStatement<'a>], trailing_whitespace: &TrailingWhitespace<'a>, state: &mut CodegenState<'a>, ) { for stmt in body { stmt.codegen(state); // TODO: semicolon } if body.is_empty() { // Empty simple statement blocks are not syntactically valid in Python // unless they contain a 'pass' statement, so add one here. state.add_token("pass") } trailing_whitespace.codegen(state); } impl<'a> Codegen<'a> for SimpleStatementSuite<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.leading_whitespace.codegen(state); _simple_statement_codegen(&self.body, &self.trailing_whitespace, state); } } #[cst_node] pub struct SimpleStatementLine<'a> { /// Sequence of small statements. All but the last statement are required to have /// a semicolon. pub body: Vec>, /// Sequence of empty lines appearing before this simple statement line. pub leading_lines: Vec>, /// Any optional trailing comment and the final ``NEWLINE`` at the end of the line. pub trailing_whitespace: TrailingWhitespace<'a>, pub(crate) first_tok: TokenRef<'a>, pub(crate) newline_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for SimpleStatementLine<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for line in &self.leading_lines { line.codegen(state); } state.add_indent(); _simple_statement_codegen(&self.body, &self.trailing_whitespace, state); } } impl<'r, 'a> Inflate<'a> for DeflatedSimpleStatementLine<'r, 'a> { type Inflated = SimpleStatementLine<'a>; fn inflate(self, config: &Config<'a>) -> Result { let leading_lines = parse_empty_lines( config, &mut (*self.first_tok).whitespace_before.borrow_mut(), None, )?; let body = self.body.inflate(config)?; let trailing_whitespace = parse_trailing_whitespace( config, &mut (*self.newline_tok).whitespace_before.borrow_mut(), )?; Ok(Self::Inflated { body, leading_lines, trailing_whitespace, }) } } #[allow(dead_code, clippy::large_enum_variant)] #[cst_node(Codegen, Inflate)] pub enum SmallStatement<'a> { Pass(Pass<'a>), Break(Break<'a>), Continue(Continue<'a>), Return(Return<'a>), Expr(Expr<'a>), Assert(Assert<'a>), Import(Import<'a>), ImportFrom(ImportFrom<'a>), Assign(Assign<'a>), AnnAssign(AnnAssign<'a>), Raise(Raise<'a>), Global(Global<'a>), Nonlocal(Nonlocal<'a>), AugAssign(AugAssign<'a>), Del(Del<'a>), TypeAlias(TypeAlias<'a>), } impl<'r, 'a> DeflatedSmallStatement<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { match self { Self::Pass(p) => Self::Pass(p.with_semicolon(semicolon)), Self::Break(p) => Self::Break(p.with_semicolon(semicolon)), Self::Continue(p) => Self::Continue(p.with_semicolon(semicolon)), Self::Expr(p) => Self::Expr(p.with_semicolon(semicolon)), Self::Import(i) => Self::Import(i.with_semicolon(semicolon)), Self::ImportFrom(i) => Self::ImportFrom(i.with_semicolon(semicolon)), Self::Assign(a) => Self::Assign(a.with_semicolon(semicolon)), Self::AnnAssign(a) => Self::AnnAssign(a.with_semicolon(semicolon)), Self::Return(r) => Self::Return(r.with_semicolon(semicolon)), Self::Assert(a) => Self::Assert(a.with_semicolon(semicolon)), Self::Raise(r) => Self::Raise(r.with_semicolon(semicolon)), Self::Global(g) => Self::Global(g.with_semicolon(semicolon)), Self::Nonlocal(l) => Self::Nonlocal(l.with_semicolon(semicolon)), Self::AugAssign(a) => Self::AugAssign(a.with_semicolon(semicolon)), Self::Del(d) => Self::Del(d.with_semicolon(semicolon)), Self::TypeAlias(t) => Self::TypeAlias(t.with_semicolon(semicolon)), } } } #[cst_node] pub struct Pass<'a> { pub semicolon: Option>, } impl<'r, 'a> DeflatedPass<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { Self { semicolon } } } impl<'a> Codegen<'a> for Pass<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("pass"); self.semicolon.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedPass<'r, 'a> { type Inflated = Pass<'a>; fn inflate(self, config: &Config<'a>) -> Result { let semicolon = self.semicolon.inflate(config)?; Ok(Self::Inflated { semicolon }) } } #[cst_node] pub struct Break<'a> { pub semicolon: Option>, } impl<'r, 'a> DeflatedBreak<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { Self { semicolon } } } impl<'a> Codegen<'a> for Break<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("break"); self.semicolon.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedBreak<'r, 'a> { type Inflated = Break<'a>; fn inflate(self, config: &Config<'a>) -> Result { let semicolon = self.semicolon.inflate(config)?; Ok(Self::Inflated { semicolon }) } } #[cst_node] pub struct Continue<'a> { pub semicolon: Option>, } impl<'r, 'a> DeflatedContinue<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { Self { semicolon } } } impl<'a> Codegen<'a> for Continue<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("continue"); self.semicolon.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedContinue<'r, 'a> { type Inflated = Continue<'a>; fn inflate(self, config: &Config<'a>) -> Result { let semicolon = self.semicolon.inflate(config)?; Ok(Self::Inflated { semicolon }) } } #[cst_node] pub struct Expr<'a> { pub value: Expression<'a>, pub semicolon: Option>, } impl<'r, 'a> DeflatedExpr<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { Self { semicolon, ..self } } } impl<'a> Codegen<'a> for Expr<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.value.codegen(state); self.semicolon.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedExpr<'r, 'a> { type Inflated = Expr<'a>; fn inflate(self, config: &Config<'a>) -> Result { let value = self.value.inflate(config)?; let semicolon = self.semicolon.inflate(config)?; Ok(Self::Inflated { value, semicolon }) } } #[cst_node] pub struct Assign<'a> { pub targets: Vec>, pub value: Expression<'a>, pub semicolon: Option>, } impl<'a> Codegen<'a> for Assign<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for target in &self.targets { target.codegen(state); } self.value.codegen(state); if let Some(semi) = &self.semicolon { semi.codegen(state); } } } impl<'r, 'a> Inflate<'a> for DeflatedAssign<'r, 'a> { type Inflated = Assign<'a>; fn inflate(self, config: &Config<'a>) -> Result { let targets = self.targets.inflate(config)?; let value = self.value.inflate(config)?; let semicolon = self.semicolon.inflate(config)?; Ok(Self::Inflated { targets, value, semicolon, }) } } impl<'r, 'a> DeflatedAssign<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { Self { semicolon, ..self } } } #[cst_node] pub struct AssignTarget<'a> { pub target: AssignTargetExpression<'a>, pub whitespace_before_equal: SimpleWhitespace<'a>, pub whitespace_after_equal: SimpleWhitespace<'a>, pub(crate) equal_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for AssignTarget<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.target.codegen(state); self.whitespace_before_equal.codegen(state); state.add_token("="); self.whitespace_after_equal.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedAssignTarget<'r, 'a> { type Inflated = AssignTarget<'a>; fn inflate(self, config: &Config<'a>) -> Result { let target = self.target.inflate(config)?; let whitespace_before_equal = parse_simple_whitespace( config, &mut (*self.equal_tok).whitespace_before.borrow_mut(), )?; let whitespace_after_equal = parse_simple_whitespace(config, &mut (*self.equal_tok).whitespace_after.borrow_mut())?; Ok(Self::Inflated { target, whitespace_before_equal, whitespace_after_equal, }) } } #[allow(clippy::large_enum_variant)] #[cst_node(Codegen, ParenthesizedNode, Inflate)] pub enum AssignTargetExpression<'a> { Name(Box>), Attribute(Box>), StarredElement(Box>), Tuple(Box>), List(Box>), Subscript(Box>), } #[cst_node] pub struct Import<'a> { pub names: Vec>, pub semicolon: Option>, pub whitespace_after_import: SimpleWhitespace<'a>, pub(crate) import_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for Import<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("import"); self.whitespace_after_import.codegen(state); for (i, name) in self.names.iter().enumerate() { name.codegen(state); if name.comma.is_none() && i < self.names.len() - 1 { state.add_token(", "); } } if let Some(semi) = &self.semicolon { semi.codegen(state); } } } impl<'r, 'a> Inflate<'a> for DeflatedImport<'r, 'a> { type Inflated = Import<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_after_import = parse_simple_whitespace( config, &mut (*self.import_tok).whitespace_after.borrow_mut(), )?; let names = self.names.inflate(config)?; let semicolon = self.semicolon.inflate(config)?; Ok(Self::Inflated { names, semicolon, whitespace_after_import, }) } } impl<'r, 'a> DeflatedImport<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { Self { semicolon, ..self } } } #[cst_node] pub struct ImportFrom<'a> { #[cfg_attr(feature = "py", no_py_default)] pub module: Option>, pub names: ImportNames<'a>, pub relative: Vec>, pub lpar: Option>, pub rpar: Option>, pub semicolon: Option>, pub whitespace_after_from: SimpleWhitespace<'a>, pub whitespace_before_import: SimpleWhitespace<'a>, pub whitespace_after_import: SimpleWhitespace<'a>, pub(crate) from_tok: TokenRef<'a>, pub(crate) import_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for ImportFrom<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("from"); self.whitespace_after_from.codegen(state); for dot in &self.relative { dot.codegen(state); } if let Some(module) = &self.module { module.codegen(state); } self.whitespace_before_import.codegen(state); state.add_token("import"); self.whitespace_after_import.codegen(state); if let Some(lpar) = &self.lpar { lpar.codegen(state); } self.names.codegen(state); if let Some(rpar) = &self.rpar { rpar.codegen(state); } if let Some(semi) = &self.semicolon { semi.codegen(state); } } } impl<'r, 'a> Inflate<'a> for DeflatedImportFrom<'r, 'a> { type Inflated = ImportFrom<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_after_from = parse_simple_whitespace(config, &mut (*self.from_tok).whitespace_after.borrow_mut())?; let module = self.module.inflate(config)?; let whitespace_after_import = parse_simple_whitespace( config, &mut (*self.import_tok).whitespace_after.borrow_mut(), )?; let mut relative = inflate_dots(self.relative, config)?; let mut whitespace_before_import = Default::default(); if !relative.is_empty() && module.is_none() { // For relative-only imports relocate the space after the final dot to be owned // by the import token. if let Some(Dot { whitespace_after: ParenthesizableWhitespace::SimpleWhitespace(dot_ws), .. }) = relative.last_mut() { swap(dot_ws, &mut whitespace_before_import); } } else { whitespace_before_import = parse_simple_whitespace( config, &mut (*self.import_tok).whitespace_before.borrow_mut(), )?; } let lpar = self.lpar.inflate(config)?; let names = self.names.inflate(config)?; let rpar = self.rpar.inflate(config)?; let semicolon = self.semicolon.inflate(config)?; Ok(Self::Inflated { module, names, relative, lpar, rpar, semicolon, whitespace_after_from, whitespace_before_import, whitespace_after_import, }) } } fn inflate_dots<'r, 'a>( dots: Vec>, config: &Config<'a>, ) -> Result>> { let mut ret: Vec> = vec![]; let mut last_tok: Option> = None; for dot in dots { if let Some(last_tokref) = &last_tok { // Consecutive dots having the same Token can only happen if `...` was // parsed as a single ELLIPSIS token. In this case the token's // whitespace_before belongs to the first dot, but the whitespace_after is // moved to the 3rd dot (by swapping it twice) if last_tokref.start_pos == dot.tok.start_pos { let mut subsequent_dot = Dot { whitespace_before: Default::default(), whitespace_after: Default::default(), }; swap( &mut ret.last_mut().unwrap().whitespace_after, &mut subsequent_dot.whitespace_after, ); ret.push(subsequent_dot); continue; } } last_tok = Some(dot.tok); ret.push(dot.inflate(config)?); } Ok(ret) } impl<'r, 'a> DeflatedImportFrom<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { Self { semicolon, ..self } } } #[cst_node] pub struct ImportAlias<'a> { pub name: NameOrAttribute<'a>, pub asname: Option>, pub comma: Option>, } impl<'r, 'a> Inflate<'a> for DeflatedImportAlias<'r, 'a> { type Inflated = ImportAlias<'a>; fn inflate(self, config: &Config<'a>) -> Result { let name = self.name.inflate(config)?; let asname = self.asname.inflate(config)?; let comma = self.comma.inflate(config)?; Ok(Self::Inflated { name, asname, comma, }) } } impl<'r, 'a> WithComma<'r, 'a> for DeflatedImportAlias<'r, 'a> { fn with_comma(self, comma: DeflatedComma<'r, 'a>) -> Self { let comma = Some(comma); Self { comma, ..self } } } impl<'a> Codegen<'a> for ImportAlias<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.name.codegen(state); if let Some(asname) = &self.asname { asname.codegen(state); } if let Some(comma) = &self.comma { comma.codegen(state); } } } #[cst_node] pub struct AsName<'a> { pub name: AssignTargetExpression<'a>, pub whitespace_before_as: ParenthesizableWhitespace<'a>, pub whitespace_after_as: ParenthesizableWhitespace<'a>, pub(crate) as_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for AsName<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.whitespace_before_as.codegen(state); state.add_token("as"); self.whitespace_after_as.codegen(state); self.name.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedAsName<'r, 'a> { type Inflated = AsName<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_before_as = parse_parenthesizable_whitespace( config, &mut (*self.as_tok).whitespace_before.borrow_mut(), )?; let whitespace_after_as = parse_parenthesizable_whitespace( config, &mut (*self.as_tok).whitespace_after.borrow_mut(), )?; let name = self.name.inflate(config)?; Ok(Self::Inflated { name, whitespace_before_as, whitespace_after_as, }) } } #[cst_node(Inflate)] pub enum ImportNames<'a> { Star(ImportStar), Aliases(Vec>), } impl<'a> Codegen<'a> for ImportNames<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { match self { Self::Star(s) => s.codegen(state), Self::Aliases(aliases) => { for (i, alias) in aliases.iter().enumerate() { alias.codegen(state); if alias.comma.is_none() && i < aliases.len() - 1 { state.add_token(", "); } } } } } } #[cst_node] pub struct FunctionDef<'a> { pub name: Name<'a>, pub type_parameters: Option>, pub params: Parameters<'a>, pub body: Suite<'a>, pub decorators: Vec>, pub returns: Option>, pub asynchronous: Option>, pub leading_lines: Vec>, pub lines_after_decorators: Vec>, pub whitespace_after_def: SimpleWhitespace<'a>, pub whitespace_after_name: SimpleWhitespace<'a>, pub whitespace_after_type_parameters: SimpleWhitespace<'a>, pub whitespace_before_params: ParenthesizableWhitespace<'a>, pub whitespace_before_colon: SimpleWhitespace<'a>, pub(crate) async_tok: Option>, pub(crate) def_tok: TokenRef<'a>, pub(crate) open_paren_tok: TokenRef<'a>, pub(crate) close_paren_tok: TokenRef<'a>, pub(crate) colon_tok: TokenRef<'a>, } impl<'r, 'a> DeflatedFunctionDef<'r, 'a> { pub fn with_decorators(self, decorators: Vec>) -> Self { Self { decorators, ..self } } } impl<'a> Codegen<'a> for FunctionDef<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for l in &self.leading_lines { l.codegen(state); } for dec in self.decorators.iter() { dec.codegen(state); } for l in &self.lines_after_decorators { l.codegen(state); } state.add_indent(); if let Some(asy) = &self.asynchronous { asy.codegen(state); } state.add_token("def"); self.whitespace_after_def.codegen(state); self.name.codegen(state); self.whitespace_after_name.codegen(state); if let Some(tp) = &self.type_parameters { tp.codegen(state); self.whitespace_after_type_parameters.codegen(state); } state.add_token("("); self.whitespace_before_params.codegen(state); self.params.codegen(state); state.add_token(")"); if let Some(ann) = &self.returns { ann.codegen(state, "->"); } self.whitespace_before_colon.codegen(state); state.add_token(":"); self.body.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedFunctionDef<'r, 'a> { type Inflated = FunctionDef<'a>; fn inflate(mut self, config: &Config<'a>) -> Result { let mut decorators = self.decorators.inflate(config)?; let (asynchronous, leading_lines) = if let Some(asy) = self.async_tok.as_mut() { let whitespace_after = parse_parenthesizable_whitespace(config, &mut asy.whitespace_after.borrow_mut())?; ( Some(Asynchronous { whitespace_after }), Some(parse_empty_lines( config, &mut asy.whitespace_before.borrow_mut(), None, )?), ) } else { (None, None) }; let mut leading_lines = if let Some(ll) = leading_lines { ll } else { parse_empty_lines( config, &mut (*self.def_tok).whitespace_before.borrow_mut(), None, )? }; let mut lines_after_decorators = Default::default(); if let Some(dec) = decorators.first_mut() { swap(&mut lines_after_decorators, &mut leading_lines); swap(&mut dec.leading_lines, &mut leading_lines); } let whitespace_after_def = parse_simple_whitespace(config, &mut (*self.def_tok).whitespace_after.borrow_mut())?; let name = self.name.inflate(config)?; let whitespace_after_name; let mut type_parameters = Default::default(); let mut whitespace_after_type_parameters = Default::default(); if let Some(tp) = self.type_parameters { let rbracket_tok = tp.rbracket.tok.clone(); whitespace_after_name = parse_simple_whitespace( config, &mut tp.lbracket.tok.whitespace_before.borrow_mut(), )?; type_parameters = Some(tp.inflate(config)?); whitespace_after_type_parameters = parse_simple_whitespace(config, &mut rbracket_tok.whitespace_after.borrow_mut())?; } else { whitespace_after_name = parse_simple_whitespace( config, &mut self.open_paren_tok.whitespace_before.borrow_mut(), )?; } let whitespace_before_params = parse_parenthesizable_whitespace( config, &mut (*self.open_paren_tok).whitespace_after.borrow_mut(), )?; let mut params = self.params.inflate(config)?; adjust_parameters_trailing_whitespace(config, &mut params, &self.close_paren_tok)?; let returns = self.returns.inflate(config)?; let whitespace_before_colon = parse_simple_whitespace( config, &mut (*self.colon_tok).whitespace_before.borrow_mut(), )?; let body = self.body.inflate(config)?; Ok(Self::Inflated { name, type_parameters, params, body, decorators, returns, asynchronous, leading_lines, lines_after_decorators, whitespace_after_def, whitespace_after_name, whitespace_after_type_parameters, whitespace_before_params, whitespace_before_colon, }) } } #[cst_node] pub struct Decorator<'a> { pub decorator: Expression<'a>, pub leading_lines: Vec>, pub whitespace_after_at: SimpleWhitespace<'a>, pub trailing_whitespace: TrailingWhitespace<'a>, pub(crate) at_tok: TokenRef<'a>, pub(crate) newline_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for Decorator<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for ll in self.leading_lines.iter() { ll.codegen(state); } state.add_indent(); state.add_token("@"); self.whitespace_after_at.codegen(state); self.decorator.codegen(state); self.trailing_whitespace.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedDecorator<'r, 'a> { type Inflated = Decorator<'a>; fn inflate(self, config: &Config<'a>) -> Result { let leading_lines = parse_empty_lines( config, &mut (*self.at_tok).whitespace_before.borrow_mut(), None, )?; let whitespace_after_at = parse_simple_whitespace(config, &mut (*self.at_tok).whitespace_after.borrow_mut())?; let decorator = self.decorator.inflate(config)?; let trailing_whitespace = parse_trailing_whitespace( config, &mut (*self.newline_tok).whitespace_before.borrow_mut(), )?; Ok(Self::Inflated { decorator, leading_lines, whitespace_after_at, trailing_whitespace, }) } } #[cst_node] pub struct If<'a> { /// The expression that, when evaluated, should give us a truthy value pub test: Expression<'a>, // The body of this compound statement. pub body: Suite<'a>, /// An optional ``elif`` or ``else`` clause. ``If`` signifies an ``elif`` block. pub orelse: Option>>, /// Sequence of empty lines appearing before this compound statement line. pub leading_lines: Vec>, /// The whitespace appearing after the ``if`` keyword but before the test /// expression. pub whitespace_before_test: SimpleWhitespace<'a>, /// The whitespace appearing after the test expression but before the colon. pub whitespace_after_test: SimpleWhitespace<'a>, /// Signifies if this instance represents an ``elif`` or an ``if`` block. #[cfg_attr(feature = "py", skip_py)] pub is_elif: bool, pub(crate) if_tok: TokenRef<'a>, pub(crate) colon_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for If<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for l in &self.leading_lines { l.codegen(state); } state.add_indent(); state.add_token(if self.is_elif { "elif" } else { "if" }); self.whitespace_before_test.codegen(state); self.test.codegen(state); self.whitespace_after_test.codegen(state); state.add_token(":"); self.body.codegen(state); if let Some(orelse) = &self.orelse { orelse.codegen(state) } } } impl<'r, 'a> Inflate<'a> for DeflatedIf<'r, 'a> { type Inflated = If<'a>; fn inflate(self, config: &Config<'a>) -> Result { let leading_lines = parse_empty_lines( config, &mut (*self.if_tok).whitespace_before.borrow_mut(), None, )?; let whitespace_before_test = parse_simple_whitespace(config, &mut (*self.if_tok).whitespace_after.borrow_mut())?; let test = self.test.inflate(config)?; let whitespace_after_test = parse_simple_whitespace( config, &mut (*self.colon_tok).whitespace_before.borrow_mut(), )?; let body = self.body.inflate(config)?; let orelse = self.orelse.inflate(config)?; Ok(Self::Inflated { test, body, orelse, leading_lines, whitespace_before_test, whitespace_after_test, is_elif: self.is_elif, }) } } #[allow(clippy::large_enum_variant)] #[cst_node(Inflate, Codegen)] pub enum OrElse<'a> { Elif(If<'a>), Else(Else<'a>), } #[cst_node] pub struct Else<'a> { pub body: Suite<'a>, /// Sequence of empty lines appearing before this compound statement line. pub leading_lines: Vec>, /// The whitespace appearing after the ``else`` keyword but before the colon. pub whitespace_before_colon: SimpleWhitespace<'a>, pub(crate) else_tok: TokenRef<'a>, pub(crate) colon_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for Else<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for l in &self.leading_lines { l.codegen(state); } state.add_indent(); state.add_token("else"); self.whitespace_before_colon.codegen(state); state.add_token(":"); self.body.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedElse<'r, 'a> { type Inflated = Else<'a>; fn inflate(self, config: &Config<'a>) -> Result { let leading_lines = parse_empty_lines( config, &mut (*self.else_tok).whitespace_before.borrow_mut(), None, )?; let whitespace_before_colon = parse_simple_whitespace( config, &mut (*self.colon_tok).whitespace_before.borrow_mut(), )?; let body = self.body.inflate(config)?; Ok(Self::Inflated { body, leading_lines, whitespace_before_colon, }) } } #[cst_node] pub struct Annotation<'a> { pub annotation: Expression<'a>, pub whitespace_before_indicator: Option>, pub whitespace_after_indicator: ParenthesizableWhitespace<'a>, pub(crate) tok: TokenRef<'a>, } impl<'a> Annotation<'a> { pub fn codegen(&self, state: &mut CodegenState<'a>, default_indicator: &'a str) { if let Some(ws) = &self.whitespace_before_indicator { ws.codegen(state); } else if default_indicator == "->" { state.add_token(" "); } else { panic!("Variable annotation but whitespace is None"); } state.add_token(default_indicator); self.whitespace_after_indicator.codegen(state); self.annotation.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedAnnotation<'r, 'a> { type Inflated = Annotation<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_before_indicator = Some(parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_before.borrow_mut(), )?); let whitespace_after_indicator = parse_parenthesizable_whitespace( config, &mut (*self.tok).whitespace_after.borrow_mut(), )?; let annotation = self.annotation.inflate(config)?; Ok(Self::Inflated { annotation, whitespace_before_indicator, whitespace_after_indicator, }) } } #[cst_node] pub struct AnnAssign<'a> { pub target: AssignTargetExpression<'a>, pub annotation: Annotation<'a>, pub value: Option>, pub equal: Option>, pub semicolon: Option>, } impl<'a> Codegen<'a> for AnnAssign<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.target.codegen(state); self.annotation.codegen(state, ":"); if let Some(eq) = &self.equal { eq.codegen(state); } else if self.value.is_some() { state.add_token(" = "); } if let Some(value) = &self.value { value.codegen(state); } if let Some(semi) = &self.semicolon { semi.codegen(state); } } } impl<'r, 'a> Inflate<'a> for DeflatedAnnAssign<'r, 'a> { type Inflated = AnnAssign<'a>; fn inflate(self, config: &Config<'a>) -> Result { let target = self.target.inflate(config)?; let annotation = self.annotation.inflate(config)?; let value = self.value.inflate(config)?; let equal = self.equal.inflate(config)?; let semicolon = self.semicolon.inflate(config)?; Ok(Self::Inflated { target, annotation, value, equal, semicolon, }) } } impl<'r, 'a> DeflatedAnnAssign<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { Self { semicolon, ..self } } } #[cst_node] pub struct Return<'a> { pub value: Option>, pub whitespace_after_return: Option>, pub semicolon: Option>, pub(crate) return_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for Return<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("return"); if let Some(ws) = &self.whitespace_after_return { ws.codegen(state); } else if self.value.is_some() { state.add_token(" "); } if let Some(val) = &self.value { val.codegen(state); } if let Some(semi) = &self.semicolon { semi.codegen(state); } } } impl<'r, 'a> Inflate<'a> for DeflatedReturn<'r, 'a> { type Inflated = Return<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_after_return = if self.value.is_some() { Some(parse_simple_whitespace( config, &mut (*self.return_tok).whitespace_after.borrow_mut(), )?) } else { // otherwise space is owned by semicolon or small statement // whitespace is not None to preserve a quirk of the pure python parser Some(Default::default()) }; let value = self.value.inflate(config)?; let semicolon = self.semicolon.inflate(config)?; Ok(Self::Inflated { value, whitespace_after_return, semicolon, }) } } impl<'r, 'a> DeflatedReturn<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { Self { semicolon, ..self } } } #[cst_node] pub struct Assert<'a> { pub test: Expression<'a>, pub msg: Option>, pub comma: Option>, pub whitespace_after_assert: SimpleWhitespace<'a>, pub semicolon: Option>, pub(crate) assert_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for Assert<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("assert"); self.whitespace_after_assert.codegen(state); self.test.codegen(state); if let Some(comma) = &self.comma { comma.codegen(state); } else if self.msg.is_some() { state.add_token(", "); } if let Some(msg) = &self.msg { msg.codegen(state); } if let Some(semi) = &self.semicolon { semi.codegen(state); } } } impl<'r, 'a> Inflate<'a> for DeflatedAssert<'r, 'a> { type Inflated = Assert<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_after_assert = parse_simple_whitespace( config, &mut (*self.assert_tok).whitespace_after.borrow_mut(), )?; let test = self.test.inflate(config)?; let comma = self.comma.inflate(config)?; let msg = self.msg.inflate(config)?; let semicolon = self.semicolon.inflate(config)?; Ok(Self::Inflated { test, msg, comma, whitespace_after_assert, semicolon, }) } } impl<'r, 'a> DeflatedAssert<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { Self { semicolon, ..self } } } #[cst_node] pub struct Raise<'a> { pub exc: Option>, pub cause: Option>, pub whitespace_after_raise: Option>, pub semicolon: Option>, pub(crate) raise_tok: TokenRef<'a>, } impl<'r, 'a> Inflate<'a> for DeflatedRaise<'r, 'a> { type Inflated = Raise<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_after_raise = if self.exc.is_some() { Some(parse_simple_whitespace( config, &mut (*self.raise_tok).whitespace_after.borrow_mut(), )?) } else { Default::default() }; let exc = self.exc.inflate(config)?; let mut cause = self.cause.inflate(config)?; if exc.is_none() { if let Some(cause) = cause.as_mut() { // in `raise from`, `raise` owns the shared whitespace cause.whitespace_before_from = None; } } let semicolon = self.semicolon.inflate(config)?; Ok(Self::Inflated { exc, cause, whitespace_after_raise, semicolon, }) } } impl<'a> Codegen<'a> for Raise<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("raise"); if let Some(ws) = &self.whitespace_after_raise { ws.codegen(state); } else if self.exc.is_some() { state.add_token(" "); } if let Some(exc) = &self.exc { exc.codegen(state); } if let Some(cause) = &self.cause { cause.codegen(state, " "); } if let Some(semi) = &self.semicolon { semi.codegen(state); } } } impl<'r, 'a> DeflatedRaise<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { Self { semicolon, ..self } } } #[cst_node] pub struct NameItem<'a> { pub name: Name<'a>, pub comma: Option>, } impl<'r, 'a> Inflate<'a> for DeflatedNameItem<'r, 'a> { type Inflated = NameItem<'a>; fn inflate(self, config: &Config<'a>) -> Result { let name = self.name.inflate(config)?; let comma = self.comma.inflate(config)?; Ok(Self::Inflated { name, comma }) } } impl<'a> NameItem<'a> { fn codegen(&self, state: &mut CodegenState<'a>, default_comma: bool) { self.name.codegen(state); if let Some(comma) = &self.comma { comma.codegen(state); } else if default_comma { state.add_token(", "); } } } #[cst_node] pub struct Global<'a> { pub names: Vec>, pub whitespace_after_global: SimpleWhitespace<'a>, pub semicolon: Option>, pub(crate) tok: TokenRef<'a>, } impl<'r, 'a> Inflate<'a> for DeflatedGlobal<'r, 'a> { type Inflated = Global<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_after_global = parse_simple_whitespace(config, &mut (*self.tok).whitespace_after.borrow_mut())?; let names = self.names.inflate(config)?; let semicolon = self.semicolon.inflate(config)?; Ok(Self::Inflated { names, whitespace_after_global, semicolon, }) } } impl<'a> Codegen<'a> for Global<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("global"); self.whitespace_after_global.codegen(state); let len = self.names.len(); for (i, name) in self.names.iter().enumerate() { name.codegen(state, i + 1 != len); } if let Some(semicolon) = &self.semicolon { semicolon.codegen(state); } } } impl<'r, 'a> DeflatedGlobal<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { Self { semicolon, ..self } } } #[cst_node] pub struct Nonlocal<'a> { pub names: Vec>, pub whitespace_after_nonlocal: SimpleWhitespace<'a>, pub semicolon: Option>, pub(crate) tok: TokenRef<'a>, } impl<'r, 'a> Inflate<'a> for DeflatedNonlocal<'r, 'a> { type Inflated = Nonlocal<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_after_nonlocal = parse_simple_whitespace(config, &mut (*self.tok).whitespace_after.borrow_mut())?; let names = self.names.inflate(config)?; let semicolon = self.semicolon.inflate(config)?; Ok(Self::Inflated { names, whitespace_after_nonlocal, semicolon, }) } } impl<'a> Codegen<'a> for Nonlocal<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("nonlocal"); self.whitespace_after_nonlocal.codegen(state); let len = self.names.len(); for (i, name) in self.names.iter().enumerate() { name.codegen(state, i + 1 != len); } if let Some(semicolon) = &self.semicolon { semicolon.codegen(state); } } } impl<'r, 'a> DeflatedNonlocal<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { Self { semicolon, ..self } } } #[cst_node] pub struct For<'a> { pub target: AssignTargetExpression<'a>, pub iter: Expression<'a>, pub body: Suite<'a>, pub orelse: Option>, pub asynchronous: Option>, pub leading_lines: Vec>, pub whitespace_after_for: SimpleWhitespace<'a>, pub whitespace_before_in: SimpleWhitespace<'a>, pub whitespace_after_in: SimpleWhitespace<'a>, pub whitespace_before_colon: SimpleWhitespace<'a>, pub(crate) async_tok: Option>, pub(crate) for_tok: TokenRef<'a>, pub(crate) in_tok: TokenRef<'a>, pub(crate) colon_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for For<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for ll in &self.leading_lines { ll.codegen(state); } state.add_indent(); if let Some(asy) = &self.asynchronous { asy.codegen(state); } state.add_token("for"); self.whitespace_after_for.codegen(state); self.target.codegen(state); self.whitespace_before_in.codegen(state); state.add_token("in"); self.whitespace_after_in.codegen(state); self.iter.codegen(state); self.whitespace_before_colon.codegen(state); state.add_token(":"); self.body.codegen(state); if let Some(e) = &self.orelse { e.codegen(state); } } } impl<'r, 'a> Inflate<'a> for DeflatedFor<'r, 'a> { type Inflated = For<'a>; fn inflate(mut self, config: &Config<'a>) -> Result { let (asynchronous, leading_lines) = if let Some(asy) = self.async_tok.as_mut() { let whitespace_after = parse_parenthesizable_whitespace(config, &mut asy.whitespace_after.borrow_mut())?; ( Some(Asynchronous { whitespace_after }), Some(parse_empty_lines( config, &mut asy.whitespace_before.borrow_mut(), None, )?), ) } else { (None, None) }; let leading_lines = if let Some(ll) = leading_lines { ll } else { parse_empty_lines( config, &mut (*self.for_tok).whitespace_before.borrow_mut(), None, )? }; let whitespace_after_for = parse_simple_whitespace(config, &mut (*self.for_tok).whitespace_after.borrow_mut())?; let target = self.target.inflate(config)?; let whitespace_before_in = parse_simple_whitespace(config, &mut (*self.in_tok).whitespace_before.borrow_mut())?; let whitespace_after_in = parse_simple_whitespace(config, &mut (*self.in_tok).whitespace_after.borrow_mut())?; let iter = self.iter.inflate(config)?; let whitespace_before_colon = parse_simple_whitespace( config, &mut (*self.colon_tok).whitespace_before.borrow_mut(), )?; let body = self.body.inflate(config)?; let orelse = self.orelse.inflate(config)?; Ok(Self::Inflated { target, iter, body, orelse, asynchronous, leading_lines, whitespace_after_for, whitespace_before_in, whitespace_after_in, whitespace_before_colon, }) } } #[cst_node] pub struct While<'a> { pub test: Expression<'a>, pub body: Suite<'a>, pub orelse: Option>, pub leading_lines: Vec>, pub whitespace_after_while: SimpleWhitespace<'a>, pub whitespace_before_colon: SimpleWhitespace<'a>, pub(crate) while_tok: TokenRef<'a>, pub(crate) colon_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for While<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for ll in &self.leading_lines { ll.codegen(state); } state.add_indent(); state.add_token("while"); self.whitespace_after_while.codegen(state); self.test.codegen(state); self.whitespace_before_colon.codegen(state); state.add_token(":"); self.body.codegen(state); if let Some(orelse) = &self.orelse { orelse.codegen(state); } } } impl<'r, 'a> Inflate<'a> for DeflatedWhile<'r, 'a> { type Inflated = While<'a>; fn inflate(self, config: &Config<'a>) -> Result { let leading_lines = parse_empty_lines( config, &mut (*self.while_tok).whitespace_before.borrow_mut(), None, )?; let whitespace_after_while = parse_simple_whitespace(config, &mut (*self.while_tok).whitespace_after.borrow_mut())?; let test = self.test.inflate(config)?; let whitespace_before_colon = parse_simple_whitespace( config, &mut (*self.colon_tok).whitespace_before.borrow_mut(), )?; let body = self.body.inflate(config)?; let orelse = self.orelse.inflate(config)?; Ok(Self::Inflated { test, body, orelse, leading_lines, whitespace_after_while, whitespace_before_colon, }) } } #[cst_node] pub struct ClassDef<'a> { pub name: Name<'a>, pub type_parameters: Option>, pub body: Suite<'a>, pub bases: Vec>, pub keywords: Vec>, pub decorators: Vec>, pub lpar: Option>, pub rpar: Option>, pub leading_lines: Vec>, pub lines_after_decorators: Vec>, pub whitespace_after_class: SimpleWhitespace<'a>, pub whitespace_after_name: SimpleWhitespace<'a>, pub whitespace_after_type_parameters: SimpleWhitespace<'a>, pub whitespace_before_colon: SimpleWhitespace<'a>, pub(crate) class_tok: TokenRef<'a>, pub(crate) lpar_tok: Option>, pub(crate) rpar_tok: Option>, pub(crate) colon_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for ClassDef<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for ll in &self.leading_lines { ll.codegen(state); } for dec in &self.decorators { dec.codegen(state); } for lad in &self.lines_after_decorators { lad.codegen(state); } state.add_indent(); state.add_token("class"); self.whitespace_after_class.codegen(state); self.name.codegen(state); self.whitespace_after_name.codegen(state); if let Some(tp) = &self.type_parameters { tp.codegen(state); self.whitespace_after_type_parameters.codegen(state); } let need_parens = !self.bases.is_empty() || !self.keywords.is_empty(); if let Some(lpar) = &self.lpar { lpar.codegen(state); } else if need_parens { state.add_token("("); } let args = self.bases.iter().chain(self.keywords.iter()); let len = self.bases.len() + self.keywords.len(); for (i, arg) in args.enumerate() { arg.codegen(state, i + 1 < len); } if let Some(rpar) = &self.rpar { rpar.codegen(state); } else if need_parens { state.add_token(")"); } self.whitespace_before_colon.codegen(state); state.add_token(":"); self.body.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedClassDef<'r, 'a> { type Inflated = ClassDef<'a>; fn inflate(mut self, config: &Config<'a>) -> Result { let mut leading_lines = parse_empty_lines( config, &mut (*self.class_tok).whitespace_before.borrow_mut(), None, )?; let mut decorators = self.decorators.inflate(config)?; let mut lines_after_decorators = Default::default(); if let Some(dec) = decorators.first_mut() { swap(&mut lines_after_decorators, &mut leading_lines); swap(&mut dec.leading_lines, &mut leading_lines); } let whitespace_after_class = parse_simple_whitespace(config, &mut (*self.class_tok).whitespace_after.borrow_mut())?; let name = self.name.inflate(config)?; let (mut whitespace_after_name, mut type_parameters, mut whitespace_after_type_parameters) = Default::default(); if let Some(tparams) = self.type_parameters { let rbracket_tok = tparams.rbracket.tok.clone(); whitespace_after_name = parse_simple_whitespace( config, &mut tparams.lbracket.tok.whitespace_before.borrow_mut(), )?; type_parameters = Some(tparams.inflate(config)?); whitespace_after_type_parameters = parse_simple_whitespace(config, &mut rbracket_tok.whitespace_after.borrow_mut())?; } else if let Some(lpar_tok) = self.lpar_tok.as_mut() { whitespace_after_name = parse_simple_whitespace(config, &mut lpar_tok.whitespace_before.borrow_mut())?; } let lpar = self.lpar.inflate(config)?; let bases = self.bases.inflate(config)?; let keywords = self.keywords.inflate(config)?; let rpar = self.rpar.inflate(config)?; let whitespace_before_colon = parse_simple_whitespace( config, &mut (*self.colon_tok).whitespace_before.borrow_mut(), )?; let body = self.body.inflate(config)?; Ok(Self::Inflated { name, type_parameters, body, bases, keywords, decorators, lpar, rpar, leading_lines, lines_after_decorators, whitespace_after_class, whitespace_after_type_parameters, whitespace_after_name, whitespace_before_colon, }) } } impl<'r, 'a> DeflatedClassDef<'r, 'a> { pub fn with_decorators(self, decorators: Vec>) -> Self { Self { decorators, ..self } } } #[cst_node] pub struct Finally<'a> { pub body: Suite<'a>, pub leading_lines: Vec>, pub whitespace_before_colon: SimpleWhitespace<'a>, pub(crate) finally_tok: TokenRef<'a>, pub(crate) colon_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for Finally<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for ll in &self.leading_lines { ll.codegen(state); } state.add_indent(); state.add_token("finally"); self.whitespace_before_colon.codegen(state); state.add_token(":"); self.body.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedFinally<'r, 'a> { type Inflated = Finally<'a>; fn inflate(self, config: &Config<'a>) -> Result { let leading_lines = parse_empty_lines( config, &mut (*self.finally_tok).whitespace_before.borrow_mut(), None, )?; let whitespace_before_colon = parse_simple_whitespace( config, &mut (*self.colon_tok).whitespace_before.borrow_mut(), )?; let body = self.body.inflate(config)?; Ok(Self::Inflated { body, leading_lines, whitespace_before_colon, }) } } #[cst_node] pub struct ExceptHandler<'a> { pub body: Suite<'a>, pub r#type: Option>, pub name: Option>, pub leading_lines: Vec>, pub whitespace_after_except: SimpleWhitespace<'a>, pub whitespace_before_colon: SimpleWhitespace<'a>, pub(crate) except_tok: TokenRef<'a>, pub(crate) colon_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for ExceptHandler<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for ll in &self.leading_lines { ll.codegen(state); } state.add_indent(); state.add_token("except"); self.whitespace_after_except.codegen(state); if let Some(t) = &self.r#type { t.codegen(state); } if let Some(n) = &self.name { n.codegen(state); } self.whitespace_before_colon.codegen(state); state.add_token(":"); self.body.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedExceptHandler<'r, 'a> { type Inflated = ExceptHandler<'a>; fn inflate(self, config: &Config<'a>) -> Result { let leading_lines = parse_empty_lines( config, &mut (*self.except_tok).whitespace_before.borrow_mut(), None, )?; let whitespace_after_except = parse_simple_whitespace( config, &mut (*self.except_tok).whitespace_after.borrow_mut(), )?; let r#type = self.r#type.inflate(config)?; let name = self.name.inflate(config)?; let whitespace_before_colon = if name.is_some() { parse_simple_whitespace( config, &mut (*self.colon_tok).whitespace_before.borrow_mut(), )? } else { Default::default() }; let body = self.body.inflate(config)?; Ok(Self::Inflated { body, r#type, name, leading_lines, whitespace_after_except, whitespace_before_colon, }) } } #[cst_node] pub struct ExceptStarHandler<'a> { pub body: Suite<'a>, pub r#type: Expression<'a>, pub name: Option>, pub leading_lines: Vec>, pub whitespace_after_except: SimpleWhitespace<'a>, pub whitespace_after_star: SimpleWhitespace<'a>, pub whitespace_before_colon: SimpleWhitespace<'a>, pub(crate) except_tok: TokenRef<'a>, pub(crate) star_tok: TokenRef<'a>, pub(crate) colon_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for ExceptStarHandler<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for ll in &self.leading_lines { ll.codegen(state); } state.add_indent(); state.add_token("except"); self.whitespace_after_except.codegen(state); state.add_token("*"); self.whitespace_after_star.codegen(state); self.r#type.codegen(state); if let Some(n) = &self.name { n.codegen(state); } self.whitespace_before_colon.codegen(state); state.add_token(":"); self.body.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedExceptStarHandler<'r, 'a> { type Inflated = ExceptStarHandler<'a>; fn inflate(self, config: &Config<'a>) -> Result { let leading_lines = parse_empty_lines( config, &mut self.except_tok.whitespace_before.borrow_mut(), None, )?; let whitespace_after_except = parse_simple_whitespace(config, &mut self.except_tok.whitespace_after.borrow_mut())?; let whitespace_after_star = parse_simple_whitespace(config, &mut self.star_tok.whitespace_after.borrow_mut())?; let r#type = self.r#type.inflate(config)?; let name = self.name.inflate(config)?; let whitespace_before_colon = if name.is_some() { parse_simple_whitespace(config, &mut self.colon_tok.whitespace_before.borrow_mut())? } else { Default::default() }; let body = self.body.inflate(config)?; Ok(Self::Inflated { body, r#type, name, leading_lines, whitespace_after_except, whitespace_after_star, whitespace_before_colon, }) } } #[cst_node] pub struct Try<'a> { pub body: Suite<'a>, pub handlers: Vec>, pub orelse: Option>, pub finalbody: Option>, pub leading_lines: Vec>, pub whitespace_before_colon: SimpleWhitespace<'a>, pub(crate) try_tok: TokenRef<'a>, // colon_tok unnecessary } impl<'a> Codegen<'a> for Try<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for ll in &self.leading_lines { ll.codegen(state); } state.add_indent(); state.add_token("try"); self.whitespace_before_colon.codegen(state); state.add_token(":"); self.body.codegen(state); for h in &self.handlers { h.codegen(state); } if let Some(e) = &self.orelse { e.codegen(state); } if let Some(f) = &self.finalbody { f.codegen(state); } } } impl<'r, 'a> Inflate<'a> for DeflatedTry<'r, 'a> { type Inflated = Try<'a>; fn inflate(self, config: &Config<'a>) -> Result { let leading_lines = parse_empty_lines( config, &mut (*self.try_tok).whitespace_before.borrow_mut(), None, )?; let whitespace_before_colon = parse_simple_whitespace(config, &mut (*self.try_tok).whitespace_after.borrow_mut())?; let body = self.body.inflate(config)?; let handlers = self.handlers.inflate(config)?; let orelse = self.orelse.inflate(config)?; let finalbody = self.finalbody.inflate(config)?; Ok(Self::Inflated { body, handlers, orelse, finalbody, leading_lines, whitespace_before_colon, }) } } #[cst_node] pub struct TryStar<'a> { pub body: Suite<'a>, pub handlers: Vec>, pub orelse: Option>, pub finalbody: Option>, pub leading_lines: Vec>, pub whitespace_before_colon: SimpleWhitespace<'a>, pub(crate) try_tok: TokenRef<'a>, // colon_tok unnecessary } impl<'a> Codegen<'a> for TryStar<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for ll in &self.leading_lines { ll.codegen(state); } state.add_indent(); state.add_token("try"); self.whitespace_before_colon.codegen(state); state.add_token(":"); self.body.codegen(state); for h in &self.handlers { h.codegen(state); } if let Some(e) = &self.orelse { e.codegen(state); } if let Some(f) = &self.finalbody { f.codegen(state); } } } impl<'r, 'a> Inflate<'a> for DeflatedTryStar<'r, 'a> { type Inflated = TryStar<'a>; fn inflate(self, config: &Config<'a>) -> Result { let leading_lines = parse_empty_lines( config, &mut (*self.try_tok).whitespace_before.borrow_mut(), None, )?; let whitespace_before_colon = parse_simple_whitespace(config, &mut (*self.try_tok).whitespace_after.borrow_mut())?; let body = self.body.inflate(config)?; let handlers = self.handlers.inflate(config)?; let orelse = self.orelse.inflate(config)?; let finalbody = self.finalbody.inflate(config)?; Ok(Self::Inflated { body, handlers, orelse, finalbody, leading_lines, whitespace_before_colon, }) } } #[cst_node] pub struct AugAssign<'a> { pub target: AssignTargetExpression<'a>, pub operator: AugOp<'a>, pub value: Expression<'a>, pub semicolon: Option>, } impl<'r, 'a> Inflate<'a> for DeflatedAugAssign<'r, 'a> { type Inflated = AugAssign<'a>; fn inflate(self, config: &Config<'a>) -> Result { let target = self.target.inflate(config)?; let operator = self.operator.inflate(config)?; let value = self.value.inflate(config)?; let semicolon = self.semicolon.inflate(config)?; Ok(Self::Inflated { target, operator, value, semicolon, }) } } impl<'a> Codegen<'a> for AugAssign<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.target.codegen(state); self.operator.codegen(state); self.value.codegen(state); if let Some(s) = &self.semicolon { s.codegen(state); } } } impl<'r, 'a> DeflatedAugAssign<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { Self { semicolon, ..self } } } #[cst_node] pub struct WithItem<'a> { pub item: Expression<'a>, pub asname: Option>, pub comma: Option>, } impl<'r, 'a> DeflatedWithItem<'r, 'a> { fn inflate_withitem(self, config: &Config<'a>, is_last: bool) -> Result> { let item = self.item.inflate(config)?; let asname = self.asname.inflate(config)?; let comma = if is_last { self.comma.map(|c| c.inflate_before(config)).transpose()? } else { self.comma.map(|c| c.inflate(config)).transpose()? }; Ok(WithItem { item, asname, comma, }) } } impl<'a> Codegen<'a> for WithItem<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.item.codegen(state); if let Some(n) = &self.asname { n.codegen(state); } if let Some(c) = &self.comma { c.codegen(state); } } } impl<'r, 'a> WithComma<'r, 'a> for DeflatedWithItem<'r, 'a> { fn with_comma(self, comma: DeflatedComma<'r, 'a>) -> Self { Self { comma: Some(comma), ..self } } } #[cst_node] pub struct With<'a> { pub items: Vec>, pub body: Suite<'a>, pub asynchronous: Option>, pub leading_lines: Vec>, pub lpar: Option>, pub rpar: Option>, pub whitespace_after_with: SimpleWhitespace<'a>, pub whitespace_before_colon: SimpleWhitespace<'a>, pub(crate) async_tok: Option>, pub(crate) with_tok: TokenRef<'a>, pub(crate) colon_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for With<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for ll in &self.leading_lines { ll.codegen(state); } state.add_indent(); if let Some(asy) = &self.asynchronous { asy.codegen(state); } state.add_token("with"); self.whitespace_after_with.codegen(state); // TODO: Force parens whenever there are newlines in // the commas of self.items. // // For now, only the python API does this. let need_parens = false; if let Some(lpar) = &self.lpar { lpar.codegen(state); } else if need_parens { state.add_token("("); } let len = self.items.len(); for (i, item) in self.items.iter().enumerate() { item.codegen(state); if item.comma.is_none() && i + 1 < len { state.add_token(", "); } } if let Some(rpar) = &self.rpar { rpar.codegen(state); } else if need_parens { state.add_token(")"); } self.whitespace_before_colon.codegen(state); state.add_token(":"); self.body.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedWith<'r, 'a> { type Inflated = With<'a>; fn inflate(mut self, config: &Config<'a>) -> Result { let (asynchronous, leading_lines) = if let Some(asy) = self.async_tok.as_mut() { let whitespace_after = parse_parenthesizable_whitespace(config, &mut asy.whitespace_after.borrow_mut())?; ( Some(Asynchronous { whitespace_after }), Some(parse_empty_lines( config, &mut asy.whitespace_before.borrow_mut(), None, )?), ) } else { (None, None) }; let leading_lines = if let Some(ll) = leading_lines { ll } else { parse_empty_lines( config, &mut (*self.with_tok).whitespace_before.borrow_mut(), None, )? }; let whitespace_after_with = parse_simple_whitespace(config, &mut (*self.with_tok).whitespace_after.borrow_mut())?; let lpar = self.lpar.map(|lpar| lpar.inflate(config)).transpose()?; let len = self.items.len(); let items = self .items .into_iter() .enumerate() .map(|(idx, el)| el.inflate_withitem(config, idx + 1 == len)) .collect::>>()?; let rpar = if !items.is_empty() { // rpar only has whitespace if items is non empty self.rpar.map(|rpar| rpar.inflate(config)).transpose()? } else { Default::default() }; let whitespace_before_colon = parse_simple_whitespace( config, &mut (*self.colon_tok).whitespace_before.borrow_mut(), )?; let body = self.body.inflate(config)?; Ok(Self::Inflated { items, body, asynchronous, leading_lines, lpar, rpar, whitespace_after_with, whitespace_before_colon, }) } } #[cst_node(Codegen, ParenthesizedNode, Inflate)] pub enum DelTargetExpression<'a> { Name(Box>), Attribute(Box>), Tuple(Box>), List(Box>), Subscript(Box>), } impl<'r, 'a> std::convert::From> for DeflatedExpression<'r, 'a> { fn from(d: DeflatedDelTargetExpression<'r, 'a>) -> Self { match d { DeflatedDelTargetExpression::Attribute(a) => Self::Attribute(a), DeflatedDelTargetExpression::List(l) => Self::List(l), DeflatedDelTargetExpression::Name(n) => Self::Name(n), DeflatedDelTargetExpression::Subscript(s) => Self::Subscript(s), DeflatedDelTargetExpression::Tuple(t) => Self::Tuple(t), } } } impl<'r, 'a> std::convert::From> for DeflatedElement<'r, 'a> { fn from(d: DeflatedDelTargetExpression<'r, 'a>) -> Self { Self::Simple { value: d.into(), comma: None, } } } #[cst_node] pub struct Del<'a> { pub target: DelTargetExpression<'a>, pub whitespace_after_del: SimpleWhitespace<'a>, pub semicolon: Option>, pub(crate) tok: TokenRef<'a>, } impl<'r, 'a> Inflate<'a> for DeflatedDel<'r, 'a> { type Inflated = Del<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_after_del = parse_simple_whitespace(config, &mut (*self.tok).whitespace_after.borrow_mut())?; let target = self.target.inflate(config)?; let semicolon = self.semicolon.inflate(config)?; Ok(Self::Inflated { target, whitespace_after_del, semicolon, }) } } impl<'a> Codegen<'a> for Del<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("del"); self.whitespace_after_del.codegen(state); self.target.codegen(state); if let Some(semi) = &self.semicolon { semi.codegen(state); } } } impl<'r, 'a> DeflatedDel<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { Self { semicolon, ..self } } } #[cst_node] pub struct Match<'a> { pub subject: Expression<'a>, pub cases: Vec>, pub leading_lines: Vec>, pub whitespace_after_match: SimpleWhitespace<'a>, pub whitespace_before_colon: SimpleWhitespace<'a>, pub whitespace_after_colon: TrailingWhitespace<'a>, pub indent: Option<&'a str>, pub footer: Vec>, pub(crate) match_tok: TokenRef<'a>, pub(crate) colon_tok: TokenRef<'a>, pub(crate) indent_tok: TokenRef<'a>, pub(crate) dedent_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for Match<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for l in &self.leading_lines { l.codegen(state); } state.add_indent(); state.add_token("match"); self.whitespace_after_match.codegen(state); self.subject.codegen(state); self.whitespace_before_colon.codegen(state); state.add_token(":"); self.whitespace_after_colon.codegen(state); let indent = self.indent.unwrap_or(state.default_indent); state.indent(indent); // Note: empty cases is a syntax error for c in &self.cases { c.codegen(state); } for f in &self.footer { f.codegen(state); } state.dedent(); } } impl<'r, 'a> Inflate<'a> for DeflatedMatch<'r, 'a> { type Inflated = Match<'a>; fn inflate(self, config: &Config<'a>) -> Result { let leading_lines = parse_empty_lines( config, &mut self.match_tok.whitespace_before.borrow_mut(), None, )?; let whitespace_after_match = parse_simple_whitespace(config, &mut self.match_tok.whitespace_after.borrow_mut())?; let subject = self.subject.inflate(config)?; let whitespace_before_colon = parse_simple_whitespace(config, &mut self.colon_tok.whitespace_before.borrow_mut())?; let whitespace_after_colon = parse_trailing_whitespace(config, &mut self.colon_tok.whitespace_after.borrow_mut())?; let mut indent = self.indent_tok.relative_indent; if indent == Some(config.default_indent) { indent = None; } let cases = self.cases.inflate(config)?; // See note about footers in `IndentedBlock`'s inflate fn let footer = parse_empty_lines( config, &mut self.dedent_tok.whitespace_after.borrow_mut(), Some(self.indent_tok.whitespace_before.borrow().absolute_indent), )?; Ok(Self::Inflated { subject, cases, leading_lines, whitespace_after_match, whitespace_before_colon, whitespace_after_colon, indent, footer, }) } } #[cst_node] pub struct MatchCase<'a> { pub pattern: MatchPattern<'a>, pub guard: Option>, pub body: Suite<'a>, pub leading_lines: Vec>, pub whitespace_after_case: SimpleWhitespace<'a>, pub whitespace_before_if: SimpleWhitespace<'a>, pub whitespace_after_if: SimpleWhitespace<'a>, pub whitespace_before_colon: SimpleWhitespace<'a>, pub(crate) case_tok: TokenRef<'a>, pub(crate) if_tok: Option>, pub(crate) colon_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for MatchCase<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { for l in &self.leading_lines { l.codegen(state); } state.add_indent(); state.add_token("case"); self.whitespace_after_case.codegen(state); self.pattern.codegen(state); if let Some(guard) = &self.guard { self.whitespace_before_if.codegen(state); state.add_token("if"); self.whitespace_after_if.codegen(state); guard.codegen(state); } self.whitespace_before_colon.codegen(state); state.add_token(":"); self.body.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedMatchCase<'r, 'a> { type Inflated = MatchCase<'a>; fn inflate(mut self, config: &Config<'a>) -> Result { let leading_lines = parse_empty_lines( config, &mut self.case_tok.whitespace_before.borrow_mut(), None, )?; let whitespace_after_case = parse_simple_whitespace(config, &mut self.case_tok.whitespace_after.borrow_mut())?; let pattern = self.pattern.inflate(config)?; let (whitespace_before_if, whitespace_after_if, guard) = if let Some(if_tok) = self.if_tok.as_mut() { ( parse_simple_whitespace(config, &mut if_tok.whitespace_before.borrow_mut())?, parse_simple_whitespace(config, &mut if_tok.whitespace_after.borrow_mut())?, self.guard.inflate(config)?, ) } else { Default::default() }; let whitespace_before_colon = parse_simple_whitespace(config, &mut self.colon_tok.whitespace_before.borrow_mut())?; let body = self.body.inflate(config)?; Ok(Self::Inflated { pattern, guard, body, leading_lines, whitespace_after_case, whitespace_before_if, whitespace_after_if, whitespace_before_colon, }) } } #[allow(clippy::large_enum_variant)] #[cst_node(Codegen, Inflate, ParenthesizedNode)] pub enum MatchPattern<'a> { Value(MatchValue<'a>), Singleton(MatchSingleton<'a>), Sequence(MatchSequence<'a>), Mapping(MatchMapping<'a>), Class(MatchClass<'a>), As(Box>), Or(Box>), } #[cst_node] pub struct MatchValue<'a> { pub value: Expression<'a>, } impl<'a> ParenthesizedNode<'a> for MatchValue<'a> { fn lpar(&self) -> &Vec> { self.value.lpar() } fn rpar(&self) -> &Vec> { self.value.rpar() } fn parenthesize(&self, state: &mut CodegenState<'a>, f: F) where F: FnOnce(&mut CodegenState<'a>), { self.value.parenthesize(state, f) } fn with_parens(self, left: LeftParen<'a>, right: RightParen<'a>) -> Self { Self { value: self.value.with_parens(left, right), } } } impl<'a> Codegen<'a> for MatchValue<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.value.codegen(state) } } impl<'r, 'a> Inflate<'a> for DeflatedMatchValue<'r, 'a> { type Inflated = MatchValue<'a>; fn inflate(self, config: &Config<'a>) -> Result { let value = self.value.inflate(config)?; Ok(Self::Inflated { value }) } } impl<'r, 'a> ParenthesizedDeflatedNode<'r, 'a> for DeflatedMatchValue<'r, 'a> { fn lpar(&self) -> &Vec> { self.value.lpar() } fn rpar(&self) -> &Vec> { self.value.rpar() } fn with_parens( self, left: DeflatedLeftParen<'r, 'a>, right: DeflatedRightParen<'r, 'a>, ) -> Self { Self { value: self.value.with_parens(left, right), } } } #[cst_node] pub struct MatchSingleton<'a> { pub value: Name<'a>, } impl<'a> ParenthesizedNode<'a> for MatchSingleton<'a> { fn lpar(&self) -> &Vec> { self.value.lpar() } fn rpar(&self) -> &Vec> { self.value.rpar() } fn parenthesize(&self, state: &mut CodegenState<'a>, f: F) where F: FnOnce(&mut CodegenState<'a>), { self.value.parenthesize(state, f) } fn with_parens(self, left: LeftParen<'a>, right: RightParen<'a>) -> Self { Self { value: self.value.with_parens(left, right), } } } impl<'a> Codegen<'a> for MatchSingleton<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.value.codegen(state) } } impl<'r, 'a> Inflate<'a> for DeflatedMatchSingleton<'r, 'a> { type Inflated = MatchSingleton<'a>; fn inflate(self, config: &Config<'a>) -> Result { let value = self.value.inflate(config)?; Ok(Self::Inflated { value }) } } impl<'r, 'a> ParenthesizedDeflatedNode<'r, 'a> for DeflatedMatchSingleton<'r, 'a> { fn lpar(&self) -> &Vec> { self.value.lpar() } fn rpar(&self) -> &Vec> { self.value.rpar() } fn with_parens( self, left: DeflatedLeftParen<'r, 'a>, right: DeflatedRightParen<'r, 'a>, ) -> Self { Self { value: self.value.with_parens(left, right), } } } #[allow(clippy::large_enum_variant)] #[cst_node(Codegen, Inflate, ParenthesizedNode)] pub enum MatchSequence<'a> { MatchList(MatchList<'a>), MatchTuple(MatchTuple<'a>), } #[cst_node(ParenthesizedNode)] pub struct MatchList<'a> { pub patterns: Vec>, pub lbracket: Option>, pub rbracket: Option>, pub lpar: Vec>, pub rpar: Vec>, } impl<'a> Codegen<'a> for MatchList<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.lbracket.codegen(state); let len = self.patterns.len(); if len == 1 { self.patterns.first().unwrap().codegen(state, false, false); } else { for (idx, pat) in self.patterns.iter().enumerate() { pat.codegen(state, idx < len - 1, true); } } self.rbracket.codegen(state); }) } } impl<'r, 'a> Inflate<'a> for DeflatedMatchList<'r, 'a> { type Inflated = MatchList<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let lbracket = self.lbracket.inflate(config)?; let len = self.patterns.len(); let patterns = self .patterns .into_iter() .enumerate() .map(|(idx, el)| el.inflate_element(config, idx + 1 == len)) .collect::>>()?; let rbracket = self.rbracket.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { patterns, lbracket, rbracket, lpar, rpar, }) } } #[cst_node(ParenthesizedNode)] pub struct MatchTuple<'a> { pub patterns: Vec>, pub lpar: Vec>, pub rpar: Vec>, } impl<'a> Codegen<'a> for MatchTuple<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { let len = self.patterns.len(); if len == 1 { self.patterns.first().unwrap().codegen(state, true, false); } else { for (idx, pat) in self.patterns.iter().enumerate() { pat.codegen(state, idx < len - 1, true); } } }) } } impl<'r, 'a> Inflate<'a> for DeflatedMatchTuple<'r, 'a> { type Inflated = MatchTuple<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let len = self.patterns.len(); let patterns = self .patterns .into_iter() .enumerate() .map(|(idx, el)| el.inflate_element(config, idx + 1 == len)) .collect::>>()?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { patterns, lpar, rpar, }) } } #[allow(clippy::large_enum_variant)] #[cst_node] pub enum StarrableMatchSequenceElement<'a> { Simple(MatchSequenceElement<'a>), Starred(MatchStar<'a>), } impl<'a> StarrableMatchSequenceElement<'a> { fn codegen( &self, state: &mut CodegenState<'a>, default_comma: bool, default_comma_whitespace: bool, ) { match &self { Self::Simple(s) => s.codegen(state, default_comma, default_comma_whitespace), Self::Starred(s) => s.codegen(state, default_comma, default_comma_whitespace), } } } impl<'r, 'a> DeflatedStarrableMatchSequenceElement<'r, 'a> { fn inflate_element( self, config: &Config<'a>, last_element: bool, ) -> Result> { Ok(match self { Self::Simple(s) => { StarrableMatchSequenceElement::Simple(s.inflate_element(config, last_element)?) } Self::Starred(s) => { StarrableMatchSequenceElement::Starred(s.inflate_element(config, last_element)?) } }) } } impl<'r, 'a> WithComma<'r, 'a> for DeflatedStarrableMatchSequenceElement<'r, 'a> { fn with_comma(self, comma: DeflatedComma<'r, 'a>) -> Self { match self { Self::Simple(s) => Self::Simple(s.with_comma(comma)), Self::Starred(s) => Self::Starred(s.with_comma(comma)), } } } #[cst_node] pub struct MatchSequenceElement<'a> { pub value: MatchPattern<'a>, pub comma: Option>, } impl<'a> MatchSequenceElement<'a> { fn codegen( &self, state: &mut CodegenState<'a>, default_comma: bool, default_comma_whitespace: bool, ) { self.value.codegen(state); self.comma.codegen(state); if self.comma.is_none() && default_comma { state.add_token(if default_comma_whitespace { ", " } else { "," }); } } } impl<'r, 'a> DeflatedMatchSequenceElement<'r, 'a> { fn inflate_element( self, config: &Config<'a>, last_element: bool, ) -> Result> { let value = self.value.inflate(config)?; let comma = if last_element { self.comma.map(|c| c.inflate_before(config)).transpose() } else { self.comma.inflate(config) }?; Ok(MatchSequenceElement { value, comma }) } } impl<'r, 'a> WithComma<'r, 'a> for DeflatedMatchSequenceElement<'r, 'a> { fn with_comma(self, comma: DeflatedComma<'r, 'a>) -> Self { Self { comma: Some(comma), ..self } } } #[cst_node] pub struct MatchStar<'a> { pub name: Option>, pub comma: Option>, pub whitespace_before_name: ParenthesizableWhitespace<'a>, pub(crate) star_tok: TokenRef<'a>, } impl<'a> MatchStar<'a> { fn codegen( &self, state: &mut CodegenState<'a>, default_comma: bool, default_comma_whitespace: bool, ) { state.add_token("*"); self.whitespace_before_name.codegen(state); if let Some(name) = &self.name { name.codegen(state); } else { state.add_token("_"); } self.comma.codegen(state); if self.comma.is_none() && default_comma { state.add_token(if default_comma_whitespace { ", " } else { "," }); } } } impl<'r, 'a> DeflatedMatchStar<'r, 'a> { fn inflate_element(self, config: &Config<'a>, last_element: bool) -> Result> { let whitespace_before_name = parse_parenthesizable_whitespace( config, &mut self.star_tok.whitespace_after.borrow_mut(), )?; let name = self.name.inflate(config)?; let comma = if last_element { self.comma.map(|c| c.inflate_before(config)).transpose() } else { self.comma.inflate(config) }?; Ok(MatchStar { name, comma, whitespace_before_name, }) } } impl<'r, 'a> WithComma<'r, 'a> for DeflatedMatchStar<'r, 'a> { fn with_comma(self, comma: DeflatedComma<'r, 'a>) -> Self { Self { comma: Some(comma), ..self } } } #[cst_node(ParenthesizedNode)] pub struct MatchMapping<'a> { pub elements: Vec>, pub rest: Option>, pub trailing_comma: Option>, pub lbrace: LeftCurlyBrace<'a>, pub rbrace: RightCurlyBrace<'a>, pub lpar: Vec>, pub rpar: Vec>, pub whitespace_before_rest: SimpleWhitespace<'a>, pub(crate) star_tok: Option>, } impl<'a> Codegen<'a> for MatchMapping<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.lbrace.codegen(state); let len = self.elements.len(); for (idx, el) in self.elements.iter().enumerate() { el.codegen(state, self.rest.is_some() || idx < len - 1); } if let Some(rest) = &self.rest { state.add_token("**"); self.whitespace_before_rest.codegen(state); rest.codegen(state); self.trailing_comma.codegen(state); } self.rbrace.codegen(state); }) } } impl<'r, 'a> Inflate<'a> for DeflatedMatchMapping<'r, 'a> { type Inflated = MatchMapping<'a>; fn inflate(mut self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let lbrace = self.lbrace.inflate(config)?; let len = self.elements.len(); let no_star = self.star_tok.is_none(); let elements = self .elements .into_iter() .enumerate() .map(|(idx, el)| el.inflate_element(config, no_star && idx + 1 == len)) .collect::>>()?; let (whitespace_before_rest, rest, trailing_comma) = if let Some(star_tok) = self.star_tok.as_mut() { ( parse_simple_whitespace(config, &mut star_tok.whitespace_after.borrow_mut())?, self.rest.inflate(config)?, self.trailing_comma .map(|c| c.inflate_before(config)) .transpose()?, ) } else { Default::default() }; let rbrace = self.rbrace.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { elements, rest, trailing_comma, lbrace, rbrace, lpar, rpar, whitespace_before_rest, }) } } #[cst_node] pub struct MatchMappingElement<'a> { pub key: Expression<'a>, pub pattern: MatchPattern<'a>, pub comma: Option>, pub whitespace_before_colon: ParenthesizableWhitespace<'a>, pub whitespace_after_colon: ParenthesizableWhitespace<'a>, pub(crate) colon_tok: TokenRef<'a>, } impl<'a> MatchMappingElement<'a> { fn codegen(&self, state: &mut CodegenState<'a>, default_comma: bool) { self.key.codegen(state); self.whitespace_before_colon.codegen(state); state.add_token(":"); self.whitespace_after_colon.codegen(state); self.pattern.codegen(state); self.comma.codegen(state); if self.comma.is_none() && default_comma { state.add_token(", "); } } } impl<'r, 'a> DeflatedMatchMappingElement<'r, 'a> { fn inflate_element( self, config: &Config<'a>, last_element: bool, ) -> Result> { let key = self.key.inflate(config)?; let whitespace_before_colon = parse_parenthesizable_whitespace( config, &mut self.colon_tok.whitespace_before.borrow_mut(), )?; let whitespace_after_colon = parse_parenthesizable_whitespace( config, &mut self.colon_tok.whitespace_after.borrow_mut(), )?; let pattern = self.pattern.inflate(config)?; let comma = if last_element { self.comma.map(|c| c.inflate_before(config)).transpose() } else { self.comma.inflate(config) }?; Ok(MatchMappingElement { key, pattern, comma, whitespace_before_colon, whitespace_after_colon, }) } } impl<'r, 'a> WithComma<'r, 'a> for DeflatedMatchMappingElement<'r, 'a> { fn with_comma(self, comma: DeflatedComma<'r, 'a>) -> Self { Self { comma: Some(comma), ..self } } } #[cst_node(ParenthesizedNode)] pub struct MatchClass<'a> { pub cls: NameOrAttribute<'a>, pub patterns: Vec>, pub kwds: Vec>, pub lpar: Vec>, pub rpar: Vec>, pub whitespace_after_cls: ParenthesizableWhitespace<'a>, pub whitespace_before_patterns: ParenthesizableWhitespace<'a>, pub whitespace_after_kwds: ParenthesizableWhitespace<'a>, pub(crate) lpar_tok: TokenRef<'a>, pub(crate) rpar_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for MatchClass<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { self.cls.codegen(state); self.whitespace_after_cls.codegen(state); state.add_token("("); self.whitespace_before_patterns.codegen(state); let patlen = self.patterns.len(); let kwdlen = self.kwds.len(); for (idx, pat) in self.patterns.iter().enumerate() { pat.codegen(state, idx < patlen - 1 + kwdlen, patlen == 1 && kwdlen == 0); } for (idx, kwd) in self.kwds.iter().enumerate() { kwd.codegen(state, idx < kwdlen - 1); } self.whitespace_after_kwds.codegen(state); state.add_token(")"); }) } } impl<'r, 'a> Inflate<'a> for DeflatedMatchClass<'r, 'a> { type Inflated = MatchClass<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let cls = self.cls.inflate(config)?; let whitespace_after_cls = parse_parenthesizable_whitespace( config, &mut self.lpar_tok.whitespace_before.borrow_mut(), )?; let whitespace_before_patterns = parse_parenthesizable_whitespace( config, &mut self.lpar_tok.whitespace_after.borrow_mut(), )?; let patlen = self.patterns.len(); let kwdlen = self.kwds.len(); let patterns = self .patterns .into_iter() .enumerate() .map(|(idx, pat)| pat.inflate_element(config, idx + 1 == patlen + kwdlen)) .collect::>()?; let kwds = self .kwds .into_iter() .enumerate() .map(|(idx, kwd)| kwd.inflate_element(config, idx + 1 == kwdlen)) .collect::>()?; let whitespace_after_kwds = parse_parenthesizable_whitespace( config, &mut self.rpar_tok.whitespace_before.borrow_mut(), )?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { cls, patterns, kwds, lpar, rpar, whitespace_after_cls, whitespace_before_patterns, whitespace_after_kwds, }) } } #[cst_node] pub struct MatchKeywordElement<'a> { pub key: Name<'a>, pub pattern: MatchPattern<'a>, pub comma: Option>, pub whitespace_before_equal: ParenthesizableWhitespace<'a>, pub whitespace_after_equal: ParenthesizableWhitespace<'a>, pub(crate) equal_tok: TokenRef<'a>, } impl<'a> MatchKeywordElement<'a> { fn codegen(&self, state: &mut CodegenState<'a>, default_comma: bool) { self.key.codegen(state); self.whitespace_before_equal.codegen(state); state.add_token("="); self.whitespace_after_equal.codegen(state); self.pattern.codegen(state); self.comma.codegen(state); if self.comma.is_none() && default_comma { state.add_token(", "); } } } impl<'r, 'a> DeflatedMatchKeywordElement<'r, 'a> { fn inflate_element( self, config: &Config<'a>, last_element: bool, ) -> Result> { let key = self.key.inflate(config)?; let whitespace_before_equal = parse_parenthesizable_whitespace( config, &mut self.equal_tok.whitespace_before.borrow_mut(), )?; let whitespace_after_equal = parse_parenthesizable_whitespace( config, &mut self.equal_tok.whitespace_after.borrow_mut(), )?; let pattern = self.pattern.inflate(config)?; let comma = if last_element { self.comma.map(|c| c.inflate_before(config)).transpose() } else { self.comma.inflate(config) }?; Ok(MatchKeywordElement { key, pattern, comma, whitespace_before_equal, whitespace_after_equal, }) } } impl<'r, 'a> WithComma<'r, 'a> for DeflatedMatchKeywordElement<'r, 'a> { fn with_comma(self, comma: DeflatedComma<'r, 'a>) -> Self { Self { comma: Some(comma), ..self } } } #[cst_node(ParenthesizedNode)] pub struct MatchAs<'a> { pub pattern: Option>, pub name: Option>, pub lpar: Vec>, pub rpar: Vec>, pub whitespace_before_as: Option>, pub whitespace_after_as: Option>, pub(crate) as_tok: Option>, } impl<'a> Codegen<'a> for MatchAs<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { if let Some(pat) = &self.pattern { pat.codegen(state); self.whitespace_before_as.codegen(state); state.add_token("as"); self.whitespace_after_as.codegen(state); } if let Some(name) = &self.name { name.codegen(state); } else { state.add_token("_"); } }) } } impl<'r, 'a> Inflate<'a> for DeflatedMatchAs<'r, 'a> { type Inflated = MatchAs<'a>; fn inflate(mut self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let pattern = self.pattern.inflate(config)?; let (whitespace_before_as, whitespace_after_as) = if let Some(as_tok) = self.as_tok.as_mut() { ( Some(parse_parenthesizable_whitespace( config, &mut as_tok.whitespace_before.borrow_mut(), )?), Some(parse_parenthesizable_whitespace( config, &mut as_tok.whitespace_after.borrow_mut(), )?), ) } else { Default::default() }; let name = self.name.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { pattern, name, lpar, rpar, whitespace_before_as, whitespace_after_as, }) } } #[cst_node] pub struct MatchOrElement<'a> { pub pattern: MatchPattern<'a>, pub separator: Option>, } impl<'a> MatchOrElement<'a> { fn codegen(&self, state: &mut CodegenState<'a>, default_separator: bool) { self.pattern.codegen(state); self.separator.codegen(state); if self.separator.is_none() && default_separator { state.add_token(" | "); } } } impl<'r, 'a> Inflate<'a> for DeflatedMatchOrElement<'r, 'a> { type Inflated = MatchOrElement<'a>; fn inflate(self, config: &Config<'a>) -> Result { let pattern = self.pattern.inflate(config)?; let separator = self.separator.inflate(config)?; Ok(Self::Inflated { pattern, separator }) } } #[cst_node(ParenthesizedNode)] pub struct MatchOr<'a> { pub patterns: Vec>, pub lpar: Vec>, pub rpar: Vec>, } impl<'a> Codegen<'a> for MatchOr<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.parenthesize(state, |state| { let len = self.patterns.len(); for (idx, pat) in self.patterns.iter().enumerate() { pat.codegen(state, idx + 1 < len) } }) } } impl<'r, 'a> Inflate<'a> for DeflatedMatchOr<'r, 'a> { type Inflated = MatchOr<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lpar = self.lpar.inflate(config)?; let patterns = self.patterns.inflate(config)?; let rpar = self.rpar.inflate(config)?; Ok(Self::Inflated { patterns, lpar, rpar, }) } } #[cst_node] pub struct TypeVar<'a> { pub name: Name<'a>, pub bound: Option>>, pub colon: Option>, } impl<'a> Codegen<'a> for TypeVar<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.name.codegen(state); self.colon.codegen(state); if let Some(bound) = &self.bound { bound.codegen(state); } } } impl<'r, 'a> Inflate<'a> for DeflatedTypeVar<'r, 'a> { type Inflated = TypeVar<'a>; fn inflate(self, config: &Config<'a>) -> Result { let name = self.name.inflate(config)?; let colon = self.colon.inflate(config)?; let bound = self.bound.inflate(config)?; Ok(Self::Inflated { name, bound, colon }) } } #[cst_node] pub struct TypeVarTuple<'a> { pub name: Name<'a>, pub whitespace_after_star: SimpleWhitespace<'a>, pub(crate) star_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for TypeVarTuple<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("*"); self.whitespace_after_star.codegen(state); self.name.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedTypeVarTuple<'r, 'a> { type Inflated = TypeVarTuple<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_after_star = parse_simple_whitespace(config, &mut self.star_tok.whitespace_after.borrow_mut())?; let name = self.name.inflate(config)?; Ok(Self::Inflated { name, whitespace_after_star, }) } } #[cst_node] pub struct ParamSpec<'a> { pub name: Name<'a>, pub whitespace_after_star: SimpleWhitespace<'a>, pub(crate) star_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for ParamSpec<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("**"); self.whitespace_after_star.codegen(state); self.name.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedParamSpec<'r, 'a> { type Inflated = ParamSpec<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_after_star = parse_simple_whitespace(config, &mut self.star_tok.whitespace_after.borrow_mut())?; let name = self.name.inflate(config)?; Ok(Self::Inflated { name, whitespace_after_star, }) } } #[cst_node(Inflate, Codegen)] pub enum TypeVarLike<'a> { TypeVar(TypeVar<'a>), TypeVarTuple(TypeVarTuple<'a>), ParamSpec(ParamSpec<'a>), } #[cst_node] pub struct TypeParam<'a> { pub param: TypeVarLike<'a>, pub comma: Option>, } impl<'a> Codegen<'a> for TypeParam<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.param.codegen(state); self.comma.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedTypeParam<'r, 'a> { type Inflated = TypeParam<'a>; fn inflate(self, config: &Config<'a>) -> Result { let param = self.param.inflate(config)?; let comma = self.comma.inflate(config)?; Ok(Self::Inflated { param, comma }) } } impl<'r, 'a> WithComma<'r, 'a> for DeflatedTypeParam<'r, 'a> { fn with_comma(self, comma: DeflatedComma<'r, 'a>) -> Self { Self { comma: Some(comma), ..self } } } #[cst_node] pub struct TypeParameters<'a> { pub params: Vec>, pub lbracket: LeftSquareBracket<'a>, pub rbracket: RightSquareBracket<'a>, } impl<'a> Codegen<'a> for TypeParameters<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.lbracket.codegen(state); let params_len = self.params.len(); for (idx, param) in self.params.iter().enumerate() { param.codegen(state); if idx + 1 < params_len && param.comma.is_none() { state.add_token(", "); } } self.rbracket.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedTypeParameters<'r, 'a> { type Inflated = TypeParameters<'a>; fn inflate(self, config: &Config<'a>) -> Result { let lbracket = self.lbracket.inflate(config)?; let params = self.params.inflate(config)?; let rbracket = self.rbracket.inflate(config)?; Ok(Self::Inflated { params, lbracket, rbracket, }) } } #[cst_node] pub struct TypeAlias<'a> { pub name: Name<'a>, pub value: Box>, pub type_parameters: Option>, pub whitespace_after_type: SimpleWhitespace<'a>, pub whitespace_after_name: Option>, pub whitespace_after_type_parameters: Option>, pub whitespace_after_equals: SimpleWhitespace<'a>, pub semicolon: Option>, pub(crate) type_tok: TokenRef<'a>, pub(crate) lbracket_tok: Option>, pub(crate) equals_tok: TokenRef<'a>, } impl<'a> Codegen<'a> for TypeAlias<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token("type"); self.whitespace_after_type.codegen(state); self.name.codegen(state); if self.whitespace_after_name.is_none() && self.type_parameters.is_none() { state.add_token(" "); } else { self.whitespace_after_name.codegen(state); } if self.type_parameters.is_some() { self.type_parameters.codegen(state); self.whitespace_after_type_parameters.codegen(state); } state.add_token("="); self.whitespace_after_equals.codegen(state); self.value.codegen(state); self.semicolon.codegen(state); } } impl<'r, 'a> Inflate<'a> for DeflatedTypeAlias<'r, 'a> { type Inflated = TypeAlias<'a>; fn inflate(self, config: &Config<'a>) -> Result { let whitespace_after_type = parse_simple_whitespace(config, &mut self.type_tok.whitespace_after.borrow_mut())?; let name = self.name.inflate(config)?; let whitespace_after_name = Some(if let Some(tok) = self.lbracket_tok { parse_simple_whitespace(config, &mut tok.whitespace_before.borrow_mut()) } else { parse_simple_whitespace(config, &mut self.equals_tok.whitespace_before.borrow_mut()) }?); let type_parameters = self.type_parameters.inflate(config)?; let whitespace_after_type_parameters = if type_parameters.is_some() { Some(parse_simple_whitespace( config, &mut self.equals_tok.whitespace_before.borrow_mut(), )?) } else { None }; let whitespace_after_equals = parse_simple_whitespace(config, &mut self.equals_tok.whitespace_after.borrow_mut())?; let value = self.value.inflate(config)?; let semicolon = self.semicolon.inflate(config)?; Ok(Self::Inflated { name, value, type_parameters, whitespace_after_type, whitespace_after_name, whitespace_after_type_parameters, whitespace_after_equals, semicolon, }) } } impl<'r, 'a> DeflatedTypeAlias<'r, 'a> { pub fn with_semicolon(self, semicolon: Option>) -> Self { Self { semicolon, ..self } } } libcst-0.1.0/src/nodes/test_utils.rs000064400000000000000000000022030072674642500155730ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. use pyo3::prelude::*; py_import!("libcst._nodes.deep_equals", "deep_equals", get_deep_equals); pub fn repr_or_panic(py: Python, value: T) -> String where T: ToPyObject, { value .to_object(py) .as_ref(py) .repr() .expect("failed to call repr") .extract() .expect("repr should've returned str") } pub fn py_assert_deep_equals(py: Python, left: L, right: R) where L: ToPyObject, R: ToPyObject, { let (left, right) = (left.to_object(py), right.to_object(py)); let equals = get_deep_equals(py) .expect("failed to import deep_equals") .call1((&left, &right)) .expect("failed to call deep_equals") .extract::() .expect("deep_equals should return a bool"); if !equals { panic!( "assertion failed: {} was not deeply equal to {}", repr_or_panic(py, &left), repr_or_panic(py, &right), ); } } libcst-0.1.0/src/nodes/traits.rs000064400000000000000000000126370072674642500147160ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree use crate::{ nodes::expression::{DeflatedLeftParen, DeflatedRightParen}, nodes::op::DeflatedComma, tokenizer::whitespace_parser::{Config, WhitespaceError}, Codegen, CodegenState, EmptyLine, LeftParen, RightParen, }; use std::ops::Deref; pub trait WithComma<'r, 'a> { fn with_comma(self, comma: DeflatedComma<'r, 'a>) -> Self; } pub trait ParenthesizedNode<'a> { fn lpar(&self) -> &Vec>; fn rpar(&self) -> &Vec>; fn parenthesize(&self, state: &mut CodegenState<'a>, f: F) where F: FnOnce(&mut CodegenState<'a>), { for lpar in self.lpar() { lpar.codegen(state); } f(state); for rpar in self.rpar() { rpar.codegen(state); } } fn with_parens(self, left: LeftParen<'a>, right: RightParen<'a>) -> Self; } impl<'a, T: ParenthesizedNode<'a>> ParenthesizedNode<'a> for Box { fn lpar(&self) -> &Vec> { self.deref().lpar() } fn rpar(&self) -> &Vec> { self.deref().rpar() } fn parenthesize(&self, state: &mut CodegenState<'a>, f: F) where F: FnOnce(&mut CodegenState<'a>), { self.deref().parenthesize(state, f) } fn with_parens(self, left: LeftParen<'a>, right: RightParen<'a>) -> Self { Self::new((*self).with_parens(left, right)) } } pub trait ParenthesizedDeflatedNode<'r, 'a> { fn lpar(&self) -> &Vec>; fn rpar(&self) -> &Vec>; fn with_parens( self, left: DeflatedLeftParen<'r, 'a>, right: DeflatedRightParen<'r, 'a>, ) -> Self; } impl<'r, 'a, T: ParenthesizedDeflatedNode<'r, 'a>> ParenthesizedDeflatedNode<'r, 'a> for Box { fn lpar(&self) -> &Vec> { self.deref().lpar() } fn rpar(&self) -> &Vec> { self.deref().rpar() } fn with_parens( self, left: DeflatedLeftParen<'r, 'a>, right: DeflatedRightParen<'r, 'a>, ) -> Self { Self::new((*self).with_parens(left, right)) } } pub trait WithLeadingLines<'a> { fn leading_lines(&mut self) -> &mut Vec>; } pub type Result = std::result::Result; pub trait Inflate<'a> where Self: Sized, { type Inflated; fn inflate(self, config: &Config<'a>) -> Result; } impl<'a, T: Inflate<'a>> Inflate<'a> for Option { type Inflated = Option; fn inflate(self, config: &Config<'a>) -> Result { self.map(|x| x.inflate(config)).transpose() } } impl<'a, T: Inflate<'a> + ?Sized> Inflate<'a> for Box { type Inflated = Box; fn inflate(self, config: &Config<'a>) -> Result { match (*self).inflate(config) { Ok(a) => Ok(Box::new(a)), Err(e) => Err(e), } } } impl<'a, T: Inflate<'a>> Inflate<'a> for Vec { type Inflated = Vec; fn inflate(self, config: &Config<'a>) -> Result { self.into_iter().map(|item| item.inflate(config)).collect() } } #[cfg(feature = "py")] pub mod py { use pyo3::{types::PyTuple, AsPyPointer, IntoPy, PyObject, PyResult, Python}; // TODO: replace with upstream implementation once // https://github.com/PyO3/pyo3/issues/1813 is resolved pub trait TryIntoPy: Sized { fn try_into_py(self, py: Python) -> PyResult; } // I wish: // impl> TryIntoPy for T { // fn try_into_py(self, py: Python) -> PyResult { // Ok(self.into_py(py)) // } // } impl TryIntoPy for bool { fn try_into_py(self, py: Python) -> PyResult { Ok(self.into_py(py)) } } impl> TryIntoPy for Box where T: TryIntoPy, { fn try_into_py(self, py: Python) -> PyResult { (*self).try_into_py(py) } } impl TryIntoPy for Option where T: TryIntoPy, { fn try_into_py(self, py: Python) -> PyResult { Ok(match self { None => py.None(), Some(x) => x.try_into_py(py)?, }) } } impl TryIntoPy for Vec where T: TryIntoPy, { fn try_into_py(self, py: Python) -> PyResult { let converted = self .into_iter() .map(|x| x.try_into_py(py)) .collect::>>()? .into_iter(); Ok(PyTuple::new(py, converted).into()) } } impl TryIntoPy for PyTuple { fn try_into_py(self, py: Python) -> PyResult { Ok(self.into_py(py)) } } impl<'a> TryIntoPy for &'a str { fn try_into_py(self, py: Python) -> PyResult { Ok(self.into_py(py)) } } impl TryIntoPy for &'_ T where T: AsPyPointer, { fn try_into_py(self, py: Python) -> PyResult { Ok(self.into_py(py)) } } } libcst-0.1.0/src/nodes/whitespace.rs000064400000000000000000000106370072674642500155420ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. #[cfg(feature = "py")] use libcst_derive::TryIntoPy; use super::{Codegen, CodegenState}; #[derive(Debug, Eq, PartialEq, Default, Clone)] #[cfg_attr(feature = "py", derive(TryIntoPy))] pub struct SimpleWhitespace<'a>(pub &'a str); impl<'a> Codegen<'a> for SimpleWhitespace<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token(self.0); } } #[derive(Debug, Eq, PartialEq, Clone)] #[cfg_attr(feature = "py", derive(TryIntoPy))] pub struct Comment<'a>(pub &'a str); impl<'a> Default for Comment<'a> { fn default() -> Self { Self("#") } } impl<'a> Codegen<'a> for Comment<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { state.add_token(self.0); } } #[derive(Debug, Eq, PartialEq, Default, Clone)] #[cfg_attr(feature = "py", derive(TryIntoPy))] pub struct Newline<'a>(pub Option<&'a str>, pub Fakeness); #[derive(Debug, PartialEq, Eq, Clone)] pub enum Fakeness { Fake, Real, } impl Default for Fakeness { fn default() -> Self { Self::Real } } impl<'a> Codegen<'a> for Newline<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { if let Fakeness::Fake = self.1 { return; } if let Some(value) = self.0 { state.add_token(value); } else { state.add_token(state.default_newline); } } } #[derive(Debug, Eq, PartialEq, Default, Clone)] #[cfg_attr(feature = "py", derive(TryIntoPy))] pub struct TrailingWhitespace<'a> { pub whitespace: SimpleWhitespace<'a>, pub comment: Option>, pub newline: Newline<'a>, } impl<'a> Codegen<'a> for TrailingWhitespace<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.whitespace.codegen(state); if let Some(comment) = &self.comment { comment.codegen(state); } self.newline.codegen(state); } } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(feature = "py", derive(TryIntoPy))] pub struct EmptyLine<'a> { pub indent: bool, pub whitespace: SimpleWhitespace<'a>, pub comment: Option>, pub newline: Newline<'a>, } impl<'a> Codegen<'a> for EmptyLine<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { if self.indent { state.add_indent() } self.whitespace.codegen(state); if let Some(comment) = &self.comment { comment.codegen(state); } self.newline.codegen(state); } } impl<'a> Default for EmptyLine<'a> { fn default() -> Self { Self { indent: true, whitespace: Default::default(), comment: Default::default(), newline: Default::default(), } } } impl<'a> EmptyLine<'a> { pub fn new( indent: bool, whitespace: SimpleWhitespace<'a>, comment: Option>, newline: Newline<'a>, ) -> Self { Self { indent, whitespace, comment, newline, } } } #[derive(Debug, Eq, PartialEq, Default, Clone)] #[cfg_attr(feature = "py", derive(TryIntoPy))] pub struct ParenthesizedWhitespace<'a> { pub first_line: TrailingWhitespace<'a>, pub empty_lines: Vec>, pub indent: bool, pub last_line: SimpleWhitespace<'a>, } impl<'a> Codegen<'a> for ParenthesizedWhitespace<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { self.first_line.codegen(state); for line in &self.empty_lines { line.codegen(state); } if self.indent { state.add_indent() } self.last_line.codegen(state); } } #[derive(Debug, Eq, PartialEq, Clone)] #[cfg_attr(feature = "py", derive(TryIntoPy))] pub enum ParenthesizableWhitespace<'a> { SimpleWhitespace(SimpleWhitespace<'a>), ParenthesizedWhitespace(ParenthesizedWhitespace<'a>), } impl<'a> Codegen<'a> for ParenthesizableWhitespace<'a> { fn codegen(&self, state: &mut CodegenState<'a>) { match self { Self::SimpleWhitespace(w) => w.codegen(state), Self::ParenthesizedWhitespace(w) => w.codegen(state), } } } impl<'a> Default for ParenthesizableWhitespace<'a> { fn default() -> Self { Self::SimpleWhitespace(SimpleWhitespace("")) } } libcst-0.1.0/src/parser/errors.rs000064400000000000000000000057760072674642500151160ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree use crate::parser::grammar::TokVec; use crate::tokenizer::whitespace_parser::WhitespaceError; use crate::tokenizer::TokError; use peg::Parse; use thiserror::Error; #[allow(clippy::enum_variant_names)] #[derive(Debug, Error, PartialEq, Eq)] pub enum ParserError<'a> { #[error("tokenizer error: {0}")] TokenizerError(TokError<'a>, &'a str), #[error("parser error: {0}")] ParserError( peg::error::ParseError< as Parse>::PositionRepr>, &'a str, ), #[error(transparent)] WhitespaceError(#[from] WhitespaceError), #[error("invalid operator")] OperatorError, } #[cfg(feature = "py")] mod py_error { use pyo3::types::{IntoPyDict, PyModule}; use pyo3::{IntoPy, PyErr, PyErrArguments, Python}; use super::ParserError; struct Details { message: String, lines: Vec, raw_line: u32, raw_column: u32, } impl<'a> From> for PyErr { fn from(e: ParserError) -> Self { Python::with_gil(|py| { let lines = match &e { ParserError::TokenizerError(_, text) | ParserError::ParserError(_, text) => { text.lines().collect::>() } _ => vec![""], }; let (mut line, mut col) = match &e { ParserError::ParserError(err, ..) => { (err.location.start_pos.line, err.location.start_pos.column) } _ => (0, 0), }; if line + 1 > lines.len() { line = lines.len() - 1; col = 0; } let kwargs = [ ("message", e.to_string().into_py(py)), ("lines", lines.into_py(py)), ("raw_line", (line + 1).into_py(py)), ("raw_column", col.into_py(py)), ] .into_py_dict(py); let libcst = PyModule::import(py, "libcst").expect("libcst cannot be imported"); PyErr::from_value( libcst .getattr("ParserSyntaxError") .expect("ParserSyntaxError not found") .call((), Some(kwargs)) .expect("failed to instantiate"), ) }) } } impl<'a> PyErrArguments for Details { fn arguments(self, py: pyo3::Python) -> pyo3::PyObject { [ ("message", self.message.into_py(py)), ("lines", self.lines.into_py(py)), ("raw_line", self.raw_line.into_py(py)), ("raw_column", self.raw_column.into_py(py)), ] .into_py_dict(py) .into_py(py) } } } libcst-0.1.0/src/parser/grammar.rs000064400000000000000000003350100072674642500152130ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. use std::rc::Rc; use crate::expression::make_async; use crate::nodes::deflated::*; use crate::nodes::expression::make_fstringtext; use crate::nodes::op::make_importstar; use crate::nodes::traits::ParenthesizedDeflatedNode; use crate::parser::ParserError; use crate::tokenizer::{TokType, Token}; use crate::WithComma; use peg::str::LineCol; use peg::{parser, Parse, ParseElem, RuleResult}; use TokType::{ Async, Await as AWAIT, Dedent, EndMarker, FStringEnd, FStringStart, FStringString, Indent, Name as NameTok, Newline as NL, Number, String as STRING, }; pub type Result<'a, T> = std::result::Result>; type GrammarResult = std::result::Result; #[derive(Debug)] pub struct TokVec<'a>(Vec>>); impl<'a> std::convert::From>> for TokVec<'a> { fn from(vec: Vec>) -> Self { TokVec(vec.into_iter().map(Rc::new).collect()) } } #[derive(Debug, PartialEq, Eq)] pub struct ParseLoc { pub start_pos: LineCol, pub end_pos: LineCol, } impl std::fmt::Display for ParseLoc { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.start_pos.fmt(f) } } impl<'a> Parse for TokVec<'a> { type PositionRepr = ParseLoc; fn start(&self) -> usize { 0 } fn is_eof(&self, pos: usize) -> bool { pos >= self.0.len() } fn position_repr(&self, pos: usize) -> Self::PositionRepr { let tok = self.0.get(pos).unwrap_or_else(|| self.0.last().unwrap()); ParseLoc { start_pos: LineCol { line: tok.start_pos.line_number(), column: tok.start_pos.char_column_number(), offset: tok.start_pos.byte_idx(), }, end_pos: LineCol { line: tok.end_pos.line_number(), column: tok.end_pos.char_column_number(), offset: tok.end_pos.byte_idx(), }, } } } type TokenRef<'input, 'a> = &'input Token<'a>; impl<'input, 'a: 'input> ParseElem<'input> for TokVec<'a> { type Element = TokenRef<'input, 'a>; fn parse_elem(&'input self, pos: usize) -> RuleResult { match self.0.get(pos) { Some(tok) => RuleResult::Matched(pos + 1, tok), None => RuleResult::Failed, } } } const MAX_RECURSION_DEPTH: usize = 3000; parser! { pub grammar python<'a>(input: &'a str) for TokVec<'a> { // Starting Rules pub rule file(encoding: Option<&str>) -> Module<'input, 'a> = traced(<_file(encoding.unwrap_or("utf-8"))>) pub rule expression_input() -> Expression<'input, 'a> = traced() pub rule statement_input() -> Statement<'input, 'a> = traced() rule _file(encoding: &str) -> Module<'input, 'a> = s:statements()? eof:tok(EndMarker, "EOF") { make_module(s.unwrap_or_default(), eof, encoding) } // General statements rule statements() -> Vec> = statement()+ rule statement() -> Statement<'input, 'a> = c:compound_stmt() { Statement::Compound(c) } / s:simple_stmts() { Statement::Simple(make_simple_statement_line(s)) } rule simple_stmts() -> SimpleStatementParts<'input, 'a> = first_tok:&_ stmts:separated_trailer(, ) nl:tok(NL, "NEWLINE") { SimpleStatementParts { first_tok, first_statement: stmts.0, rest: stmts.1, last_semi: stmts.2, nl, } } #[cache] rule simple_stmt() -> SmallStatement<'input, 'a> = assignment() / &lit("type") s: type_stmt() {SmallStatement::TypeAlias(s)} / e:star_expressions() { SmallStatement::Expr(Expr { value: e, semicolon: None }) } / &lit("return") s:return_stmt() { SmallStatement::Return(s) } // this is expanded from the original grammar's import_stmt rule / &lit("import") i:import_name() { SmallStatement::Import(i) } / &lit("from") i:import_from() { SmallStatement::ImportFrom(i) } / &lit("raise") r:raise_stmt() { SmallStatement::Raise(r) } / lit("pass") { SmallStatement::Pass(Pass { semicolon: None }) } / &lit("del") s:del_stmt() { SmallStatement::Del(s) } / &lit("yield") s:yield_stmt() { SmallStatement::Expr(Expr { value: s, semicolon: None }) } / &lit("assert") s:assert_stmt() {SmallStatement::Assert(s)} / lit("break") { SmallStatement::Break(Break { semicolon: None })} / lit("continue") { SmallStatement::Continue(Continue { semicolon: None })} / &lit("global") s:global_stmt() {SmallStatement::Global(s)} / &lit("nonlocal") s:nonlocal_stmt() {SmallStatement::Nonlocal(s)} rule compound_stmt() -> CompoundStatement<'input, 'a> = &(lit("def") / lit("@") / tok(Async, "ASYNC")) f:function_def() { CompoundStatement::FunctionDef(f) } / &lit("if") f:if_stmt() { CompoundStatement::If(f) } / &(lit("class") / lit("@")) c:class_def() { CompoundStatement::ClassDef(c) } / &(lit("with") / tok(Async, "ASYNC")) w:with_stmt() { CompoundStatement::With(w) } / &(lit("for") / tok(Async, "ASYNC")) f:for_stmt() { CompoundStatement::For(f) } / &lit("try") t:try_stmt() { CompoundStatement::Try(t) } / &lit("try") t:try_star_stmt() { CompoundStatement::TryStar(t) } / &lit("while") w:while_stmt() { CompoundStatement::While(w) } / m:match_stmt() { CompoundStatement::Match(m) } // Simple statements rule assignment() -> SmallStatement<'input, 'a> = a:name() col:lit(":") ann:expression() rhs:(eq:lit("=") d:annotated_rhs() {(eq, d)})? { SmallStatement::AnnAssign(make_ann_assignment( AssignTargetExpression::Name(Box::new(a)), col, ann, rhs)) } // TODO: there's an extra '(' single_target ')' clause here in upstream / a:single_subscript_attribute_target() col:lit(":") ann:expression() rhs:(eq:lit("=") d:annotated_rhs() {(eq, d)})? { SmallStatement::AnnAssign(make_ann_assignment(a, col, ann, rhs)) } / lhs:(t:star_targets() eq:lit("=") {(t, eq)})+ rhs:(yield_expr() / star_expressions()) !lit("=") { SmallStatement::Assign(make_assignment(lhs, rhs)) } / t:single_target() op:augassign() rhs:(yield_expr() / star_expressions()) { SmallStatement::AugAssign(make_aug_assign(t, op, rhs)) } rule annotated_rhs() -> Expression<'input, 'a> = yield_expr() / star_expressions() rule augassign() -> AugOp<'input, 'a> = &(lit("+=") / lit("-=") / lit("*=") / lit("@=") / lit("/=") / lit("%=") / lit("&=") / lit("|=") / lit("^=") / lit("<<=") / lit(">>=") / lit("**=") / lit("//=")) tok:_ {? make_aug_op(tok).map_err(|_| "aug_op") } rule return_stmt() -> Return<'input, 'a> = kw:lit("return") a:star_expressions()? { make_return(kw, a) } rule raise_stmt() -> Raise<'input, 'a> = kw:lit("raise") exc:expression() rest:(f:lit("from") cau:expression() {(f, cau)})? { make_raise(kw, Some(exc), rest) } / kw:lit("raise") { make_raise(kw, None, None) } rule global_stmt() -> Global<'input, 'a> = kw:lit("global") init:(n:name() c:comma() {(n, c)})* last:name() { make_global(kw, init, last) } rule nonlocal_stmt() -> Nonlocal<'input, 'a> = kw:lit("nonlocal") init:(n:name() c:comma() {(n, c)})* last:name() { make_nonlocal(kw, init, last) } rule del_stmt() -> Del<'input, 'a> = kw:lit("del") t:del_target() &(lit(";") / tok(NL, "NEWLINE")) { make_del(kw, t) } / kw:lit("del") t:del_targets() &(lit(";") / tok(NL, "NEWLINE")) { make_del(kw, make_del_tuple(None, t, None)) } rule yield_stmt() -> Expression<'input, 'a> = yield_expr() rule assert_stmt() -> Assert<'input, 'a> = kw:lit("assert") test:expression() rest:(c:comma() msg:expression() {(c, msg)})? { make_assert(kw, test, rest) } // Import statements rule import_name() -> Import<'input, 'a> = kw:lit("import") a:dotted_as_names() { make_import(kw, a) } rule import_from() -> ImportFrom<'input, 'a> = from:lit("from") dots:dots()? m:dotted_name() import:lit("import") als:import_from_targets() { make_import_from(from, dots.unwrap_or_default(), Some(m), import, als) } / from:lit("from") dots:dots() import:lit("import") als:import_from_targets() { make_import_from(from, dots, None, import, als) } rule import_from_targets() -> ParenthesizedImportNames<'input, 'a> = lpar:lpar() als:import_from_as_names() c:comma()? rpar:rpar() { let mut als = als; if let (comma@Some(_), Some(mut last)) = (c, als.last_mut()) { last.comma = comma; } (Some(lpar), ImportNames::Aliases(als), Some(rpar)) } / als:import_from_as_names() !lit(",") { (None, ImportNames::Aliases(als), None)} / star:lit("*") { (None, ImportNames::Star(make_importstar()), None) } rule import_from_as_names() -> Vec> = items:separated(, ) { make_import_from_as_names(items.0, items.1) } rule import_from_as_name() -> ImportAlias<'input, 'a> = n:name() asname:(kw:lit("as") z:name() {(kw, z)})? { make_import_alias(NameOrAttribute::N(Box::new(n)), asname) } rule dotted_as_names() -> Vec> = init:(d:dotted_as_name() c:comma() {d.with_comma(c)})* last:dotted_as_name() { concat(init, vec![last]) } rule dotted_as_name() -> ImportAlias<'input, 'a> = n:dotted_name() asname:(kw:lit("as") z:name() {(kw, z)})? { make_import_alias(n, asname) } // TODO: why does this diverge from CPython? rule dotted_name() -> NameOrAttribute<'input, 'a> = first:name() tail:(dot:lit(".") n:name() {(dot, n)})* { make_name_or_attr(first, tail) } // Compound statements // Common elements #[cache] rule block() -> Suite<'input, 'a> = n:tok(NL, "NEWLINE") ind:tok(Indent, "INDENT") s:statements() ded:tok(Dedent, "DEDENT") { make_indented_block(n, ind, s, ded) } / s:simple_stmts() { make_simple_statement_suite(s) } rule decorators() -> Vec> = (at:lit("@") e:named_expression() nl:tok(NL, "NEWLINE") { make_decorator(at, e, nl) } )+ // Class definitions rule class_def() -> ClassDef<'input, 'a> = d:decorators() c:class_def_raw() { c.with_decorators(d) } / class_def_raw() rule class_def_raw() -> ClassDef<'input, 'a> = kw:lit("class") n:name() t:type_params()? arg:(l:lpar() a:arguments()? r:rpar() {(l, a, r)})? col:lit(":") b:block() {? make_class_def(kw, n, t, arg, col, b) } // Function definitions rule function_def() -> FunctionDef<'input, 'a> = d:decorators() f:function_def_raw() {f.with_decorators(d)} / function_def_raw() rule _returns() -> Annotation<'input, 'a> = l:lit("->") e:expression() { make_annotation(l, e) } rule function_def_raw() -> FunctionDef<'input, 'a> = def:lit("def") n:name() t:type_params()? op:lit("(") params:params()? cp:lit(")") ty:_returns()? c:lit(":") b:block() { make_function_def(None, def, n, t, op, params, cp, ty, c, b) } / asy:tok(Async, "ASYNC") def:lit("def") n:name() t:type_params()? op:lit("(") params:params()? cp:lit(")") ty:_returns()? c:lit(":") b:block() { make_function_def(Some(asy), def, n, t, op, params, cp, ty, c, b) } // Function parameters rule params() -> Parameters<'input, 'a> = parameters() rule parameters() -> Parameters<'input, 'a> = a:slash_no_default() b:param_no_default()* c:param_with_default()* d:star_etc()? { make_parameters(Some(a), concat(b, c), d) } / a:slash_with_default() b:param_with_default()* d:star_etc()? { make_parameters(Some(a), b, d) } / a:param_no_default()+ b:param_with_default()* d:star_etc()? { make_parameters(None, concat(a, b), d) } / a:param_with_default()+ d:star_etc()? { make_parameters(None, a, d) } / d:star_etc() { make_parameters(None, vec![], Some(d)) } rule slash_no_default() -> (Vec>, ParamSlash<'input, 'a>) = a:param_no_default()+ tok:lit("/") com:comma() { (a, ParamSlash { comma: Some(com), tok }) } / a:param_no_default()+ tok:lit("/") &lit(")") { (a, ParamSlash { comma: None, tok }) } rule slash_with_default() -> (Vec>, ParamSlash<'input, 'a>) = a:param_no_default()* b:param_with_default()+ tok:lit("/") c:comma() { (concat(a, b), ParamSlash { comma: Some(c), tok }) } / a:param_no_default()* b:param_with_default()+ tok:lit("/") &lit(")") { (concat(a, b), ParamSlash { comma: None, tok }) } rule star_etc() -> StarEtc<'input, 'a> = star:lit("*") a:param_no_default() b:param_maybe_default()* kw:kwds()? { StarEtc(Some(StarArg::Param(Box::new( add_param_star(a, star)))), b, kw) } / star:lit("*") a:param_no_default_star_annotation() b:param_maybe_default()* kw:kwds()? { StarEtc(Some(StarArg::Param(Box::new( add_param_star(a, star)))), b, kw) } / lit("*") c:comma() b:param_maybe_default()+ kw:kwds()? { StarEtc(Some(StarArg::Star(Box::new(ParamStar {comma:c }))), b, kw) } / kw:kwds() { StarEtc(None, vec![], Some(kw)) } rule kwds() -> Param<'input, 'a> = star:lit("**") a:param_no_default() { add_param_star(a, star) } rule param_no_default() -> Param<'input, 'a> = a:param() c:lit(",") { add_param_default(a, None, Some(c)) } / a:param() &lit(")") {a} rule param_no_default_star_annotation() -> Param<'input, 'a> = a:param_star_annotation() c:lit(",") { add_param_default(a, None, Some(c))} / a:param_star_annotation() &lit(")") {a} rule param_with_default() -> Param<'input, 'a> = a:param() def:default() c:lit(",") { add_param_default(a, Some(def), Some(c)) } / a:param() def:default() &lit(")") { add_param_default(a, Some(def), None) } rule param_maybe_default() -> Param<'input, 'a> = a:param() def:default()? c:lit(",") { add_param_default(a, def, Some(c)) } / a:param() def:default()? &lit(")") { add_param_default(a, def, None) } rule param() -> Param<'input, 'a> = n:name() a:annotation()? { Param {name: n, annotation: a, ..Default::default() } } rule param_star_annotation() -> Param<'input, 'a> = n:name() a:star_annotation() { Param {name: n, annotation: Some(a), ..Default::default() } } rule annotation() -> Annotation<'input, 'a> = col:lit(":") e:expression() { make_annotation(col, e) } rule star_annotation() -> Annotation<'input, 'a> = col:lit(":") e:star_expression() { make_annotation(col, e) } rule default() -> (AssignEqual<'input, 'a>, Expression<'input, 'a>) = eq:lit("=") ex:expression() { (make_assign_equal(eq), ex) } // If statement rule if_stmt() -> If<'input, 'a> = i:lit("if") a:named_expression() col:lit(":") b:block() elif:elif_stmt() { make_if(i, a, col, b, Some(OrElse::Elif(elif)), false) } / i:lit("if") a:named_expression() col:lit(":") b:block() el:else_block()? { make_if(i, a, col, b, el.map(OrElse::Else), false) } rule elif_stmt() -> If<'input, 'a> = i:lit("elif") a:named_expression() col:lit(":") b:block() elif:elif_stmt() { make_if(i, a, col, b, Some(OrElse::Elif(elif)), true) } / i:lit("elif") a:named_expression() col:lit(":") b:block() el:else_block()? { make_if(i, a, col, b, el.map(OrElse::Else), true) } rule else_block() -> Else<'input, 'a> = el:lit("else") col:lit(":") b:block() { make_else(el, col, b) } // While statement rule while_stmt() -> While<'input, 'a> = kw:lit("while") test:named_expression() col:lit(":") b:block() el:else_block()? { make_while(kw, test, col, b, el) } // For statement rule for_stmt() -> For<'input, 'a> = f:lit("for") t:star_targets() i:lit("in") it:star_expressions() c:lit(":") b:block() el:else_block()? { make_for(None, f, t, i, it, c, b, el) } / asy:tok(Async, "ASYNC") f:lit("for") t:star_targets() i:lit("in") it:star_expressions() c:lit(":") b:block() el:else_block()? { make_for(Some(asy), f, t, i, it, c, b, el) } // With statement rule with_stmt() -> With<'input, 'a> = kw:lit("with") l:lpar() items:separated_trailer(, ) r:rpar() col:lit(":") b:block() { make_with(None, kw, Some(l), comma_separate(items.0, items.1, items.2), Some(r), col, b) } / kw:lit("with") items:separated(, ) col:lit(":") b:block() { make_with(None, kw, None, comma_separate(items.0, items.1, None), None, col, b) } / asy:tok(Async, "ASYNC") kw:lit("with") l:lpar() items:separated_trailer(, ) r:rpar() col:lit(":") b:block() { make_with(Some(asy), kw, Some(l), comma_separate(items.0, items.1, items.2), Some(r), col, b) } / asy:tok(Async, "ASYNC") kw:lit("with") items:separated(, ) col:lit(":") b:block() { make_with(Some(asy), kw, None, comma_separate(items.0, items.1, None), None, col, b) } rule with_item() -> WithItem<'input, 'a> = e:expression() a:lit("as") t:star_target() &(lit(",") / lit(":") / rpar()) { make_with_item(e, Some(a), Some(t)) } / e:expression() { make_with_item(e, None, None) } // Try statement rule try_stmt() -> Try<'input, 'a> = kw:lit("try") lit(":") b:block() f:finally_block() { make_try(kw, b, vec![], None, Some(f)) } / kw:lit("try") lit(":") b:block() ex:except_block()+ el:else_block()? f:finally_block()? { make_try(kw, b, ex, el, f) } // Note: this is separate because TryStar is a different type in LibCST rule try_star_stmt() -> TryStar<'input, 'a> = kw:lit("try") lit(":") b:block() ex:except_star_block()+ el:else_block()? f:finally_block()? { make_try_star(kw, b, ex, el, f) } // Except statement rule except_block() -> ExceptHandler<'input, 'a> = kw:lit("except") e:expression() a:(k:lit("as") n:name() {(k, n)})? col:lit(":") b:block() { make_except(kw, Some(e), a, col, b) } / kw:lit("except") col:lit(":") b:block() { make_except(kw, None, None, col, b) } rule except_star_block() -> ExceptStarHandler<'input, 'a> = kw:lit("except") star:lit("*") e:expression() a:(k:lit("as") n:name() {(k, n)})? col:lit(":") b:block() { make_except_star(kw, star, e, a, col, b) } rule finally_block() -> Finally<'input, 'a> = kw:lit("finally") col:lit(":") b:block() { make_finally(kw, col, b) } // Match statement rule match_stmt() -> Match<'input, 'a> = kw:lit("match") subject:subject_expr() col:lit(":") tok(NL, "NEWLINE") i:tok(Indent, "INDENT") cases:case_block()+ d:tok(Dedent, "DEDENT") { make_match(kw, subject, col, i, cases, d) } rule subject_expr() -> Expression<'input, 'a> = first:star_named_expression() c:comma() rest:star_named_expressions()? { Expression::Tuple(Box::new( make_tuple_from_elements(first.with_comma(c), rest.unwrap_or_default())) ) } / named_expression() rule case_block() -> MatchCase<'input, 'a> = kw:lit("case") pattern:patterns() guard:guard()? col:lit(":") body:block() { make_case(kw, pattern, guard, col, body) } rule guard() -> (TokenRef<'input, 'a>, Expression<'input, 'a>) = kw:lit("if") exp:named_expression() { (kw, exp) } rule patterns() -> MatchPattern<'input, 'a> = pats:open_sequence_pattern() { MatchPattern::Sequence(make_list_pattern(None, pats, None)) } / pattern() rule pattern() -> MatchPattern<'input, 'a> = as_pattern() / or_pattern() rule as_pattern() -> MatchPattern<'input, 'a> = pat:or_pattern() kw:lit("as") target:pattern_capture_target() { make_as_pattern(Some(pat), Some(kw), Some(target)) } rule or_pattern() -> MatchPattern<'input, 'a> = pats:separated(, ) { make_or_pattern(pats.0, pats.1) } rule closed_pattern() -> MatchPattern<'input, 'a> = literal_pattern() / capture_pattern() / wildcard_pattern() / value_pattern() / group_pattern() / sequence_pattern() / mapping_pattern() / class_pattern() rule literal_pattern() -> MatchPattern<'input, 'a> = val:signed_number() !(lit("+") / lit("-")) { make_match_value(val) } / val:complex_number() { make_match_value(val) } / val:strings() { make_match_value(val.into()) } / n:lit("None") { make_match_singleton(make_name(n)) } / n:lit("True") { make_match_singleton(make_name(n)) } / n:lit("False") { make_match_singleton(make_name(n)) } rule literal_expr() -> Expression<'input, 'a> = val:signed_number() !(lit("+") / lit("-")) { val } / val:complex_number() { val } / val:strings() { val.into() } / n:lit("None") { Expression::Name(Box::new(make_name(n))) } / n:lit("True") { Expression::Name(Box::new(make_name(n))) } / n:lit("False") { Expression::Name(Box::new(make_name(n))) } rule complex_number() -> Expression<'input, 'a> = re:signed_real_number() op:(lit("+")/lit("-")) im:imaginary_number() {? make_binary_op(re, op, im).map_err(|_| "complex number") } rule signed_number() -> Expression<'input, 'a> = n:tok(Number, "number") { make_number(n) } / op:lit("-") n:tok(Number, "number") {? make_unary_op(op, make_number(n)).map_err(|_| "signed number") } rule signed_real_number() -> Expression<'input, 'a> = real_number() / op:lit("-") n:real_number() {? make_unary_op(op, n).map_err(|_| "signed real number") } rule real_number() -> Expression<'input, 'a> = n:tok(Number, "number") {? ensure_real_number(n) } rule imaginary_number() -> Expression<'input, 'a> = n:tok(Number, "number") {? ensure_imaginary_number(n) } rule capture_pattern() -> MatchPattern<'input, 'a> = t:pattern_capture_target() { make_as_pattern(None, None, Some(t)) } rule pattern_capture_target() -> Name<'input, 'a> = !lit("_") n:name() !(lit(".") / lit("(") / lit("=")) { n } rule wildcard_pattern() -> MatchPattern<'input, 'a> = lit("_") { make_as_pattern(None, None, None) } rule value_pattern() -> MatchPattern<'input, 'a> = v:attr() !(lit(".") / lit("(") / lit("=")) { make_match_value(v.into()) } // In upstream attr and name_or_attr are mutually recursive, but rust-peg // doesn't support this yet. rule attr() -> NameOrAttribute<'input, 'a> = &(name() lit(".")) v:name_or_attr() { v } #[cache_left_rec] rule name_or_attr() -> NameOrAttribute<'input, 'a> = val:name_or_attr() d:lit(".") attr:name() { NameOrAttribute::A(Box::new(make_attribute(val.into(), d, attr))) } / n:name() { NameOrAttribute::N(Box::new(n)) } rule group_pattern() -> MatchPattern<'input, 'a> = l:lpar() pat:pattern() r:rpar() { pat.with_parens(l, r) } rule sequence_pattern() -> MatchPattern<'input, 'a> = l:lbrak() pats:maybe_sequence_pattern()? r:rbrak() { MatchPattern::Sequence( make_list_pattern(Some(l), pats.unwrap_or_default(), Some(r)) ) } / l:lpar() pats:open_sequence_pattern()? r:rpar() { MatchPattern::Sequence(make_tuple_pattern(l, pats.unwrap_or_default(), r)) } rule open_sequence_pattern() -> Vec> = pat:maybe_star_pattern() c:comma() pats:maybe_sequence_pattern()? { make_open_sequence_pattern(pat, c, pats.unwrap_or_default()) } rule maybe_sequence_pattern() -> Vec> = pats:separated_trailer(, ) { comma_separate(pats.0, pats.1, pats.2) } rule maybe_star_pattern() -> StarrableMatchSequenceElement<'input, 'a> = s:star_pattern() { StarrableMatchSequenceElement::Starred(s) } / p:pattern() { StarrableMatchSequenceElement::Simple( make_match_sequence_element(p) ) } rule star_pattern() -> MatchStar<'input, 'a> = star:lit("*") t:pattern_capture_target() {make_match_star(star, Some(t))} / star:lit("*") t:wildcard_pattern() { make_match_star(star, None) } rule mapping_pattern() -> MatchPattern<'input, 'a> = l:lbrace() r:rbrace() { make_match_mapping(l, vec![], None, None, None, None, r) } / l:lbrace() rest:double_star_pattern() trail:comma()? r:rbrace() { make_match_mapping(l, vec![], None, Some(rest.0), Some(rest.1), trail, r) } / l:lbrace() items:items_pattern() c:comma() rest:double_star_pattern() trail:comma()? r:rbrace() { make_match_mapping(l, items, Some(c), Some(rest.0), Some(rest.1), trail, r) } / l:lbrace() items:items_pattern() trail:comma()? r:rbrace() { make_match_mapping(l, items, trail, None, None, None, r) } rule items_pattern() -> Vec> = pats:separated(, ) { comma_separate(pats.0, pats.1, None) } rule key_value_pattern() -> MatchMappingElement<'input, 'a> = key:(literal_expr() / a:attr() {a.into()}) colon:lit(":") pat:pattern() { make_match_mapping_element(key, colon, pat) } rule double_star_pattern() -> (TokenRef<'input, 'a>, Name<'input, 'a>) = star:lit("**") n:pattern_capture_target() { (star, n) } rule class_pattern() -> MatchPattern<'input, 'a> = cls:name_or_attr() l:lit("(") r:lit(")") { make_class_pattern(cls, l, vec![], None, vec![], None, r) } / cls:name_or_attr() l:lit("(") pats:positional_patterns() c:comma()? r:lit(")") { make_class_pattern(cls, l, pats, c, vec![], None, r) } / cls:name_or_attr() l:lit("(") kwds:keyword_patterns() c:comma()? r:lit(")") { make_class_pattern(cls, l, vec![], None, kwds, c, r) } / cls:name_or_attr() l:lit("(") pats:positional_patterns() c:comma() kwds:keyword_patterns() trail:comma()? r:lit(")") { make_class_pattern(cls, l, pats, Some(c), kwds, trail, r) } rule positional_patterns() -> Vec> = pats:separated(, ) { comma_separate(pats.0, pats.1, None) } rule keyword_patterns() -> Vec> = pats:separated(, ) { comma_separate(pats.0, pats.1, None) } rule keyword_pattern() -> MatchKeywordElement<'input, 'a> = arg:name() eq:lit("=") value:pattern() { make_match_keyword_element(arg, eq, value) } // Type statement rule type_stmt() -> TypeAlias<'input, 'a> = t:lit("type") n:name() ps:type_params()? eq:lit("=") v:expression() { make_type_alias(t, n, ps, eq, v) } // Type parameter declaration rule type_params() -> TypeParameters<'input, 'a> = lb:lbrak() ps:separated_trailer(, ) rb:rbrak() { make_type_parameters(lb, comma_separate(ps.0, ps.1, ps.2), rb) } rule type_param() -> TypeParam<'input, 'a> = n:name() b:type_param_bound()? { make_type_var(n, b) } / s:lit("*") n:name() { make_type_var_tuple(s, n) } / s:lit("**") n:name() { make_param_spec(s, n) } rule type_param_bound() -> TypeParamBound<'input, 'a> = c:lit(":") e:expression() { make_type_param_bound(c, e) } // Expressions #[cache] rule expression() -> Expression<'input, 'a> = _conditional_expression() / lambdef() rule _conditional_expression() -> Expression<'input, 'a> = body:disjunction() i:lit("if") test:disjunction() e:lit("else") oe:expression() { Expression::IfExp(Box::new(make_ifexp(body, i, test, e, oe))) } / disjunction() rule yield_expr() -> Expression<'input, 'a> = y:lit("yield") f:lit("from") a:expression() { Expression::Yield(Box::new(make_yield(y, Some(f), Some(a)))) } / y:lit("yield") a:star_expressions()? { Expression::Yield(Box::new(make_yield(y, None, a))) } rule star_expressions() -> Expression<'input, 'a> = first:star_expression() rest:(comma:comma() e:star_expression() { (comma, expr_to_element(e)) })+ comma:comma()? { Expression::Tuple(Box::new(make_tuple(expr_to_element(first), rest, comma, None, None))) } / e:star_expression() comma:comma() { Expression::Tuple(Box::new(make_tuple(expr_to_element(e), vec![], Some(comma), None, None))) } / star_expression() #[cache] rule star_expression() -> Expression<'input, 'a> = star:lit("*") e:bitwise_or() { Expression::StarredElement(Box::new(make_starred_element(star, expr_to_element(e)))) } / expression() rule star_named_expressions() -> Vec> = exps:separated_trailer(, ) { comma_separate(exps.0, exps.1, exps.2) } rule star_named_expression() -> Element<'input, 'a> = star:lit("*") e:bitwise_or() { Element::Starred(Box::new(make_starred_element(star, expr_to_element(e)))) } / e:named_expression() { expr_to_element(e) } rule named_expression() -> Expression<'input, 'a> = a:name() op:lit(":=") b:expression() { Expression::NamedExpr(Box::new(make_named_expr(a, op, b))) } / e:expression() !lit(":=") { e } #[cache] rule disjunction() -> Expression<'input, 'a> = a:conjunction() b:(or:lit("or") inner:conjunction() { (or, inner) })+ {? make_boolean_op(a, b).map_err(|e| "expected disjunction") } / conjunction() #[cache] rule conjunction() -> Expression<'input, 'a> = a:inversion() b:(and:lit("and") inner:inversion() { (and, inner) })+ {? make_boolean_op(a, b).map_err(|e| "expected conjunction") } / inversion() #[cache] rule inversion() -> Expression<'input, 'a> = not:lit("not") a:inversion() {? make_unary_op(not, a).map_err(|e| "expected inversion") } / comparison() // Comparison operators #[cache] rule comparison() -> Expression<'input, 'a> = a:bitwise_or() b:compare_op_bitwise_or_pair()+ { make_comparison(a, b) } / bitwise_or() // This implementation diverges slightly from CPython (3.9) to avoid bloating // the parser cache and increase readability. #[cache] rule compare_op_bitwise_or_pair() -> (CompOp<'input, 'a>, Expression<'input, 'a>) = _op_bitwise_or("==") / _op_bitwise_or("!=") // TODO: support barry_as_flufl / _op_bitwise_or("<=") / _op_bitwise_or("<") / _op_bitwise_or(">=") / _op_bitwise_or(">") / _op_bitwise_or2("not", "in") / _op_bitwise_or("in") / _op_bitwise_or2("is", "not") / _op_bitwise_or("is") rule _op_bitwise_or(o: &'static str) -> (CompOp<'input, 'a>, Expression<'input, 'a>) = op:lit(o) e:bitwise_or() {? make_comparison_operator(op) .map(|op| (op, e)) .map_err(|_| "comparison") } rule _op_bitwise_or2(first: &'static str, second: &'static str) -> (CompOp<'input, 'a>, Expression<'input, 'a>) = f:lit(first) s:lit(second) e:bitwise_or() {? make_comparison_operator_2(f, s) .map(|op| (op, e)) .map_err(|_| "comparison") } #[cache_left_rec] rule bitwise_or() -> Expression<'input, 'a> = a:bitwise_or() op:lit("|") b:bitwise_xor() {? make_binary_op(a, op, b).map_err(|e| "expected bitwise_or") } / bitwise_xor() #[cache_left_rec] rule bitwise_xor() -> Expression<'input, 'a> = a:bitwise_xor() op:lit("^") b:bitwise_and() {? make_binary_op(a, op, b).map_err(|e| "expected bitwise_xor") } / bitwise_and() #[cache_left_rec] rule bitwise_and() -> Expression<'input, 'a> = a:bitwise_and() op:lit("&") b:shift_expr() {? make_binary_op(a, op, b).map_err(|e| "expected bitwise_and") } / shift_expr() #[cache_left_rec] rule shift_expr() -> Expression<'input, 'a> = a:shift_expr() op:lit("<<") b:sum() {? make_binary_op(a, op, b).map_err(|e| "expected shift_expr") } / a:shift_expr() op:lit(">>") b:sum() {? make_binary_op(a, op, b).map_err(|e| "expected shift_expr") } / sum() #[cache_left_rec] rule sum() -> Expression<'input, 'a> = a:sum() op:lit("+") b:term() {? make_binary_op(a, op, b).map_err(|e| "expected sum") } / a:sum() op:lit("-") b:term() {? make_binary_op(a, op, b).map_err(|e| "expected sum") } / term() #[cache_left_rec] rule term() -> Expression<'input, 'a> = a:term() op:lit("*") b:factor() {? make_binary_op(a, op, b).map_err(|e| "expected term") } / a:term() op:lit("/") b:factor() {? make_binary_op(a, op, b).map_err(|e| "expected term") } / a:term() op:lit("//") b:factor() {? make_binary_op(a, op, b).map_err(|e| "expected term") } / a:term() op:lit("%") b:factor() {? make_binary_op(a, op, b).map_err(|e| "expected term") } / a:term() op:lit("@") b:factor() {? make_binary_op(a, op, b).map_err(|e| "expected term") } / factor() #[cache] rule factor() -> Expression<'input, 'a> = op:lit("+") a:factor() {? make_unary_op(op, a).map_err(|e| "expected factor") } / op:lit("-") a:factor() {? make_unary_op(op, a).map_err(|e| "expected factor") } / op:lit("~") a:factor() {? make_unary_op(op, a).map_err(|e| "expected factor") } / power() rule power() -> Expression<'input, 'a> = a:await_primary() op:lit("**") b:factor() {? make_binary_op(a, op, b).map_err(|e| "expected power") } / await_primary() // Primary elements rule await_primary() -> Expression<'input, 'a> = aw:tok(AWAIT, "AWAIT") e:primary() { Expression::Await(Box::new(make_await(aw, e))) } / primary() #[cache_left_rec] rule primary() -> Expression<'input, 'a> = v:primary() dot:lit(".") attr:name() { Expression::Attribute(Box::new(make_attribute(v, dot, attr))) } / a:primary() b:genexp() { Expression::Call(Box::new(make_genexp_call(a, b))) } / f:primary() lpar:lit("(") arg:arguments()? rpar:lit(")") { Expression::Call(Box::new(make_call(f, lpar, arg.unwrap_or_default(), rpar))) } / v:primary() lbrak:lbrak() s:slices() rbrak:rbrak() { Expression::Subscript(Box::new(make_subscript(v, lbrak, s, rbrak))) } / atom() rule slices() -> Vec> = s:slice() !lit(",") { vec![SubscriptElement { slice: s, comma: None }] } / slices:separated_trailer(, ) { make_slices(slices.0, slices.1, slices.2) } rule slice() -> BaseSlice<'input, 'a> = l:expression()? col:lit(":") u:expression()? rest:(c:lit(":") s:expression()? {(c, s)})? { make_slice(l, col, u, rest) } / e:starred_expression() { make_index_from_arg(e) } / v:named_expression() { make_index(v) } rule atom() -> Expression<'input, 'a> = n:name() { Expression::Name(Box::new(n)) } / n:lit("True") { Expression::Name(Box::new(make_name(n))) } / n:lit("False") { Expression::Name(Box::new(make_name(n))) } / n:lit("None") { Expression::Name(Box::new(make_name(n))) } / &(tok(STRING, "") / tok(FStringStart, "")) s:strings() {s.into()} / n:tok(Number, "NUMBER") { make_number(n) } / &lit("(") e:(tuple() / group() / (g:genexp() {Expression::GeneratorExp(Box::new(g))})) {e} / &lit("[") e:(list() / listcomp()) {e} / &lit("{") e:(dict() / set() / dictcomp() / setcomp()) {e} / lit("...") { Expression::Ellipsis(Box::new(Ellipsis {lpar: vec![], rpar: vec![]}))} rule group() -> Expression<'input, 'a> = lpar:lpar() e:(yield_expr() / named_expression()) rpar:rpar() { e.with_parens(lpar, rpar) } // Lambda functions rule lambdef() -> Expression<'input, 'a> = kw:lit("lambda") p:lambda_params()? c:lit(":") b:expression() { Expression::Lambda(Box::new(make_lambda(kw, p.unwrap_or_default(), c, b))) } rule lambda_params() -> Parameters<'input, 'a> = lambda_parameters() // lambda_parameters etc. duplicates parameters but without annotations or type // comments, and if there's no comma after a parameter, we expect a colon, not a // close parenthesis. rule lambda_parameters() -> Parameters<'input, 'a> = a:lambda_slash_no_default() b:lambda_param_no_default()* c:lambda_param_with_default()* d:lambda_star_etc()? { make_parameters(Some(a), concat(b, c), d) } / a:lambda_slash_with_default() b:lambda_param_with_default()* d:lambda_star_etc()? { make_parameters(Some(a), b, d) } / a:lambda_param_no_default()+ b:lambda_param_with_default()* d:lambda_star_etc()? { make_parameters(None, concat(a, b), d) } / a:lambda_param_with_default()+ d:lambda_star_etc()? { make_parameters(None, a, d) } / d:lambda_star_etc() { make_parameters(None, vec![], Some(d)) } rule lambda_slash_no_default() -> (Vec>, ParamSlash<'input, 'a>) = a:lambda_param_no_default()+ tok:lit("/") com:comma() { (a, ParamSlash { comma: Some(com), tok } ) } / a:lambda_param_no_default()+ tok:lit("/") &lit(":") { (a, ParamSlash { comma: None, tok }) } rule lambda_slash_with_default() -> (Vec>, ParamSlash<'input, 'a>) = a:lambda_param_no_default()* b:lambda_param_with_default()+ tok:lit("/") c:comma(){ (concat(a, b), ParamSlash { comma: Some(c), tok }) } / a:lambda_param_no_default()* b:lambda_param_with_default()+ tok:lit("/") &lit(":") { (concat(a, b), ParamSlash { comma: None, tok }) } rule lambda_star_etc() -> StarEtc<'input, 'a> = star:lit("*") a:lambda_param_no_default() b:lambda_param_maybe_default()* kw:lambda_kwds()? { StarEtc(Some(StarArg::Param( Box::new(add_param_star(a, star)) )), b, kw) } / lit("*") c:comma() b:lambda_param_maybe_default()+ kw:lambda_kwds()? { StarEtc(Some(StarArg::Star(Box::new(ParamStar {comma: c}))), b, kw) } / kw:lambda_kwds() { StarEtc(None, vec![], Some(kw)) } rule lambda_kwds() -> Param<'input, 'a> = star:lit("**") a:lambda_param_no_default() { add_param_star(a, star) } rule lambda_param_no_default() -> Param<'input, 'a> = a:lambda_param() c:lit(",") { add_param_default(a, None, Some(c)) } / a:lambda_param() &lit(":") {a} rule lambda_param_with_default() -> Param<'input, 'a> = a:lambda_param() def:default() c:lit(",") { add_param_default(a, Some(def), Some(c)) } / a:lambda_param() def:default() &lit(":") { add_param_default(a, Some(def), None) } rule lambda_param_maybe_default() -> Param<'input, 'a> = a:lambda_param() def:default()? c:lit(",") { add_param_default(a, def, Some(c)) } / a:lambda_param() def:default()? &lit(":") { add_param_default(a, def, None) } rule lambda_param() -> Param<'input, 'a> = name:name() { Param { name, ..Default::default() } } // Literals rule strings() -> String<'input, 'a> = s:(str:tok(STRING, "STRING") t:&_ {(make_string(str), t)} / str:fstring() t:&_ {(String::Formatted(str), t)})+ {? make_strings(s) } rule list() -> Expression<'input, 'a> = lbrak:lbrak() e:star_named_expressions()? rbrak:rbrak() { Expression::List(Box::new( make_list(lbrak, e.unwrap_or_default(), rbrak)) ) } rule tuple() -> Expression<'input, 'a> = lpar:lpar() first:star_named_expression() &lit(",") rest:(c:comma() e:star_named_expression() {(c, e)})* trailing_comma:comma()? rpar:rpar() { Expression::Tuple(Box::new( make_tuple(first, rest, trailing_comma, Some(lpar), Some(rpar)) )) } / lpar:lpar() rpar:lit(")") { Expression::Tuple(Box::new(Tuple::default().with_parens( lpar, RightParen { rpar_tok: rpar } )))} rule set() -> Expression<'input, 'a> = lbrace:lbrace() e:star_named_expressions()? rbrace:rbrace() { Expression::Set(Box::new(make_set(lbrace, e.unwrap_or_default(), rbrace))) } // Dicts rule dict() -> Expression<'input, 'a> = lbrace:lbrace() els:double_starred_keypairs()? rbrace:rbrace() { Expression::Dict(Box::new(make_dict(lbrace, els.unwrap_or_default(), rbrace))) } rule double_starred_keypairs() -> Vec> = pairs:separated_trailer(, ) { make_double_starred_keypairs(pairs.0, pairs.1, pairs.2) } rule double_starred_kvpair() -> DictElement<'input, 'a> = s:lit("**") e:bitwise_or() { DictElement::Starred(make_double_starred_element(s, e)) } / k:kvpair() { make_dict_element(k) } rule kvpair() -> (Expression<'input, 'a>, TokenRef<'input, 'a>, Expression<'input, 'a>) = k:expression() colon:lit(":") v:expression() { (k, colon, v) } // Comprehensions & generators rule for_if_clauses() -> CompFor<'input, 'a> = c:for_if_clause()+ {? merge_comp_fors(c) } rule for_if_clause() -> CompFor<'input, 'a> = asy:_async() f:lit("for") tgt:star_targets() i:lit("in") iter:disjunction() ifs:_comp_if()* { make_for_if(Some(asy), f, tgt, i, iter, ifs) } / f:lit("for") tgt:star_targets() i:lit("in") iter:disjunction() ifs:_comp_if()* { make_for_if(None, f, tgt, i, iter, ifs) } rule _comp_if() -> CompIf<'input, 'a> = kw:lit("if") cond:disjunction() { make_comp_if(kw, cond) } rule listcomp() -> Expression<'input, 'a> = lbrak:lbrak() elt:named_expression() comp:for_if_clauses() rbrak:rbrak() { Expression::ListComp(Box::new(make_list_comp(lbrak, elt, comp, rbrak))) } rule setcomp() -> Expression<'input, 'a> = l:lbrace() elt:named_expression() comp:for_if_clauses() r:rbrace() { Expression::SetComp(Box::new(make_set_comp(l, elt, comp, r))) } rule genexp() -> GeneratorExp<'input, 'a> = lpar:lpar() g:_bare_genexp() rpar:rpar() { g.with_parens(lpar, rpar) } rule _bare_genexp() -> GeneratorExp<'input, 'a> = elt:named_expression() comp:for_if_clauses() { make_bare_genexp(elt, comp) } rule dictcomp() -> Expression<'input, 'a> = lbrace:lbrace() elt:kvpair() comp:for_if_clauses() rbrace:rbrace() { Expression::DictComp(Box::new(make_dict_comp(lbrace, elt, comp, rbrace))) } // Function call arguments rule arguments() -> Vec> = a:args() trail:comma()? &lit(")") {add_arguments_trailing_comma(a, trail)} rule args() -> Vec> = first:_posarg() rest:(c:comma() a:_posarg() {(c, a)})* kw:(c:comma() k:kwargs() {(c, k)})? { let (trail, kw) = kw.map(|(x,y)| (Some(x), Some(y))).unwrap_or((None, None)); concat( comma_separate(first, rest, trail), kw.unwrap_or_default(), ) } / kwargs() rule _posarg() -> Arg<'input, 'a> = a:(starred_expression() / e:named_expression() { make_arg(e) }) !lit("=") { a } rule kwargs() -> Vec> = sitems:separated(, ) scomma:comma() ditems:separated(, ) { concat( comma_separate(sitems.0, sitems.1, Some(scomma)), comma_separate(ditems.0, ditems.1, None), ) } / items:separated(, ) { comma_separate(items.0, items.1, None) } / items:separated(, ) { comma_separate(items.0, items.1, None) } rule starred_expression() -> Arg<'input, 'a> = star:lit("*") e:expression() { make_star_arg(star, e) } rule kwarg_or_starred() -> Arg<'input, 'a> = _kwarg() / starred_expression() rule kwarg_or_double_starred() -> Arg<'input, 'a> = _kwarg() / star:lit("**") e:expression() { make_star_arg(star, e) } rule _kwarg() -> Arg<'input, 'a> = n:name() eq:lit("=") v:expression() { make_kwarg(n, eq, v) } // Assignment targets // Generic targets rule star_targets() -> AssignTargetExpression<'input, 'a> = a:star_target() !lit(",") {a} / targets:separated_trailer(, ) { AssignTargetExpression::Tuple(Box::new( make_tuple(targets.0, targets.1, targets.2, None, None) )) } rule star_targets_list_seq() -> Vec> = targets:separated_trailer(, ) { comma_separate(targets.0, targets.1, targets.2) } // This differs from star_targets below because it requires at least two items // in the tuple rule star_targets_tuple_seq() -> Tuple<'input, 'a> = first:(t:star_target() {assign_target_to_element(t)}) rest:(c:comma() t:star_target() {(c, assign_target_to_element(t))})+ trail:comma()? { make_tuple(first, rest, trail, None, None) } / t:star_target() trail:comma()? { make_tuple(assign_target_to_element(t), vec![], trail, None, None) } #[cache] rule star_target() -> AssignTargetExpression<'input, 'a> = star:lit("*") !lit("*") t:star_target() { AssignTargetExpression::StarredElement(Box::new( make_starred_element(star, assign_target_to_element(t)) )) } / target_with_star_atom() #[cache] rule target_with_star_atom() -> AssignTargetExpression<'input, 'a> = a:t_primary() dot:lit(".") n:name() !t_lookahead() { AssignTargetExpression::Attribute(Box::new(make_attribute(a, dot, n))) } / a:t_primary() lbrak:lbrak() s:slices() rbrak:rbrak() !t_lookahead() { AssignTargetExpression::Subscript(Box::new( make_subscript(a, lbrak, s, rbrak) )) } / a:star_atom() {a} rule star_atom() -> AssignTargetExpression<'input, 'a> = a:name() { AssignTargetExpression::Name(Box::new(a)) } / lpar:lpar() a:target_with_star_atom() rpar:rpar() { a.with_parens(lpar, rpar) } / lpar:lpar() a:star_targets_tuple_seq()? rpar:rpar() { AssignTargetExpression::Tuple(Box::new( a.unwrap_or_default().with_parens(lpar, rpar) )) } / lbrak:lbrak() a:star_targets_list_seq()? rbrak:rbrak() { AssignTargetExpression::List(Box::new( make_list(lbrak, a.unwrap_or_default(), rbrak) )) } rule single_target() -> AssignTargetExpression<'input, 'a> = single_subscript_attribute_target() / n:name() { AssignTargetExpression::Name(Box::new(n)) } / lpar:lpar() t:single_target() rpar:rpar() { t.with_parens(lpar, rpar) } rule single_subscript_attribute_target() -> AssignTargetExpression<'input, 'a> = a:t_primary() dot:lit(".") n:name() !t_lookahead() { AssignTargetExpression::Attribute(Box::new(make_attribute(a, dot, n))) } / a:t_primary() lbrak:lbrak() s:slices() rbrak:rbrak() !t_lookahead() { AssignTargetExpression::Subscript(Box::new( make_subscript(a, lbrak, s, rbrak) )) } #[cache_left_rec] rule t_primary() -> Expression<'input, 'a> = value:t_primary() dot:lit(".") attr:name() &t_lookahead() { Expression::Attribute(Box::new(make_attribute(value, dot, attr))) } / v:t_primary() l:lbrak() s:slices() r:rbrak() &t_lookahead() { Expression::Subscript(Box::new(make_subscript(v, l, s, r))) } / f:t_primary() gen:genexp() &t_lookahead() { Expression::Call(Box::new(make_genexp_call(f, gen))) } / f:t_primary() lpar:lit("(") arg:arguments()? rpar:lit(")") &t_lookahead() { Expression::Call(Box::new(make_call(f, lpar, arg.unwrap_or_default(), rpar))) } / a:atom() &t_lookahead() {a} rule t_lookahead() -> () = (lit("(") / lit("[") / lit(".")) {} // Targets for del statements rule del_targets() -> Vec> = t:separated_trailer(, ) { comma_separate(t.0, t.1, t.2) } rule del_target() -> DelTargetExpression<'input, 'a> = a:t_primary() d:lit(".") n:name() !t_lookahead() { DelTargetExpression::Attribute(Box::new(make_attribute(a, d, n))) } / a:t_primary() lbrak:lbrak() s:slices() rbrak:rbrak() !t_lookahead() { DelTargetExpression::Subscript(Box::new( make_subscript(a, lbrak, s, rbrak) )) } / del_t_atom() rule del_t_atom() -> DelTargetExpression<'input, 'a> = n:name() { DelTargetExpression::Name(Box::new(n)) } / l:lpar() d:del_target() r:rpar() { d.with_parens(l, r) } / l:lpar() d:del_targets()? r:rpar() { make_del_tuple(Some(l), d.unwrap_or_default(), Some(r)) } / l:lbrak() d:del_targets()? r:rbrak() { DelTargetExpression::List(Box::new( make_list(l, d.unwrap_or_default(), r) )) } // F-strings rule fstring() -> FormattedString<'input, 'a> = start:tok(FStringStart, "f\"") parts:(_f_string() / _f_replacement())* end:tok(FStringEnd, "\"") { make_fstring(start.string, parts, end.string) } rule _f_string() -> FormattedStringContent<'input, 'a> = t:tok(FStringString, "f-string contents") { FormattedStringContent::Text(make_fstringtext(t.string)) } rule _f_replacement() -> FormattedStringContent<'input, 'a> = lb:lit("{") e:_f_expr() eq:lit("=")? conv:(t:lit("!") c:_f_conversion() {(t,c)})? spec:(t:lit(":") s:_f_spec() {(t,s)})? rb:lit("}") { FormattedStringContent::Expression(Box::new( make_fstring_expression(lb, e, eq, conv, spec, rb) )) } rule _f_expr() -> Expression<'input, 'a> = (g:_bare_genexp() {Expression::GeneratorExp(Box::new(g))}) / star_expressions() / yield_expr() rule _f_conversion() -> &'a str = lit("r") {"r"} / lit("s") {"s"} / lit("a") {"a"} rule _f_spec() -> Vec> = (_f_string() / _f_replacement())* // CST helpers rule comma() -> Comma<'input, 'a> = c:lit(",") { make_comma(c) } rule dots() -> Vec> = ds:((dot:lit(".") { make_dot(dot) })+ / tok:lit("...") { vec![make_dot(tok), make_dot(tok), make_dot(tok)]} )+ { ds.into_iter().flatten().collect() } rule lpar() -> LeftParen<'input, 'a> = a:lit("(") { make_lpar(a) } rule rpar() -> RightParen<'input, 'a> = a:lit(")") { make_rpar(a) } rule lbrak() -> LeftSquareBracket<'input, 'a> = tok:lit("[") { make_left_bracket(tok) } rule rbrak() -> RightSquareBracket<'input, 'a> = tok:lit("]") { make_right_bracket(tok) } rule lbrace() -> LeftCurlyBrace<'input, 'a> = tok:lit("{") { make_left_brace(tok) } rule rbrace() -> RightCurlyBrace<'input, 'a> = tok:lit("}") { make_right_brace(tok) } /// matches any token, not just whitespace rule _() -> TokenRef<'input, 'a> = [t] { t } rule lit(lit: &'static str) -> TokenRef<'input, 'a> = [t] {? if t.string == lit { Ok(t) } else { Err(lit) } } rule tok(tok: TokType, err: &'static str) -> TokenRef<'input, 'a> = [t] {? if t.r#type == tok { Ok(t) } else { Err(err) } } rule name() -> Name<'input, 'a> = !( lit("False") / lit("None") / lit("True") / lit("and") / lit("as") / lit("assert") / lit("async") / lit("await") / lit("break") / lit("class") / lit("continue") / lit("def") / lit("del") / lit("elif") / lit("else") / lit("except") / lit("finally") / lit("for") / lit("from") / lit("global") / lit("if") / lit("import") / lit("in") / lit("is") / lit("lambda") / lit("nonlocal") / lit("not") / lit("or") / lit("pass") / lit("raise") / lit("return") / lit("try") / lit("while") / lit("with") / lit("yield") ) t:tok(NameTok, "NAME") {make_name(t)} rule _async() -> TokenRef<'input, 'a> = tok(Async, "ASYNC") rule separated_trailer(el: rule, sep: rule) -> (El, Vec<(Sep, El)>, Option) = e:el() rest:(s:sep() e:el() {(s, e)})* trailer:sep()? {(e, rest, trailer)} rule separated(el: rule, sep: rule) -> (El, Vec<(Sep, El)>) = e:el() rest:(s:sep() e:el() {(s, e)})* {(e, rest)} rule traced(e: rule) -> T = &(_* { #[cfg(feature = "trace")] { println!("[PEG_INPUT_START]"); println!("{}", input); println!("[PEG_TRACE_START]"); } }) e:e()? {? #[cfg(feature = "trace")] println!("[PEG_TRACE_STOP]"); e.ok_or("") } } } #[allow(clippy::too_many_arguments)] fn make_function_def<'input, 'a>( async_tok: Option>, def_tok: TokenRef<'input, 'a>, name: Name<'input, 'a>, type_parameters: Option>, open_paren_tok: TokenRef<'input, 'a>, params: Option>, close_paren_tok: TokenRef<'input, 'a>, returns: Option>, colon_tok: TokenRef<'input, 'a>, body: Suite<'input, 'a>, ) -> FunctionDef<'input, 'a> { let asynchronous = async_tok.as_ref().map(|_| make_async()); FunctionDef { name, type_parameters, params: params.unwrap_or_default(), body, decorators: Default::default(), returns, asynchronous, async_tok, def_tok, open_paren_tok, close_paren_tok, colon_tok, } } fn make_decorator<'input, 'a>( at_tok: TokenRef<'input, 'a>, name: Expression<'input, 'a>, newline_tok: TokenRef<'input, 'a>, ) -> Decorator<'input, 'a> { Decorator { decorator: name, newline_tok, at_tok, } } fn make_comparison<'input, 'a>( head: Expression<'input, 'a>, tail: Vec<(CompOp<'input, 'a>, Expression<'input, 'a>)>, ) -> Expression<'input, 'a> { let mut comparisons = vec![]; for (operator, e) in tail { comparisons.push(ComparisonTarget { operator, comparator: e, }); } Expression::Comparison(Box::new(Comparison { left: Box::new(head), comparisons, lpar: vec![], rpar: vec![], })) } fn make_comparison_operator<'input, 'a>( tok: TokenRef<'input, 'a>, ) -> Result<'a, CompOp<'input, 'a>> { match tok.string { "<" => Ok(CompOp::LessThan { tok }), ">" => Ok(CompOp::GreaterThan { tok }), "<=" => Ok(CompOp::LessThanEqual { tok }), ">=" => Ok(CompOp::GreaterThanEqual { tok }), "==" => Ok(CompOp::Equal { tok }), "!=" => Ok(CompOp::NotEqual { tok }), "in" => Ok(CompOp::In { tok }), "is" => Ok(CompOp::Is { tok }), _ => Err(ParserError::OperatorError), } } fn make_comparison_operator_2<'input, 'a>( first: TokenRef<'input, 'a>, second: TokenRef<'input, 'a>, ) -> Result<'a, CompOp<'input, 'a>> { match (first.string, second.string) { ("is", "not") => Ok(CompOp::IsNot { is_tok: first, not_tok: second, }), ("not", "in") => Ok(CompOp::NotIn { not_tok: first, in_tok: second, }), _ => Err(ParserError::OperatorError), } } fn make_boolean_op<'input, 'a>( head: Expression<'input, 'a>, tail: Vec<(TokenRef<'input, 'a>, Expression<'input, 'a>)>, ) -> Result<'a, Expression<'input, 'a>> { if tail.is_empty() { return Ok(head); } let mut expr = head; for (tok, right) in tail { expr = Expression::BooleanOperation(Box::new(BooleanOperation { left: Box::new(expr), operator: make_boolean_operator(tok)?, right: Box::new(right), lpar: vec![], rpar: vec![], })) } Ok(expr) } fn make_boolean_operator<'input, 'a>( tok: TokenRef<'input, 'a>, ) -> Result<'a, BooleanOp<'input, 'a>> { match tok.string { "and" => Ok(BooleanOp::And { tok }), "or" => Ok(BooleanOp::Or { tok }), _ => Err(ParserError::OperatorError), } } fn make_binary_op<'input, 'a>( left: Expression<'input, 'a>, op: TokenRef<'input, 'a>, right: Expression<'input, 'a>, ) -> Result<'a, Expression<'input, 'a>> { let operator = make_binary_operator(op)?; Ok(Expression::BinaryOperation(Box::new(BinaryOperation { left: Box::new(left), operator, right: Box::new(right), lpar: vec![], rpar: vec![], }))) } fn make_binary_operator<'input, 'a>(tok: TokenRef<'input, 'a>) -> Result<'a, BinaryOp<'input, 'a>> { match tok.string { "+" => Ok(BinaryOp::Add { tok }), "-" => Ok(BinaryOp::Subtract { tok }), "*" => Ok(BinaryOp::Multiply { tok }), "/" => Ok(BinaryOp::Divide { tok }), "//" => Ok(BinaryOp::FloorDivide { tok }), "%" => Ok(BinaryOp::Modulo { tok }), "**" => Ok(BinaryOp::Power { tok }), "<<" => Ok(BinaryOp::LeftShift { tok }), ">>" => Ok(BinaryOp::RightShift { tok }), "|" => Ok(BinaryOp::BitOr { tok }), "&" => Ok(BinaryOp::BitAnd { tok }), "^" => Ok(BinaryOp::BitXor { tok }), "@" => Ok(BinaryOp::MatrixMultiply { tok }), _ => Err(ParserError::OperatorError), } } fn make_unary_op<'input, 'a>( op: TokenRef<'input, 'a>, tail: Expression<'input, 'a>, ) -> Result<'a, Expression<'input, 'a>> { let operator = make_unary_operator(op)?; Ok(Expression::UnaryOperation(Box::new(UnaryOperation { operator, expression: Box::new(tail), lpar: vec![], rpar: vec![], }))) } fn make_unary_operator<'input, 'a>(tok: TokenRef<'input, 'a>) -> Result<'a, UnaryOp<'input, 'a>> { match tok.string { "+" => Ok(UnaryOp::Plus { tok }), "-" => Ok(UnaryOp::Minus { tok }), "~" => Ok(UnaryOp::BitInvert { tok }), "not" => Ok(UnaryOp::Not { tok }), _ => Err(ParserError::OperatorError), } } fn make_number<'input, 'a>(num: TokenRef<'input, 'a>) -> Expression<'input, 'a> { super::numbers::parse_number(num.string) } fn make_indented_block<'input, 'a>( nl: TokenRef<'input, 'a>, indent: TokenRef<'input, 'a>, statements: Vec>, dedent: TokenRef<'input, 'a>, ) -> Suite<'input, 'a> { Suite::IndentedBlock(IndentedBlock { body: statements, indent: Default::default(), newline_tok: nl, indent_tok: indent, dedent_tok: dedent, }) } struct SimpleStatementParts<'input, 'a> { first_tok: TokenRef<'input, 'a>, // The first token of the first statement. Used for its whitespace first_statement: SmallStatement<'input, 'a>, rest: Vec<(TokenRef<'input, 'a>, SmallStatement<'input, 'a>)>, // semicolon, statement pairs last_semi: Option>, nl: TokenRef<'input, 'a>, } fn make_semicolon<'input, 'a>(tok: TokenRef<'input, 'a>) -> Semicolon<'input, 'a> { Semicolon { tok } } fn _make_simple_statement<'input, 'a>( parts: SimpleStatementParts<'input, 'a>, ) -> ( TokenRef<'input, 'a>, Vec>, TokenRef<'input, 'a>, ) { let mut body = vec![]; let mut current = parts.first_statement; for (semi, next) in parts.rest { body.push(current.with_semicolon(Some(make_semicolon(semi)))); current = next; } if let Some(semi) = parts.last_semi { current = current.with_semicolon(Some(make_semicolon(semi))); } body.push(current); (parts.first_tok, body, parts.nl) } fn make_simple_statement_suite<'input, 'a>( parts: SimpleStatementParts<'input, 'a>, ) -> Suite<'input, 'a> { let (first_tok, body, newline_tok) = _make_simple_statement(parts); Suite::SimpleStatementSuite(SimpleStatementSuite { body, first_tok, newline_tok, }) } fn make_simple_statement_line<'input, 'a>( parts: SimpleStatementParts<'input, 'a>, ) -> SimpleStatementLine<'input, 'a> { let (first_tok, body, newline_tok) = _make_simple_statement(parts); SimpleStatementLine { body, first_tok, newline_tok, } } fn make_if<'input, 'a>( if_tok: TokenRef<'input, 'a>, cond: Expression<'input, 'a>, colon_tok: TokenRef<'input, 'a>, block: Suite<'input, 'a>, orelse: Option>, is_elif: bool, ) -> If<'input, 'a> { If { test: cond, body: block, orelse: orelse.map(Box::new), is_elif, if_tok, colon_tok, } } fn make_else<'input, 'a>( else_tok: TokenRef<'input, 'a>, colon_tok: TokenRef<'input, 'a>, block: Suite<'input, 'a>, ) -> Else<'input, 'a> { Else { body: block, else_tok, colon_tok, } } struct StarEtc<'input, 'a>( Option>, Vec>, Option>, ); fn make_parameters<'input, 'a>( posonly: Option<(Vec>, ParamSlash<'input, 'a>)>, params: Vec>, star_etc: Option>, ) -> Parameters<'input, 'a> { let (posonly_params, posonly_ind) = match posonly { Some((a, b)) => (a, Some(b)), None => (vec![], None), }; let (star_arg, kwonly_params, star_kwarg) = match star_etc { None => (None, vec![], None), Some(StarEtc(a, b, c)) => (a, b, c), }; Parameters { params, star_arg, kwonly_params, star_kwarg, posonly_params, posonly_ind, } } fn add_param_default<'input, 'a>( param: Param<'input, 'a>, def: Option<(AssignEqual<'input, 'a>, Expression<'input, 'a>)>, comma_tok: Option>, ) -> Param<'input, 'a> { let comma = comma_tok.map(make_comma); let (equal, default) = match def { Some((a, b)) => (Some(a), Some(b)), None => (None, None), }; Param { equal, default, comma, ..param } } fn add_param_star<'input, 'a>( param: Param<'input, 'a>, star: TokenRef<'input, 'a>, ) -> Param<'input, 'a> { let str = star.string; Param { star: Some(str), star_tok: Some(star), ..param } } fn make_assign_equal<'input, 'a>(tok: TokenRef<'input, 'a>) -> AssignEqual<'input, 'a> { AssignEqual { tok } } fn make_comma<'input, 'a>(tok: TokenRef<'input, 'a>) -> Comma<'input, 'a> { Comma { tok } } fn concat(a: Vec, b: Vec) -> Vec { a.into_iter().chain(b.into_iter()).collect() } fn make_name_or_attr<'input, 'a>( first_tok: Name<'input, 'a>, mut tail: Vec<(TokenRef<'input, 'a>, Name<'input, 'a>)>, ) -> NameOrAttribute<'input, 'a> { if let Some((dot, name)) = tail.pop() { let dot = make_dot(dot); return NameOrAttribute::A(Box::new(Attribute { attr: name, dot, lpar: Default::default(), rpar: Default::default(), value: Box::new(make_name_or_attr(first_tok, tail).into()), })); } else { NameOrAttribute::N(Box::new(first_tok)) } } fn make_name<'input, 'a>(tok: TokenRef<'input, 'a>) -> Name<'input, 'a> { Name { value: tok.string, ..Default::default() } } fn make_dot<'input, 'a>(tok: TokenRef<'input, 'a>) -> Dot<'input, 'a> { Dot { tok } } fn make_import_alias<'input, 'a>( name: NameOrAttribute<'input, 'a>, asname: Option<(TokenRef<'input, 'a>, Name<'input, 'a>)>, ) -> ImportAlias<'input, 'a> { ImportAlias { name, asname: asname.map(|(x, y)| make_as_name(x, AssignTargetExpression::Name(Box::new(y)))), comma: None, } } fn make_as_name<'input, 'a>( as_tok: TokenRef<'input, 'a>, name: AssignTargetExpression<'input, 'a>, ) -> AsName<'input, 'a> { AsName { name, as_tok } } type ParenthesizedImportNames<'input, 'a> = ( Option>, ImportNames<'input, 'a>, Option>, ); fn make_import_from<'input, 'a>( from_tok: TokenRef<'input, 'a>, dots: Vec>, module: Option>, import_tok: TokenRef<'input, 'a>, aliases: ParenthesizedImportNames<'input, 'a>, ) -> ImportFrom<'input, 'a> { let (lpar, names, rpar) = aliases; ImportFrom { module, names, relative: dots, lpar, rpar, semicolon: None, from_tok, import_tok, } } fn make_import<'input, 'a>( import_tok: TokenRef<'input, 'a>, names: Vec>, ) -> Import<'input, 'a> { Import { names, semicolon: None, import_tok, } } fn make_import_from_as_names<'input, 'a>( first: ImportAlias<'input, 'a>, tail: Vec<(Comma<'input, 'a>, ImportAlias<'input, 'a>)>, ) -> Vec> { let mut ret = vec![]; let mut cur = first; for (comma, alias) in tail { ret.push(cur.with_comma(comma)); cur = alias; } ret.push(cur); ret } fn make_lpar<'input, 'a>(tok: TokenRef<'input, 'a>) -> LeftParen<'input, 'a> { LeftParen { lpar_tok: tok } } fn make_rpar<'input, 'a>(tok: TokenRef<'input, 'a>) -> RightParen<'input, 'a> { RightParen { rpar_tok: tok } } fn make_module<'input, 'a>( body: Vec>, tok: TokenRef<'input, 'a>, encoding: &str, ) -> Module<'input, 'a> { Module { body, eof_tok: tok, default_indent: " ", default_newline: "\n", has_trailing_newline: false, encoding: encoding.to_string(), } } fn make_attribute<'input, 'a>( value: Expression<'input, 'a>, dot: TokenRef<'input, 'a>, attr: Name<'input, 'a>, ) -> Attribute<'input, 'a> { let dot = make_dot(dot); Attribute { attr, dot, lpar: Default::default(), rpar: Default::default(), value: Box::new(value), } } fn make_starred_element<'input, 'a>( star_tok: TokenRef<'input, 'a>, rest: Element<'input, 'a>, ) -> StarredElement<'input, 'a> { let value = match rest { Element::Simple { value, .. } => value, _ => panic!("Internal error while making starred element"), }; StarredElement { value: Box::new(value), lpar: Default::default(), rpar: Default::default(), comma: Default::default(), star_tok, } } fn assign_target_to_element<'input, 'a>( expr: AssignTargetExpression<'input, 'a>, ) -> Element<'input, 'a> { match expr { AssignTargetExpression::Attribute(a) => Element::Simple { value: Expression::Attribute(a), comma: Default::default(), }, AssignTargetExpression::Name(a) => Element::Simple { value: Expression::Name(a), comma: Default::default(), }, AssignTargetExpression::Tuple(a) => Element::Simple { value: Expression::Tuple(a), comma: Default::default(), }, AssignTargetExpression::StarredElement(s) => Element::Starred(s), AssignTargetExpression::List(l) => Element::Simple { value: Expression::List(l), comma: Default::default(), }, AssignTargetExpression::Subscript(s) => Element::Simple { value: Expression::Subscript(s), comma: Default::default(), }, } } fn make_assignment<'input, 'a>( lhs: Vec<(AssignTargetExpression<'input, 'a>, TokenRef<'input, 'a>)>, rhs: Expression<'input, 'a>, ) -> Assign<'input, 'a> { let mut targets = vec![]; for (target, equal_tok) in lhs { targets.push(AssignTarget { target, equal_tok }); } Assign { targets, value: rhs, semicolon: Default::default(), } } fn expr_to_element<'input, 'a>(expr: Expression<'input, 'a>) -> Element<'input, 'a> { match expr { Expression::StarredElement(inner_expr) => Element::Starred(inner_expr), _ => Element::Simple { value: expr, comma: Default::default(), }, } } fn make_tuple<'input, 'a>( first: Element<'input, 'a>, rest: Vec<(Comma<'input, 'a>, Element<'input, 'a>)>, trailing_comma: Option>, lpar: Option>, rpar: Option>, ) -> Tuple<'input, 'a> { let elements = comma_separate(first, rest, trailing_comma); let lpar = lpar.map(|l| vec![l]).unwrap_or_default(); let rpar = rpar.map(|r| vec![r]).unwrap_or_default(); Tuple { elements, lpar, rpar, } } fn make_tuple_from_elements<'input, 'a>( first: Element<'input, 'a>, mut rest: Vec>, ) -> Tuple<'input, 'a> { rest.insert(0, first); Tuple { elements: rest, lpar: Default::default(), rpar: Default::default(), } } fn make_kwarg<'input, 'a>( name: Name<'input, 'a>, eq: TokenRef<'input, 'a>, value: Expression<'input, 'a>, ) -> Arg<'input, 'a> { let equal = Some(make_assign_equal(eq)); let keyword = Some(name); Arg { value, keyword, equal, comma: None, star: "", star_tok: None, } } fn make_star_arg<'input, 'a>( star: TokenRef<'input, 'a>, expr: Expression<'input, 'a>, ) -> Arg<'input, 'a> { let str = star.string; Arg { value: expr, keyword: None, equal: None, comma: None, star: str, star_tok: Some(star), } } fn make_call<'input, 'a>( func: Expression<'input, 'a>, lpar_tok: TokenRef<'input, 'a>, args: Vec>, rpar_tok: TokenRef<'input, 'a>, ) -> Call<'input, 'a> { let lpar = vec![]; let rpar = vec![]; let func = Box::new(func); Call { func, args, lpar, rpar, lpar_tok, rpar_tok, } } fn make_genexp_call<'input, 'a>( func: Expression<'input, 'a>, mut genexp: GeneratorExp<'input, 'a>, ) -> Call<'input, 'a> { // func ( (genexp) ) // ^ // lpar_tok // lpar_tok is the same token that was used to parse genexp's first lpar. // Nothing owns the whitespace before lpar_tok, so the same token is passed in here // again, to be converted into whitespace_after_func. We then split off a pair of // parenthesis from genexp, since now Call will own them. let mut lpars = genexp.lpar.into_iter(); let lpar_tok = lpars.next().expect("genexp without lpar").lpar_tok; genexp.lpar = lpars.collect(); let rpar_tok = genexp.rpar.pop().expect("genexp without rpar").rpar_tok; Call { func: Box::new(func), args: vec![Arg { value: Expression::GeneratorExp(Box::new(genexp)), keyword: None, equal: None, comma: None, star: "", star_tok: None, }], lpar: vec![], rpar: vec![], lpar_tok, rpar_tok, } } fn make_arg<'input, 'a>(expr: Expression<'input, 'a>) -> Arg<'input, 'a> { Arg { value: expr, keyword: Default::default(), equal: Default::default(), comma: Default::default(), star: Default::default(), star_tok: None, } } fn make_comp_if<'input, 'a>( if_tok: TokenRef<'input, 'a>, test: Expression<'input, 'a>, ) -> CompIf<'input, 'a> { CompIf { test, if_tok } } fn make_for_if<'input, 'a>( async_tok: Option>, for_tok: TokenRef<'input, 'a>, target: AssignTargetExpression<'input, 'a>, in_tok: TokenRef<'input, 'a>, iter: Expression<'input, 'a>, ifs: Vec>, ) -> CompFor<'input, 'a> { let inner_for_in = None; let asynchronous = async_tok.as_ref().map(|_| make_async()); CompFor { target, iter, ifs, inner_for_in, asynchronous, async_tok, for_tok, in_tok, } } fn make_bare_genexp<'input, 'a>( elt: Expression<'input, 'a>, for_in: CompFor<'input, 'a>, ) -> GeneratorExp<'input, 'a> { GeneratorExp { elt: Box::new(elt), for_in: Box::new(for_in), lpar: Default::default(), rpar: Default::default(), } } fn merge_comp_fors<'input, 'a>( comp_fors: Vec>, ) -> GrammarResult> { if comp_fors.len() > MAX_RECURSION_DEPTH { return Err("shallower comprehension"); } let mut it = comp_fors.into_iter().rev(); let first = it.next().expect("cant merge empty comp_fors"); Ok(it.fold(first, |acc, curr| CompFor { inner_for_in: Some(Box::new(acc)), ..curr })) } fn make_left_bracket<'input, 'a>(tok: TokenRef<'input, 'a>) -> LeftSquareBracket<'input, 'a> { LeftSquareBracket { tok } } fn make_right_bracket<'input, 'a>(tok: TokenRef<'input, 'a>) -> RightSquareBracket<'input, 'a> { RightSquareBracket { tok } } fn make_left_brace<'input, 'a>(tok: TokenRef<'input, 'a>) -> LeftCurlyBrace<'input, 'a> { LeftCurlyBrace { tok } } fn make_right_brace<'input, 'a>(tok: TokenRef<'input, 'a>) -> RightCurlyBrace<'input, 'a> { RightCurlyBrace { tok } } fn make_list_comp<'input, 'a>( lbracket: LeftSquareBracket<'input, 'a>, elt: Expression<'input, 'a>, for_in: CompFor<'input, 'a>, rbracket: RightSquareBracket<'input, 'a>, ) -> ListComp<'input, 'a> { ListComp { elt: Box::new(elt), for_in: Box::new(for_in), lbracket, rbracket, lpar: Default::default(), rpar: Default::default(), } } fn make_set_comp<'input, 'a>( lbrace: LeftCurlyBrace<'input, 'a>, elt: Expression<'input, 'a>, for_in: CompFor<'input, 'a>, rbrace: RightCurlyBrace<'input, 'a>, ) -> SetComp<'input, 'a> { SetComp { elt: Box::new(elt), for_in: Box::new(for_in), lbrace, rbrace, lpar: Default::default(), rpar: Default::default(), } } fn make_dict_comp<'input, 'a>( lbrace: LeftCurlyBrace<'input, 'a>, kvpair: ( Expression<'input, 'a>, TokenRef<'input, 'a>, Expression<'input, 'a>, ), for_in: CompFor<'input, 'a>, rbrace: RightCurlyBrace<'input, 'a>, ) -> DictComp<'input, 'a> { let (key, colon_tok, value) = kvpair; DictComp { key: Box::new(key), value: Box::new(value), for_in: Box::new(for_in), lbrace, rbrace, lpar: vec![], rpar: vec![], colon_tok, } } fn make_list<'input, 'a>( lbracket: LeftSquareBracket<'input, 'a>, elements: Vec>, rbracket: RightSquareBracket<'input, 'a>, ) -> List<'input, 'a> { List { elements, lbracket, rbracket, lpar: Default::default(), rpar: Default::default(), } } fn make_set<'input, 'a>( lbrace: LeftCurlyBrace<'input, 'a>, elements: Vec>, rbrace: RightCurlyBrace<'input, 'a>, ) -> Set<'input, 'a> { Set { elements, lbrace, rbrace, lpar: Default::default(), rpar: Default::default(), } } fn comma_separate<'input, 'a, T>( first: T, rest: Vec<(Comma<'input, 'a>, T)>, trailing_comma: Option>, ) -> Vec where T: WithComma<'input, 'a>, { let mut elements = vec![]; let mut current = first; for (comma, next) in rest { elements.push(current.with_comma(comma)); current = next; } if let Some(comma) = trailing_comma { current = current.with_comma(comma); } elements.push(current); elements } fn make_dict<'input, 'a>( lbrace: LeftCurlyBrace<'input, 'a>, elements: Vec>, rbrace: RightCurlyBrace<'input, 'a>, ) -> Dict<'input, 'a> { Dict { elements, lbrace, rbrace, lpar: Default::default(), rpar: Default::default(), } } fn make_double_starred_keypairs<'input, 'a>( first: DictElement<'input, 'a>, rest: Vec<(Comma<'input, 'a>, DictElement<'input, 'a>)>, trailing_comma: Option>, ) -> Vec> { let mut elements = vec![]; let mut current = first; for (comma, next) in rest { elements.push(current.with_comma(comma)); current = next; } if let Some(comma) = trailing_comma { current = current.with_comma(comma); } elements.push(current); elements } fn make_dict_element<'input, 'a>( el: ( Expression<'input, 'a>, TokenRef<'input, 'a>, Expression<'input, 'a>, ), ) -> DictElement<'input, 'a> { let (key, colon_tok, value) = el; DictElement::Simple { key, value, comma: Default::default(), colon_tok, } } fn make_double_starred_element<'input, 'a>( star_tok: TokenRef<'input, 'a>, value: Expression<'input, 'a>, ) -> StarredDictElement<'input, 'a> { StarredDictElement { value, comma: Default::default(), star_tok, } } fn make_index<'input, 'a>(value: Expression<'input, 'a>) -> BaseSlice<'input, 'a> { BaseSlice::Index(Box::new(Index { value, star: None, star_tok: None, })) } fn make_index_from_arg<'input, 'a>(arg: Arg<'input, 'a>) -> BaseSlice<'input, 'a> { BaseSlice::Index(Box::new(Index { value: arg.value, star: Some(arg.star), star_tok: arg.star_tok, })) } fn make_colon<'input, 'a>(tok: TokenRef<'input, 'a>) -> Colon<'input, 'a> { Colon { tok } } fn make_slice<'input, 'a>( lower: Option>, first_colon: TokenRef<'input, 'a>, upper: Option>, rest: Option<(TokenRef<'input, 'a>, Option>)>, ) -> BaseSlice<'input, 'a> { let first_colon = make_colon(first_colon); let (second_colon, step) = if let Some((tok, step)) = rest { (Some(make_colon(tok)), step) } else { (None, None) }; BaseSlice::Slice(Box::new(Slice { lower, upper, step, first_colon, second_colon, })) } fn make_slices<'input, 'a>( first: BaseSlice<'input, 'a>, rest: Vec<(Comma<'input, 'a>, BaseSlice<'input, 'a>)>, trailing_comma: Option>, ) -> Vec> { let mut elements = vec![]; let mut current = first; for (comma, next) in rest { elements.push(SubscriptElement { slice: current, comma: Some(comma), }); current = next; } elements.push(SubscriptElement { slice: current, comma: trailing_comma, }); elements } fn make_subscript<'input, 'a>( value: Expression<'input, 'a>, lbracket: LeftSquareBracket<'input, 'a>, slice: Vec>, rbracket: RightSquareBracket<'input, 'a>, ) -> Subscript<'input, 'a> { Subscript { value: Box::new(value), slice, lbracket, rbracket, lpar: Default::default(), rpar: Default::default(), } } fn make_ifexp<'input, 'a>( body: Expression<'input, 'a>, if_tok: TokenRef<'input, 'a>, test: Expression<'input, 'a>, else_tok: TokenRef<'input, 'a>, orelse: Expression<'input, 'a>, ) -> IfExp<'input, 'a> { IfExp { test: Box::new(test), body: Box::new(body), orelse: Box::new(orelse), lpar: Default::default(), rpar: Default::default(), if_tok, else_tok, } } fn add_arguments_trailing_comma<'input, 'a>( mut args: Vec>, trailing_comma: Option>, ) -> Vec> { if let Some(comma) = trailing_comma { let last = args.pop().unwrap(); args.push(last.with_comma(comma)); } args } fn make_lambda<'input, 'a>( lambda_tok: TokenRef<'input, 'a>, params: Parameters<'input, 'a>, colon_tok: TokenRef<'input, 'a>, expr: Expression<'input, 'a>, ) -> Lambda<'input, 'a> { let colon = make_colon(colon_tok); Lambda { params: Box::new(params), body: Box::new(expr), colon, lpar: Default::default(), rpar: Default::default(), lambda_tok, } } fn make_annotation<'input, 'a>( tok: TokenRef<'input, 'a>, ann: Expression<'input, 'a>, ) -> Annotation<'input, 'a> { Annotation { annotation: ann, tok, } } fn make_ann_assignment<'input, 'a>( target: AssignTargetExpression<'input, 'a>, col: TokenRef<'input, 'a>, ann: Expression<'input, 'a>, rhs: Option<(TokenRef<'input, 'a>, Expression<'input, 'a>)>, ) -> AnnAssign<'input, 'a> { let annotation = make_annotation(col, ann); let (eq, value) = rhs.map(|(x, y)| (Some(x), Some(y))).unwrap_or((None, None)); let equal = eq.map(make_assign_equal); AnnAssign { target, annotation, value, equal, semicolon: None, } } fn make_yield<'input, 'a>( yield_tok: TokenRef<'input, 'a>, f: Option>, e: Option>, ) -> Yield<'input, 'a> { let value = match (f, e) { (None, None) => None, (Some(f), Some(e)) => Some(YieldValue::From(Box::new(make_from(f, e)))), (None, Some(e)) => Some(YieldValue::Expression(Box::new(e))), _ => panic!("yield from without expression"), }; Yield { value: value.map(Box::new), lpar: Default::default(), rpar: Default::default(), yield_tok, } } fn make_from<'input, 'a>(tok: TokenRef<'input, 'a>, e: Expression<'input, 'a>) -> From<'input, 'a> { From { item: e, tok } } fn make_return<'input, 'a>( return_tok: TokenRef<'input, 'a>, value: Option>, ) -> Return<'input, 'a> { Return { value, semicolon: Default::default(), return_tok, } } fn make_assert<'input, 'a>( assert_tok: TokenRef<'input, 'a>, test: Expression<'input, 'a>, rest: Option<(Comma<'input, 'a>, Expression<'input, 'a>)>, ) -> Assert<'input, 'a> { let (comma, msg) = if let Some((c, msg)) = rest { (Some(c), Some(msg)) } else { (None, None) }; Assert { test, msg, comma, semicolon: Default::default(), assert_tok, } } fn make_raise<'input, 'a>( raise_tok: TokenRef<'input, 'a>, exc: Option>, rest: Option<(TokenRef<'input, 'a>, Expression<'input, 'a>)>, ) -> Raise<'input, 'a> { let cause = rest.map(|(t, e)| make_from(t, e)); Raise { exc, cause, semicolon: Default::default(), raise_tok, } } fn make_global<'input, 'a>( tok: TokenRef<'input, 'a>, init: Vec<(Name<'input, 'a>, Comma<'input, 'a>)>, last: Name<'input, 'a>, ) -> Global<'input, 'a> { let mut names: Vec> = init .into_iter() .map(|(name, c)| NameItem { name, comma: Some(c), }) .collect(); names.push(NameItem { name: last, comma: None, }); Global { names, semicolon: Default::default(), tok, } } fn make_nonlocal<'input, 'a>( tok: TokenRef<'input, 'a>, init: Vec<(Name<'input, 'a>, Comma<'input, 'a>)>, last: Name<'input, 'a>, ) -> Nonlocal<'input, 'a> { let mut names: Vec> = init .into_iter() .map(|(name, c)| NameItem { name, comma: Some(c), }) .collect(); names.push(NameItem { name: last, comma: None, }); Nonlocal { names, semicolon: Default::default(), tok, } } #[allow(clippy::too_many_arguments)] fn make_for<'input, 'a>( async_tok: Option>, for_tok: TokenRef<'input, 'a>, target: AssignTargetExpression<'input, 'a>, in_tok: TokenRef<'input, 'a>, iter: Expression<'input, 'a>, colon_tok: TokenRef<'input, 'a>, body: Suite<'input, 'a>, orelse: Option>, ) -> For<'input, 'a> { let asynchronous = async_tok.as_ref().map(|_| make_async()); For { target, iter, body, orelse, asynchronous, async_tok, for_tok, in_tok, colon_tok, } } fn make_while<'input, 'a>( while_tok: TokenRef<'input, 'a>, test: Expression<'input, 'a>, colon_tok: TokenRef<'input, 'a>, body: Suite<'input, 'a>, orelse: Option>, ) -> While<'input, 'a> { While { test, body, orelse, while_tok, colon_tok, } } fn make_await<'input, 'a>( await_tok: TokenRef<'input, 'a>, expression: Expression<'input, 'a>, ) -> Await<'input, 'a> { Await { expression: Box::new(expression), lpar: Default::default(), rpar: Default::default(), await_tok, } } fn make_class_def<'input, 'a>( class_tok: TokenRef<'input, 'a>, name: Name<'input, 'a>, type_parameters: Option>, args: Option<( LeftParen<'input, 'a>, Option>>, RightParen<'input, 'a>, )>, colon_tok: TokenRef<'input, 'a>, body: Suite<'input, 'a>, ) -> std::result::Result, &'static str> { let mut bases = vec![]; let mut keywords = vec![]; let mut lpar_tok = None; let mut rpar_tok = None; let mut lpar = None; let mut rpar = None; if let Some((lpar_, args, rpar_)) = args { lpar_tok = Some(lpar_.lpar_tok); rpar_tok = Some(rpar_.rpar_tok); lpar = Some(lpar_); rpar = Some(rpar_); if let Some(args) = args { let mut current_arg = &mut bases; let mut seen_keyword = false; for arg in args { if arg.star == "**" || arg.keyword.is_some() { current_arg = &mut keywords; seen_keyword = true; } if seen_keyword && (arg.star == "*" || (arg.star.is_empty() && arg.keyword.is_none())) { return Err("Positional argument follows keyword argument"); } // TODO: libcst-python does validation here current_arg.push(arg); } } } Ok(ClassDef { name, type_parameters, body, bases, keywords, decorators: vec![], lpar, rpar, class_tok, lpar_tok, rpar_tok, colon_tok, }) } fn make_string<'input, 'a>(tok: TokenRef<'input, 'a>) -> String<'input, 'a> { String::Simple(SimpleString { value: tok.string, ..Default::default() }) } fn make_strings<'input, 'a>( s: Vec<(String<'input, 'a>, TokenRef<'input, 'a>)>, ) -> GrammarResult> { if s.len() > MAX_RECURSION_DEPTH { return Err("shorter concatenated string"); } let mut strings = s.into_iter().rev(); let (first, _) = strings.next().expect("no strings to make a string of"); Ok(strings.fold(first, |acc, (str, tok)| { let ret: String<'input, 'a> = String::Concatenated(ConcatenatedString { left: Box::new(str), right: Box::new(acc), lpar: Default::default(), rpar: Default::default(), right_tok: tok, }); ret })) } fn make_fstring_expression<'input, 'a>( lbrace_tok: TokenRef<'input, 'a>, expression: Expression<'input, 'a>, eq: Option>, conversion_pair: Option<(TokenRef<'input, 'a>, &'a str)>, format_pair: Option<( TokenRef<'input, 'a>, Vec>, )>, rbrace_tok: TokenRef<'input, 'a>, ) -> FormattedStringExpression<'input, 'a> { let equal = eq.map(make_assign_equal); let (conversion_tok, conversion) = if let Some((t, c)) = conversion_pair { (Some(t), Some(c)) } else { (None, None) }; let (format_tok, format_spec) = if let Some((t, f)) = format_pair { (Some(t), Some(f)) } else { (None, None) }; let after_expr_tok = if equal.is_some() { None } else if let Some(tok) = conversion_tok { Some(tok) } else if let Some(tok) = format_tok { Some(tok) } else { Some(rbrace_tok) }; FormattedStringExpression { expression, conversion, format_spec, equal, lbrace_tok, after_expr_tok, } } fn make_fstring<'input, 'a>( start: &'a str, parts: Vec>, end: &'a str, ) -> FormattedString<'input, 'a> { FormattedString { start, parts, end, lpar: Default::default(), rpar: Default::default(), } } fn make_finally<'input, 'a>( finally_tok: TokenRef<'input, 'a>, colon_tok: TokenRef<'input, 'a>, body: Suite<'input, 'a>, ) -> Finally<'input, 'a> { Finally { body, finally_tok, colon_tok, } } fn make_except<'input, 'a>( except_tok: TokenRef<'input, 'a>, exp: Option>, as_: Option<(TokenRef<'input, 'a>, Name<'input, 'a>)>, colon_tok: TokenRef<'input, 'a>, body: Suite<'input, 'a>, ) -> ExceptHandler<'input, 'a> { // TODO: AsName should come from outside let name = as_.map(|(x, y)| make_as_name(x, AssignTargetExpression::Name(Box::new(y)))); ExceptHandler { body, r#type: exp, name, except_tok, colon_tok, } } fn make_except_star<'input, 'a>( except_tok: TokenRef<'input, 'a>, star_tok: TokenRef<'input, 'a>, exp: Expression<'input, 'a>, as_: Option<(TokenRef<'input, 'a>, Name<'input, 'a>)>, colon_tok: TokenRef<'input, 'a>, body: Suite<'input, 'a>, ) -> ExceptStarHandler<'input, 'a> { // TODO: AsName should come from outside let name = as_.map(|(x, y)| make_as_name(x, AssignTargetExpression::Name(Box::new(y)))); ExceptStarHandler { body, r#type: exp, name, except_tok, colon_tok, star_tok, } } fn make_try<'input, 'a>( try_tok: TokenRef<'input, 'a>, body: Suite<'input, 'a>, handlers: Vec>, orelse: Option>, finalbody: Option>, ) -> Try<'input, 'a> { Try { body, handlers, orelse, finalbody, try_tok, } } fn make_try_star<'input, 'a>( try_tok: TokenRef<'input, 'a>, body: Suite<'input, 'a>, handlers: Vec>, orelse: Option>, finalbody: Option>, ) -> TryStar<'input, 'a> { TryStar { body, handlers, orelse, finalbody, try_tok, } } fn make_aug_op<'input, 'a>(tok: TokenRef<'input, 'a>) -> Result<'a, AugOp<'input, 'a>> { Ok(match tok.string { "+=" => AugOp::AddAssign { tok }, "-=" => AugOp::SubtractAssign { tok }, "*=" => AugOp::MultiplyAssign { tok }, "@=" => AugOp::MatrixMultiplyAssign { tok }, "/=" => AugOp::DivideAssign { tok }, "%=" => AugOp::ModuloAssign { tok }, "&=" => AugOp::BitAndAssign { tok }, "|=" => AugOp::BitOrAssign { tok }, "^=" => AugOp::BitXorAssign { tok }, "<<=" => AugOp::LeftShiftAssign { tok }, ">>=" => AugOp::RightShiftAssign { tok }, "**=" => AugOp::PowerAssign { tok }, "//=" => AugOp::FloorDivideAssign { tok }, _ => return Err(ParserError::OperatorError), }) } fn make_aug_assign<'input, 'a>( target: AssignTargetExpression<'input, 'a>, operator: AugOp<'input, 'a>, value: Expression<'input, 'a>, ) -> AugAssign<'input, 'a> { AugAssign { target, operator, value, semicolon: Default::default(), } } fn make_with_item<'input, 'a>( item: Expression<'input, 'a>, as_: Option>, n: Option>, ) -> WithItem<'input, 'a> { let asname = match (as_, n) { (Some(as_), Some(n)) => Some(make_as_name(as_, n)), (None, None) => None, _ => panic!("as and name should be present or missing together"), }; WithItem { item, asname, comma: Default::default(), } } fn make_with<'input, 'a>( async_tok: Option>, with_tok: TokenRef<'input, 'a>, lpar: Option>, items: Vec>, rpar: Option>, colon_tok: TokenRef<'input, 'a>, body: Suite<'input, 'a>, ) -> With<'input, 'a> { let asynchronous = async_tok.as_ref().map(|_| make_async()); With { items, body, asynchronous, lpar, rpar, async_tok, with_tok, colon_tok, } } fn make_del<'input, 'a>( tok: TokenRef<'input, 'a>, target: DelTargetExpression<'input, 'a>, ) -> Del<'input, 'a> { Del { target, semicolon: Default::default(), tok, } } fn make_del_tuple<'input, 'a>( lpar: Option>, elements: Vec>, rpar: Option>, ) -> DelTargetExpression<'input, 'a> { DelTargetExpression::Tuple(Box::new(Tuple { elements, lpar: lpar.map(|x| vec![x]).unwrap_or_default(), rpar: rpar.map(|x| vec![x]).unwrap_or_default(), })) } fn make_named_expr<'input, 'a>( name: Name<'input, 'a>, tok: TokenRef<'input, 'a>, expr: Expression<'input, 'a>, ) -> NamedExpr<'input, 'a> { NamedExpr { target: Box::new(Expression::Name(Box::new(name))), value: Box::new(expr), lpar: Default::default(), rpar: Default::default(), walrus_tok: tok, } } fn make_match<'input, 'a>( match_tok: TokenRef<'input, 'a>, subject: Expression<'input, 'a>, colon_tok: TokenRef<'input, 'a>, indent_tok: TokenRef<'input, 'a>, cases: Vec>, dedent_tok: TokenRef<'input, 'a>, ) -> Match<'input, 'a> { Match { subject, cases, indent: Default::default(), match_tok, colon_tok, indent_tok, dedent_tok, } } fn make_case<'input, 'a>( case_tok: TokenRef<'input, 'a>, pattern: MatchPattern<'input, 'a>, guard: Option<(TokenRef<'input, 'a>, Expression<'input, 'a>)>, colon_tok: TokenRef<'input, 'a>, body: Suite<'input, 'a>, ) -> MatchCase<'input, 'a> { let (if_tok, guard) = match guard { Some((if_tok, guard)) => (Some(if_tok), Some(guard)), None => (None, None), }; MatchCase { pattern, guard, body, case_tok, if_tok, colon_tok, } } fn make_match_value<'input, 'a>(value: Expression<'input, 'a>) -> MatchPattern<'input, 'a> { MatchPattern::Value(MatchValue { value }) } fn make_match_singleton<'input, 'a>(value: Name<'input, 'a>) -> MatchPattern<'input, 'a> { MatchPattern::Singleton(MatchSingleton { value }) } fn make_list_pattern<'input, 'a>( lbracket: Option>, patterns: Vec>, rbracket: Option>, ) -> MatchSequence<'input, 'a> { MatchSequence::MatchList(MatchList { patterns, lbracket, rbracket, lpar: Default::default(), rpar: Default::default(), }) } fn make_as_pattern<'input, 'a>( pattern: Option>, as_tok: Option>, name: Option>, ) -> MatchPattern<'input, 'a> { MatchPattern::As(Box::new(MatchAs { pattern, name, lpar: Default::default(), rpar: Default::default(), as_tok, })) } fn make_bit_or<'input, 'a>(tok: TokenRef<'input, 'a>) -> BitOr<'input, 'a> { BitOr { tok } } fn make_or_pattern<'input, 'a>( first: MatchPattern<'input, 'a>, rest: Vec<(TokenRef<'input, 'a>, MatchPattern<'input, 'a>)>, ) -> MatchPattern<'input, 'a> { if rest.is_empty() { return first; } let mut patterns = vec![]; let mut current = first; for (sep, next) in rest { let op = make_bit_or(sep); patterns.push(MatchOrElement { pattern: current, separator: Some(op), }); current = next; } patterns.push(MatchOrElement { pattern: current, separator: None, }); MatchPattern::Or(Box::new(MatchOr { patterns, lpar: Default::default(), rpar: Default::default(), })) } fn ensure_real_number<'input, 'a>( tok: TokenRef<'input, 'a>, ) -> GrammarResult> { match make_number(tok) { e @ (Expression::Integer(_) | Expression::Float(_)) => Ok(e), _ => Err("real number"), } } fn ensure_imaginary_number<'input, 'a>( tok: TokenRef<'input, 'a>, ) -> GrammarResult> { match make_number(tok) { e @ Expression::Imaginary(_) => Ok(e), _ => Err("imaginary number"), } } fn make_tuple_pattern<'input, 'a>( lpar: LeftParen<'input, 'a>, patterns: Vec>, rpar: RightParen<'input, 'a>, ) -> MatchSequence<'input, 'a> { MatchSequence::MatchTuple(MatchTuple { patterns, lpar: vec![lpar], rpar: vec![rpar], }) } fn make_open_sequence_pattern<'input, 'a>( first: StarrableMatchSequenceElement<'input, 'a>, comma: Comma<'input, 'a>, mut rest: Vec>, ) -> Vec> { rest.insert(0, first.with_comma(comma)); rest } fn make_match_sequence_element<'input, 'a>( value: MatchPattern<'input, 'a>, ) -> MatchSequenceElement<'input, 'a> { MatchSequenceElement { value, comma: Default::default(), } } fn make_match_star<'input, 'a>( star_tok: TokenRef<'input, 'a>, name: Option>, ) -> MatchStar<'input, 'a> { MatchStar { name, comma: Default::default(), star_tok, } } fn make_match_mapping<'input, 'a>( lbrace: LeftCurlyBrace<'input, 'a>, mut elements: Vec>, el_comma: Option>, star_tok: Option>, rest: Option>, trailing_comma: Option>, rbrace: RightCurlyBrace<'input, 'a>, ) -> MatchPattern<'input, 'a> { if let Some(c) = el_comma { if let Some(el) = elements.pop() { elements.push(el.with_comma(c)); } // TODO: else raise error } MatchPattern::Mapping(MatchMapping { elements, rest, trailing_comma, lbrace, rbrace, lpar: Default::default(), rpar: Default::default(), star_tok, }) } fn make_match_mapping_element<'input, 'a>( key: Expression<'input, 'a>, colon_tok: TokenRef<'input, 'a>, pattern: MatchPattern<'input, 'a>, ) -> MatchMappingElement<'input, 'a> { MatchMappingElement { key, pattern, comma: Default::default(), colon_tok, } } fn make_class_pattern<'input, 'a>( cls: NameOrAttribute<'input, 'a>, lpar_tok: TokenRef<'input, 'a>, mut patterns: Vec>, pat_comma: Option>, mut kwds: Vec>, kwd_comma: Option>, rpar_tok: TokenRef<'input, 'a>, ) -> MatchPattern<'input, 'a> { if let Some(c) = pat_comma { if let Some(el) = patterns.pop() { patterns.push(el.with_comma(c)); } // TODO: else raise error } if let Some(c) = kwd_comma { if let Some(el) = kwds.pop() { kwds.push(el.with_comma(c)); } // TODO: else raise error } MatchPattern::Class(MatchClass { cls, patterns, kwds, lpar: Default::default(), rpar: Default::default(), lpar_tok, rpar_tok, }) } fn make_match_keyword_element<'input, 'a>( key: Name<'input, 'a>, equal_tok: TokenRef<'input, 'a>, pattern: MatchPattern<'input, 'a>, ) -> MatchKeywordElement<'input, 'a> { MatchKeywordElement { key, pattern, comma: Default::default(), equal_tok, } } struct TypeParamBound<'input, 'a>(TokenRef<'input, 'a>, Expression<'input, 'a>); fn make_type_param_bound<'input, 'a>( colon_tok: TokenRef<'input, 'a>, e: Expression<'input, 'a>, ) -> TypeParamBound<'input, 'a> { TypeParamBound(colon_tok, e) } fn make_param_spec<'input, 'a>( star_tok: TokenRef<'input, 'a>, name: Name<'input, 'a>, ) -> TypeParam<'input, 'a> { TypeParam { param: TypeVarLike::ParamSpec(ParamSpec { name, star_tok }), comma: Default::default(), } } fn make_type_var_tuple<'input, 'a>( star_tok: TokenRef<'input, 'a>, name: Name<'input, 'a>, ) -> TypeParam<'input, 'a> { TypeParam { param: TypeVarLike::TypeVarTuple(TypeVarTuple { name, star_tok }), comma: Default::default(), } } fn make_type_var<'input, 'a>( name: Name<'input, 'a>, bound: Option>, ) -> TypeParam<'input, 'a> { let (bound, colon) = match bound { Some(TypeParamBound(c, e)) => (Some(Box::new(e)), Some(make_colon(c))), _ => (None, None), }; TypeParam { param: TypeVarLike::TypeVar(TypeVar { name, bound, colon }), comma: Default::default(), } } fn make_type_parameters<'input, 'a>( lbracket: LeftSquareBracket<'input, 'a>, params: Vec>, rbracket: RightSquareBracket<'input, 'a>, ) -> TypeParameters<'input, 'a> { TypeParameters { lbracket, params, rbracket, } } fn make_type_alias<'input, 'a>( type_tok: TokenRef<'input, 'a>, name: Name<'input, 'a>, type_parameters: Option>, equals_tok: TokenRef<'input, 'a>, value: Expression<'input, 'a>, ) -> TypeAlias<'input, 'a> { let lbracket_tok = if let Some(tp) = &type_parameters { Some(tp.lbracket.tok) } else { None }; TypeAlias { type_tok, name, type_parameters, equals_tok, value: Box::new(value), semicolon: Default::default(), lbracket_tok, } } libcst-0.1.0/src/parser/mod.rs000064400000000000000000000004760072674642500143510ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree mod errors; mod grammar; mod numbers; pub use errors::ParserError; pub(crate) use grammar::TokVec; pub use grammar::{python, Result}; libcst-0.1.0/src/parser/numbers.rs000064400000000000000000000043310072674642500152370ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree use regex::Regex; use crate::nodes::deflated::{Expression, Float, Imaginary, Integer}; static HEX: &str = r"0[xX](?:_?[0-9a-fA-F])+"; static BIN: &str = r"0[bB](?:_?[01])+"; static OCT: &str = r"0[oO](?:_?[0-7])+"; static DECIMAL: &str = r"(?:0(?:_?0)*|[1-9](?:_?[0-9])*)"; static EXPONENT: &str = r"[eE][-+]?[0-9](?:_?[0-9])*"; // Note: these don't exactly match the python implementation (exponent is not included) static POINT_FLOAT: &str = r"([0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?|\.[0-9](?:_?[0-9])*)"; static EXP_FLOAT: &str = r"[0-9](?:_?[0-9])*"; thread_local! { static INTEGER_RE: Regex = Regex::new(format!("^({}|{}|{}|{})$", HEX, BIN, OCT, DECIMAL).as_str()).expect("regex"); static FLOAT_RE: Regex = Regex::new( format!( "^({}({})?|{}{})$", POINT_FLOAT, EXPONENT, EXP_FLOAT, EXPONENT ) .as_str(), ) .expect("regex"); static IMAGINARY_RE: Regex = Regex::new( format!( r"^([0-9](?:_?[0-9])*[jJ]|({}({})?|{}{})[jJ])$", POINT_FLOAT, EXPONENT, EXP_FLOAT, EXPONENT ) .as_str(), ) .expect("regex"); } pub(crate) fn parse_number(raw: &str) -> Expression { if INTEGER_RE.with(|r| r.is_match(raw)) { Expression::Integer(Box::new(Integer { value: raw, lpar: Default::default(), rpar: Default::default(), })) } else if FLOAT_RE.with(|r| r.is_match(raw)) { Expression::Float(Box::new(Float { value: raw, lpar: Default::default(), rpar: Default::default(), })) } else if IMAGINARY_RE.with(|r| r.is_match(raw)) { Expression::Imaginary(Box::new(Imaginary { value: raw, lpar: Default::default(), rpar: Default::default(), })) } else { Expression::Integer(Box::new(Integer { value: raw, lpar: Default::default(), rpar: Default::default(), })) } } libcst-0.1.0/src/py.rs000064400000000000000000000017230072674642500127220ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree use crate::nodes::traits::py::TryIntoPy; use pyo3::prelude::*; #[pymodule] #[pyo3(name = "native")] pub fn libcst_native(_py: Python, m: &PyModule) -> PyResult<()> { #[pyfn(m)] fn parse_module(source: String, encoding: Option<&str>) -> PyResult { let m = crate::parse_module(source.as_str(), encoding)?; Python::with_gil(|py| m.try_into_py(py)) } #[pyfn(m)] fn parse_expression(source: String) -> PyResult { let expr = crate::parse_expression(source.as_str())?; Python::with_gil(|py| expr.try_into_py(py)) } #[pyfn(m)] fn parse_statement(source: String) -> PyResult { let stm = crate::parse_statement(source.as_str())?; Python::with_gil(|py| stm.try_into_py(py)) } Ok(()) } libcst-0.1.0/src/tokenizer/core/LICENSE000064400000000000000000000045430072674642500156760ustar 00000000000000PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated documentation. 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Python Software Foundation; All Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee. 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to Python. 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions. 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade name in a trademark sense to endorse or promote products or services of Licensee, or any third party. 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this License Agreement. libcst-0.1.0/src/tokenizer/core/README.md000064400000000000000000000001640072674642500161430ustar 00000000000000Files in this directory are a derivative of CPython's tokenizer, and are therefore available under the PSF license. libcst-0.1.0/src/tokenizer/core/mod.rs000064400000000000000000001403540072674642500160170ustar 00000000000000// This implementation is Copyright (c) Meta Platforms, Inc. and affiliates. // // CPython 3.10.0a5 and the original C code this is based on is // Copyright (c) 2001-2021 Python Software Foundation; All Rights Reserved // // Portions of this module (f-string splitting) are based on parso's tokenize.py, which is also PSF // licensed. /// A port of CPython's tokenizer.c to Rust, with the following significant modifications: /// /// - PEP 263 (encoding detection) support isn't implemented. We depend on other code to do this for /// us right now, and expect that the input is utf-8 by the time we see it. /// /// - Removed support for tokenizing from a file handle without reading the whole file in at once. /// This significantly complicates parsing and memory is cheap, so we require that the whole file /// is read in and converted to a unicode string before tokenization can begin. /// /// - Removed support for the interactive interpreter parsing mode. /// /// - Tweaked the `translate_newlines` functionality and moved most of it into TextPosition. `\r` /// characters are no longer removed from the input buffer, so strings may contain `\r` characters /// that should be normalized prior to being interpreted. /// /// - Added support for tracking more detailed position information via TextPosition. As a /// consequence, consuming and then backing up a character (`tok_nextc`/`tok_backup`) is more /// expensive, and we prefer to call `TextPosition::peek()` instead. /// /// - Removed support for tokenizing type comments. /// /// - Reduced the number of different supported token types to match what parso's tokenizer yields. /// /// - Uses some regular expressions. Regular expression are a good fit for a tokenizer, but we don't /// use regular expressions everywhere because we can't generate as good of error messages with /// them. /// /// - Added support for breaking apart f-strings into multiple tokens, matching Parso's tokenizer /// behavior. CPython instead runs the parser recursively to parse f-strings. /// /// Also, in general, the code is less tightly optimized. The CPython implementation is crazy /// optimized in ways that wouldn't translate well to rust (e.g. it parses the input utf-8 buffer as /// raw bytes instead of unicode codepoints). /// /// The implementation should still be faster than any pure-Python implementation, and most /// optimizations (avoiding string copies when slicing) carry over to Rust very well. /// /// Planned (not yet implemented) features: /// /// - Add more feature flags to more closely match the behavior of older versions of Python 3.x. /// /// - Support for a Python 2 mode that tokenizes Python 2.7 code and fails on certain new Python 3 /// syntax that wasn't supported in 2.7. /// /// - Maybe add back support for tokenizing type comments? /// /// This implementation is tailored to LibCST's needs. If you're looking for a more general-purpose /// pure-Rust Python parser, consider using [RustPython's parser][]. /// /// [RustPython's parser]: https://crates.io/crates/rustpython-parser mod string_types; use regex::Regex; use std::cell::RefCell; use std::cmp::Ordering; use std::convert::TryInto; use std::fmt::Debug; use std::fmt::Formatter; use std::rc::Rc; use crate::tokenizer::{ core::string_types::{FStringNode, StringQuoteChar, StringQuoteSize}, operators::OPERATOR_RE, text_position::{TextPosition, TextPositionSnapshot}, whitespace_parser::State as WhitespaceState, }; /// The maximum number of indentation levels at any given point in time. CPython's tokenizer.c caps /// this to avoid the complexity of allocating a dynamic array, but we're using a Vec, so it's not /// necessary, but we're keeping it to maintain compatibility. const MAX_INDENT: usize = 100; // MAX_CHAR should be std::char::MAX once assoc_char_consts is stablized. // https://github.com/rust-lang/rust/issues/71763 const MAX_CHAR: char = '\u{10ffff}'; thread_local! { static SPACE_TAB_FORMFEED_RE: Regex = Regex::new(r"\A[ \f\t]+").expect("regex"); static ANY_NON_NEWLINE_RE: Regex = Regex::new(r"\A[^\r\n]+").expect("regex"); static STRING_PREFIX_RE: Regex = Regex::new(r"\A(?i)(u|[bf]r|r[bf]|r|b|f)").expect("regex"); static POTENTIAL_IDENTIFIER_TAIL_RE: Regex = Regex::new(r"\A([a-zA-Z0-9_]|[^\x00-\x7f])+").expect("regex"); static DECIMAL_DOT_DIGIT_RE: Regex = Regex::new(r"\A\.[0-9]").expect("regex"); static DECIMAL_TAIL_RE: Regex = Regex::new(r"\A[0-9](_?[0-9])*").expect("regex"); static HEXADECIMAL_TAIL_RE: Regex = Regex::new(r"\A(_?[0-9a-fA-F])+").expect("regex"); static OCTAL_TAIL_RE: Regex = Regex::new(r"\A(_?[0-7])+").expect("regex"); static BINARY_TAIL_RE: Regex = Regex::new(r"\A(_?[01])+").expect("regex"); /// Used to verify identifiers when there's a non-ascii character in them. // This changes across unicode revisions. We'd need to ship our own unicode tables to 100% match a // given Python version's behavior. static UNICODE_IDENTIFIER_RE: Regex = Regex::new(r"\A[\p{XID_Start}_]\p{XID_Continue}*\z").expect("regex"); } #[derive(Debug, Eq, PartialEq, Copy, Clone)] pub enum TokType { String, Name, Number, Op, Newline, Indent, Dedent, Async, Await, FStringStart, FStringString, FStringEnd, EndMarker, } #[derive(Debug, thiserror::Error, Eq, PartialEq)] pub enum TokError<'t> { #[error("inconsistent mixing of tabs and spaces")] TabSpace, #[error("too many indentation levels")] TooDeep, #[error("no matching outer block for dedent")] Dedent, #[error("unexpected characters after a line continuation")] LineContinuation, #[error("unexpected end of file after a line continuation")] LineContinuationEof, #[error("{0:?} is not a valid identifier")] BadIdentifier(&'t str), #[error("invalid decimal literal")] BadDecimal, #[error( "{}{}", "leading zeros in decimal integer literals are not permitted; use an 0o prefix for octal ", "integers" )] BadDecimalLeadingZeros, #[error("invalid hexadecimal literal")] BadHexadecimal, #[error("invalid octal literal")] BadOctal, #[error("invalid digit {0:?} in octal literal")] BadOctalDigit(char), #[error("invalid binary literal")] BadBinary, #[error("invalid digit {0:?} in binary literal")] BadBinaryDigit(char), #[error("unterminated string literal")] UnterminatedString, #[error("unterminated triple-quoted string literal")] UnterminatedTripleQuotedString, #[error("unmatched {0:?}")] UnmatchedClosingParen(char), #[error("Closing parenthesis {1:?} does not match opening parenthesis {0:?}")] MismatchedClosingParen(char, char), #[error("Closing parenthesis {1:?} does not match opening parenthesis {0:?} on line {2:}")] MismatchedClosingParenOnLine(char, char, usize), #[error("{0:?} is not a valid character in this position")] BadCharacter(char), } // Clone is used for async_hacks, which needs to speculatively look-ahead one token. #[derive(Clone)] pub struct TokState<'t> { /// The full program's source code (similar to `tok->str` or `tok->buf` in the CPython source /// code). We don't support reading the file line-by-line from a file handle like CPython does, /// so this is the whole program pre-converted to utf-8. pub text_pos: TextPosition<'t>, /// Start of the most recently returned token. pub start_pos: TextPositionSnapshot, /// True after we've encountered an error or there's no more text to process. done: bool, /// How many spaces a tab counts as (always 8) tab_size: usize, /// How many spaces a tab counts as in alt_indent_stack (always 1) alt_tab_size: usize, /// Stack of indentation levels where a tab is counted as 8 characters, used for tracking /// dedents. Length is current indentation level. Should never have more than MAX_INDENT /// entries. indent_stack: Vec, /// Used to check that tabs and spaces are not mixed. alt_indent_stack: Vec, /// Beginning of line. True if at the beginning of a new line. at_bol: bool, /// The number of bytes at the beginning of the line, as measured by consume_bol_whitespace. /// Used by libcst to capture (and then validate and parse) the indentation. pub bol_width: usize, /// Set by `consume_bol_whitespace`, true if the current line is blank. blank_line: bool, /// Pending intents (if > 0) or dedents (if < 0). Used when multiple tokens need to be produced /// at once. pending_indents: i32, /// Length is `() [] {}` parenthesis nesting level. Used to allow free continuations inside /// them. Stack entries are to verify that closing parenthesis match opening parenthesis. /// Tuple is (character, lineno). paren_stack: Vec<(char, usize)>, /// Whether we're in a continuation line. cont_line: bool, /// True if async/await aren't always keywords. async_hacks: bool, /// True if tokens are inside an 'async def' body. async_def: bool, /// Indentation level of the outermost 'async def'. async_def_indent: usize, /// True if the outermost 'async def' had at least one NEWLINE token after it. async_def_nl: bool, /// Splits f-strings into multiple tokens instead of a STRING token if true. /// /// CPython doesn't directly split f-strings in the tokenizer (and therefore doesn't support /// this option). Instead, when the parser encounters an f-string, it recursively re-runs the /// tokenizer and parser. /// /// Supporting this at the tokenizer-level is pretty nasty and adds a lot of complexity. /// Eventually, we should probably support this at the parser-level instead. split_fstring: bool, fstring_stack: Vec, missing_nl_before_eof: bool, } pub struct TokConfig { /// Used in Python 3.5 and 3.6. If enabled, async/await are sometimes keywords and sometimes /// identifiers, depending on if they're being used in the context of an async function. This /// breaks async comprehensions outside of async functions. pub async_hacks: bool, pub split_fstring: bool, // Not currently supported: // type_comments: bool, } fn is_digit>>(ch: C) -> bool { matches!(ch.into(), Some('0'..='9')) } #[derive(Debug)] enum NumberState { StartDigit, Fraction, Exponent, Imaginary, } impl<'t> TokState<'t> { pub fn new(text: &'t str, config: &TokConfig) -> Self { let text_pos = TextPosition::new(text); let start_pos = (&text_pos).into(); Self { text_pos, start_pos, done: false, tab_size: 8, alt_tab_size: 1, indent_stack: Vec::new(), alt_indent_stack: Vec::new(), at_bol: true, bol_width: 0, blank_line: false, pending_indents: 0, paren_stack: Vec::new(), cont_line: false, async_hacks: config.async_hacks, async_def: false, async_def_indent: 0, async_def_nl: false, split_fstring: config.split_fstring, fstring_stack: Vec::new(), missing_nl_before_eof: text.is_empty() || text.as_bytes()[text.len() - 1] != b'\n', } } pub fn is_parenthesized(&self) -> bool { !self.paren_stack.is_empty() } /// Implementation of `next()`, wrapped by next() to allow for easier error handling. Roughly /// equivalent to `tok_get` in the C source code. fn next_inner(&mut self) -> Result> { if self.split_fstring { if let Some(tos) = self.fstring_stack.last() { if !tos.is_in_expr() { self.start_pos = (&self.text_pos).into(); let is_in_format_spec = tos.is_in_format_spec(); let is_raw_string = tos.is_raw_string; if let Some(tok) = self.maybe_consume_fstring_string(is_in_format_spec, is_raw_string)? { return Ok(tok); } if let Some(tok) = self.maybe_consume_fstring_end() { return Ok(tok); } } } } // This will never consume a token, but it may set blank_line and it may set // pending_indents. self.consume_bol_whitespace()?; // Return pending indents/dedents if let Some(t) = self.process_pending_indents() { self.start_pos = (&self.text_pos).into(); return Ok(t); } self.maybe_close_async_def(); 'again: loop { // Skip spaces SPACE_TAB_FORMFEED_RE.with(|v| self.text_pos.consume(v)); // Skip comment, unless it's a type comment if self.text_pos.peek() == Some('#') { ANY_NON_NEWLINE_RE.with(|v| self.text_pos.consume(v)); // type_comment is not supported } // Set start of current token self.start_pos = (&self.text_pos).into(); return match self.text_pos.peek() { // Check for EOF now None => { if self.missing_nl_before_eof && !self.blank_line { self.at_bol = true; self.missing_nl_before_eof = false; Ok(TokType::Newline) } else { let hanging_indents = self.indent_stack.len() as i32; if self.pending_indents == 0 && hanging_indents != 0 { // We've reached EOF but there are still pending indents not // accounted for. Flush them out. self.pending_indents = -hanging_indents; self.indent_stack.clear(); self.alt_indent_stack.clear(); self.missing_nl_before_eof = false; } if let Some(t) = self.process_pending_indents() { Ok(t) } else { Ok(TokType::EndMarker) } } } // Identifier (most frequent token!) Some('a'..='z') | Some('A'..='Z') | Some('_') | Some('\u{80}'..=MAX_CHAR) => { self.consume_identifier_or_prefixed_string() } // Newline Some('\n') => { self.text_pos.next(); self.at_bol = true; if self.split_fstring && !self.fstring_stack.iter().all(|node| node.allow_multiline()) { Err(TokError::UnterminatedString) } else if self.blank_line || !self.paren_stack.is_empty() { // this newline doesn't count // recurse (basically `goto nextline`) self.next_inner() } else { self.cont_line = false; if self.async_def { self.async_def_nl = true; } Ok(TokType::Newline) } } // Ellipsis Some('.') if self.text_pos.consume("...") => { return Ok(TokType::Op); } // Number starting with period Some('.') if DECIMAL_DOT_DIGIT_RE.with(|r| self.text_pos.matches(r)) => { self.consume_number(NumberState::Fraction) } // Dot Some('.') => { self.text_pos.next(); Ok(TokType::Op) } // Number Some('0'..='9') => self.consume_number(NumberState::StartDigit), // String Some('\'') | Some('"') => self.consume_string(), // Line continuation Some('\\') => { self.text_pos.next(); if let Some('\n') = self.text_pos.next() { if self.text_pos.peek() == None { Err(TokError::LineContinuationEof) } else { self.cont_line = true; // Read next line continue 'again; } } else { Err(TokError::LineContinuation) } } Some(ch @ '(') | Some(ch @ '[') | Some(ch @ '{') => { self.text_pos.next(); if let Some(tos) = self.fstring_stack.last_mut() { tos.open_parentheses(); } self.paren_stack.push((ch, self.text_pos.line_number())); Ok(TokType::Op) } Some(closing @ ')') | Some(closing @ ']') | Some(closing @ '}') => { self.text_pos.next(); if let Some(tos) = self.fstring_stack.last_mut() { tos.close_parentheses(); } if let Some((opening, line_number)) = self.paren_stack.pop() { match (opening, closing) { ('(', ')') | ('[', ']') | ('{', '}') => Ok(TokType::Op), _ => { if line_number != self.text_pos.line_number() { Err(TokError::MismatchedClosingParenOnLine( opening, closing, line_number, )) } else { Err(TokError::MismatchedClosingParen(opening, closing)) } } } } else { Err(TokError::UnmatchedClosingParen(closing)) } } Some(':') if self .fstring_stack .last() .map(|tos| tos.parentheses_count - tos.format_spec_count == 1) .unwrap_or(false) => { // N.B. This may capture the walrus operator and pass it to the formatter. // That's intentional. PEP 572 says: "Assignment expressions inside of f-strings // require parentheses." // // >>> f'{x:=10}' # Valid, passes '=10' to formatter let tos = self .fstring_stack .last_mut() .expect("fstring_stack is not empty"); tos.format_spec_count += 1; self.text_pos.next(); Ok(TokType::Op) } // Operator Some(_) if OPERATOR_RE.with(|r| self.text_pos.consume(r)) => Ok(TokType::Op), // Bad character // If nothing works, fall back to this error. CPython returns an OP in this case, // and then just relies on the parser to generate a generic syntax error. Some(ch) => Err(TokError::BadCharacter(ch)), }; } } /// Consumes the whitespace (and comments) at the beginning of the line. May emit an error. Will /// mutate `pending_indents`, so you must check `pending_indents` after calling this. fn consume_bol_whitespace(&mut self) -> Result<(), TokError<'t>> { self.blank_line = false; if !self.at_bol { return Ok(()); } let mut col = 0; // column where tab counts as 8 characters let mut altcol = 0; // column where tab counts as 1 character self.at_bol = false; self.bol_width = 0; // consume space, tab, and formfeed characters loop { match self.text_pos.peek() { Some(' ') => { col += 1; altcol += 1; self.bol_width += 1; self.text_pos.next(); } Some('\t') => { // Increment both col and altcol using different tab sizes. Tabs snap to the // next multiple of self.tab_size. col = (col / self.tab_size + 1) * self.tab_size; // altcol will later be used for detecting mixed tabs and spaces. altcol = (altcol / self.alt_tab_size + 1) * self.alt_tab_size; self.bol_width += 1; self.text_pos.next(); } // Control-L (formfeed) for emacs users Some('\x0c') => { col = 0; altcol = 0; self.bol_width += 1; self.text_pos.next(); } _ => { break; } } } // Lines with only whitespace and/or comments and/or a line continuation // character shouldn't affect the indentation and are not passed to the parser // as NEWLINE tokens. self.blank_line = matches!( self.text_pos.peek(), Some('#') | Some('\n') | Some('\\') | None ); if self.blank_line || !self.paren_stack.is_empty() { return Ok(()); } let prev_col = self.indent_stack.last().unwrap_or(&0); match col.cmp(prev_col) { Ordering::Equal => { // No change if altcol != *self.alt_indent_stack.last().unwrap_or(&0) { return Err(TokError::TabSpace); } } Ordering::Greater => { // col > prev_col // Indent -- always one if self.indent_stack.len() + 1 >= MAX_INDENT { return Err(TokError::TooDeep); } // col > prev_col, therefore altcol > prev_altcol, unless there's badly mixed tabs // and spaces if altcol <= *self.alt_indent_stack.last().unwrap_or(&0) { return Err(TokError::TabSpace); } // only emit indents if we're not at EOF if self.text_pos.peek().is_some() { self.pending_indents += 1; self.indent_stack.push(col); self.alt_indent_stack.push(altcol); } } Ordering::Less => { // c < prev_col // Dedent -- any number, must be consistent while matches!(self.indent_stack.last(), Some(&ind_cols) if col < ind_cols) { self.pending_indents -= 1; self.indent_stack.pop(); self.alt_indent_stack.pop(); } if col != *self.indent_stack.last().unwrap_or(&0) { return Err(TokError::Dedent); } if altcol != *self.alt_indent_stack.last().unwrap_or(&0) { return Err(TokError::TabSpace); } } } Ok(()) } fn process_pending_indents(&mut self) -> Option { if self.pending_indents != 0 { if self.pending_indents < 0 { self.pending_indents += 1; Some(TokType::Dedent) } else { self.pending_indents -= 1; Some(TokType::Indent) } } else { None } } fn maybe_close_async_def(&mut self) { // Check if we are closing an async function if self.async_def && !self.blank_line // (This is irrelevant to the rust implementation which doesn't support type_comments // yet, but the comment is preserved for posterity) // Due to some implementation artifacts of type comments, a TYPE_COMMENT at the start of // a function won't set an indentation level and it will produce a NEWLINE after it. To // avoid spuriously ending an async function due to this, wait until we have some // non-newline char in front of us. // && self.text_pos.peek() == Some('\n') && self.paren_stack.is_empty() // There was a NEWLINE after ASYNC DEF, so we're past the signature. && self.async_def_nl // Current indentation level is less than where the async function was defined && self.async_def_indent >= self.indent_stack.len() { self.async_def = false; self.async_def_indent = 0; self.async_def_nl = false; } } fn consume_identifier_or_prefixed_string(&mut self) -> Result> { // Process the various legal combinations of b"", r"", u"", and f"". if STRING_PREFIX_RE.with(|r| self.text_pos.consume(r)) { if let Some('"') | Some('\'') = self.text_pos.peek() { // We found a string, not an identifier. Bail! if self.split_fstring && self .text_pos .slice_from_start_pos(&self.start_pos) .contains(&['f', 'F'][..]) { return self.consume_fstring_start(); } else { return self.consume_string(); } } } else { // the next character must be a potential identifier start, aka `[a-zA-Z_]|[^\x00-\x7f]` let first_ch = self.text_pos.next(); debug_assert!(matches!( first_ch, Some('a'..='z') | Some('A'..='Z') | Some('_') | Some('\u{80}'..=MAX_CHAR) )); } POTENTIAL_IDENTIFIER_TAIL_RE.with(|r| self.text_pos.consume(r)); let identifier_str = self.text_pos.slice_from_start_pos(&self.start_pos); if !verify_identifier(identifier_str) { // TODO: async/await return Err(TokError::BadIdentifier(identifier_str)); } let allow_async = !self.async_hacks || self.async_def; match (identifier_str, allow_async) { ("async", true) => Ok(TokType::Async), ("await", true) => Ok(TokType::Await), ("async", false) => { // The current token is 'async' and async_hacks is enabled. // Look ahead one token to see if that is 'def'. // This clone is expensive, but modern code doesn't need async_hacks. let mut lookahead_state = self.clone(); if lookahead_state.next_inner() == Ok(TokType::Name) && lookahead_state .text_pos .slice_from_start_pos(&lookahead_state.start_pos) == "def" { self.async_def = true; self.async_def_indent = self.indent_stack.len(); Ok(TokType::Async) } else { Ok(TokType::Name) } } _ => Ok(TokType::Name), } } fn consume_number(&mut self, state: NumberState) -> Result> { // This is organized as a state machine. The match could also be rewritten into multiple // functions, but this is closer to how the C code is written (with gotos). match state { NumberState::StartDigit => { let start_digit_ch = self.text_pos.peek(); debug_assert!(is_digit(start_digit_ch)); if start_digit_ch == Some('0') { self.text_pos.next(); match self.text_pos.peek() { Some('x') | Some('X') => { self.text_pos.next(); if !HEXADECIMAL_TAIL_RE.with(|r| self.text_pos.consume(r)) || self.text_pos.peek() == Some('_') { Err(TokError::BadHexadecimal) } else { Ok(TokType::Number) } } Some('o') | Some('O') => { self.text_pos.next(); if !OCTAL_TAIL_RE.with(|r| self.text_pos.consume(r)) || self.text_pos.peek() == Some('_') { return Err(TokError::BadOctal); } if let Some(next_ch) = self.text_pos.peek() { if is_digit(next_ch) { return Err(TokError::BadOctalDigit(next_ch)); } } Ok(TokType::Number) } Some('b') | Some('B') => { self.text_pos.next(); if !BINARY_TAIL_RE.with(|r| self.text_pos.consume(r)) || self.text_pos.peek() == Some('_') { return Err(TokError::BadBinary); } if let Some(next_ch) = self.text_pos.peek() { if is_digit(next_ch) { return Err(TokError::BadBinaryDigit(next_ch)); } } Ok(TokType::Number) } _ => { let mut nonzero = false; // Maybe old-style octal. In any case, allow '0' as a literal loop { if self.text_pos.peek() == Some('_') { self.text_pos.next(); if !is_digit(self.text_pos.peek()) { return Err(TokError::BadDecimal); } } if self.text_pos.peek() != Some('0') { break; } self.text_pos.next(); } if is_digit(self.text_pos.peek()) { nonzero = true; self.consume_decimal_tail()?; } if self.text_pos.peek() == Some('.') { self.consume_number(NumberState::Fraction) } else if let Some('e') | Some('E') = self.text_pos.peek() { self.consume_number(NumberState::Exponent) } else if let Some('j') | Some('J') = self.text_pos.peek() { self.consume_number(NumberState::Imaginary) } else if nonzero { Err(TokError::BadDecimalLeadingZeros) } else { Ok(TokType::Number) } } } } else { self.consume_decimal_tail()?; if self.text_pos.peek() == Some('.') { self.consume_number(NumberState::Fraction) } else if let Some('e') | Some('E') = self.text_pos.peek() { self.consume_number(NumberState::Exponent) } else if let Some('j') | Some('J') = self.text_pos.peek() { self.consume_number(NumberState::Imaginary) } else { Ok(TokType::Number) } } } NumberState::Fraction => { let dot_ch = self.text_pos.next(); debug_assert!(dot_ch == Some('.')); if is_digit(self.text_pos.peek()) { self.consume_decimal_tail()?; } if let Some('e') | Some('E') = self.text_pos.peek() { self.consume_number(NumberState::Exponent) } else if let Some('j') | Some('J') = self.text_pos.peek() { self.consume_number(NumberState::Imaginary) } else { Ok(TokType::Number) } } NumberState::Exponent => { let e_ch = self.text_pos.next(); debug_assert!(matches!(e_ch, Some('e') | Some('E'))); if let Some('+') | Some('-') = self.text_pos.peek() { self.text_pos.next(); if !is_digit(self.text_pos.peek()) { return Err(TokError::BadDecimal); } } else if !is_digit(self.text_pos.peek()) { // Don't consume the 'e'. It could be part of an identifier after this number. self.text_pos.backup_no_newline(); return Ok(TokType::Number); } self.consume_decimal_tail()?; if let Some('j') | Some('J') = self.text_pos.peek() { self.consume_number(NumberState::Imaginary) } else { Ok(TokType::Number) } } NumberState::Imaginary => { let j_ch = self.text_pos.next(); debug_assert!(matches!(j_ch, Some('j') | Some('J'))); Ok(TokType::Number) } } } /// Processes a decimal tail. This is the bit after the dot or after an E in a float. fn consume_decimal_tail(&mut self) -> Result<(), TokError<'t>> { let result = DECIMAL_TAIL_RE.with(|r| self.text_pos.consume(r)); // Assumption: If we've been called, the first character is an integer, so we must have a // regex match debug_assert!(result, "try_decimal_tail was called on a non-digit char"); if self.text_pos.peek() == Some('_') { Err(TokError::BadDecimal) } else { Ok(()) } } fn consume_open_quote(&mut self) -> (StringQuoteChar, StringQuoteSize) { let quote_char: StringQuoteChar = self .text_pos .peek() .try_into() .expect("the next character must be a quote when calling consume_open_quote"); let triple_quote_pattern = quote_char.triple_str(); let quote_size = if self.text_pos.consume(triple_quote_pattern) { StringQuoteSize::Triple } else { self.text_pos.next(); // consume the single character instead StringQuoteSize::Single }; (quote_char, quote_size) } fn consume_string(&mut self) -> Result> { // Assumption: The opening quote has not been consumed. Leading characters (b, r, f, etc) // have been consumed. let (quote_char, quote_size) = self.consume_open_quote(); let quote_raw = quote_char.into(); let mut end_quote_size: usize = 0; let quote_usize: usize = quote_size.into(); while end_quote_size != quote_usize { match (self.text_pos.next(), quote_size) { (None, StringQuoteSize::Triple) => { return Err(TokError::UnterminatedTripleQuotedString); } (None, StringQuoteSize::Single) | (Some('\n'), StringQuoteSize::Single) => { return Err(TokError::UnterminatedString); } (ch @ Some('\''), _) | (ch @ Some('"'), _) if ch == Some(quote_raw) => { end_quote_size += 1; } (Some(ch), _) => { end_quote_size = 0; if ch == '\\' { // skip escaped char self.text_pos.next(); } } } } Ok(TokType::String) } fn consume_fstring_start(&mut self) -> Result> { let (quote_char, quote_size) = self.consume_open_quote(); let is_raw_string = self .text_pos .slice_from_start_pos(&self.start_pos) .contains(&['r', 'R'][..]); self.fstring_stack .push(FStringNode::new(quote_char, quote_size, is_raw_string)); Ok(TokType::FStringStart) } fn maybe_consume_fstring_string( &mut self, is_in_format_spec: bool, is_raw_string: bool, ) -> Result, TokError<'t>> { let allow_multiline = self.fstring_stack.iter().all(|node| node.allow_multiline()); let mut in_named_unicode: bool = false; let mut ok_result = Ok(None); // value to return if we reach the end and don't error out 'outer: loop { match (self.text_pos.peek(), allow_multiline) { (None, true) => { return Err(TokError::UnterminatedTripleQuotedString); } (None, false) | (Some('\n'), false) => { return Err(TokError::UnterminatedString); } (ch @ Some('\''), _) | (ch @ Some('"'), _) => { // see if this actually terminates something in fstring_stack for node in self.fstring_stack.iter() { if ch == Some(node.quote_char.into()) { match node.quote_size { StringQuoteSize::Single => { break 'outer; } StringQuoteSize::Triple => { if self.text_pos.matches(node.quote_char.triple_str()) { break 'outer; } } } } } self.text_pos.next(); } (Some('\\'), _) if !is_raw_string => { self.text_pos.next(); if is_in_format_spec { if let Some('{') | Some('}') = self.text_pos.peek() { // don't consume { or } because we want those to be interpreted as OP // tokens } else { // skip escaped char (e.g. \', \", or newline/line continuation) self.text_pos.next(); } } else if let Some( '\n' | '\\' | '\'' | '"' | 'a' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' | 'x' | '0'..='9' | 'N' | 'u' | 'U', ) = self.text_pos.peek() { // skip escaped char let next_ch = self.text_pos.next(); // check if this is a \N sequence if let Some('N') = next_ch { // swallow the next open curly brace if it exists if let Some('{') = self.text_pos.peek() { in_named_unicode = true; self.text_pos.next(); } } } } (Some('\\'), _) if is_raw_string => { self.text_pos.next(); // skip escaped end-of-string marker or backslash if let Some('"' | '\'' | '\\') = self.text_pos.peek() { self.text_pos.next(); } } (Some('{'), _) => { if is_in_format_spec { // don't actually consume the {, and generate an OP for it instead break 'outer; } let consumed_double = self.text_pos.consume("{{"); if !consumed_double { break 'outer; } } (Some('}'), _) => { if in_named_unicode { in_named_unicode = false; self.text_pos.next(); } else if is_in_format_spec { // don't actually consume the }, and generate an OP for it instead break 'outer; } else if !self.text_pos.consume("}}") { return Err(TokError::UnmatchedClosingParen('}')); } } _ => { self.text_pos.next(); } } ok_result = Ok(Some(TokType::FStringString)); } ok_result } fn maybe_consume_fstring_end(&mut self) -> Option { let ch = self.text_pos.peek(); let mut match_idx = None; for (idx, node) in self.fstring_stack.iter().enumerate() { if ch == Some(node.quote_char.into()) { if node.quote_size == StringQuoteSize::Triple { if self.text_pos.consume(node.quote_char.triple_str()) { match_idx = Some(idx); break; } } else { self.text_pos.next(); // already matched match_idx = Some(idx); break; } } } if let Some(match_idx) = match_idx { self.fstring_stack.truncate(match_idx); Some(TokType::FStringEnd) } else { None } } } impl<'t> Iterator for TokState<'t> { type Item = Result>; /// Returns the next token type. fn next(&mut self) -> Option>> { // This implementation wraps `next_inner`, which does the actual work. if self.done { None } else { match self.next_inner() { Err(err) => { self.done = true; Some(Err(err)) } Ok(TokType::EndMarker) => { self.done = true; Some(Ok(TokType::EndMarker)) } Ok(t) => Some(Ok(t)), } } } } /// Returns true if the given string is a valid Python 3.x identifier. Follows [PEP 3131][]. /// /// [PEP 3131]: https://www.python.org/dev/peps/pep-3131/ fn verify_identifier(name: &str) -> bool { // TODO: If `name` is non-ascii, must first normalize name to NFKC. // Common case: If the entire string is ascii, we can avoid the more expensive regex check, // since the tokenizer already validates ascii characters before calling us. name.is_ascii() || UNICODE_IDENTIFIER_RE.with(|r| r.is_match(name)) } #[derive(Clone)] pub struct Token<'a> { pub r#type: TokType, pub string: &'a str, pub start_pos: TextPositionSnapshot, pub end_pos: TextPositionSnapshot, pub whitespace_before: Rc>>, pub whitespace_after: Rc>>, pub relative_indent: Option<&'a str>, } impl<'a> Debug for Token<'a> { fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), std::fmt::Error> { write!( f, "Token({:?}, {}, start={:?}, end={:?}, relative_indent={:?}, ws_before={:?}, ws_after={:?}", self.r#type, self.string, self.start_pos, self.end_pos, self.relative_indent, self.whitespace_before, self.whitespace_after ) } } // Dummy Eq implementation. We never compare Tokens like this impl<'a> PartialEq for Token<'a> { fn eq(&self, _other: &Self) -> bool { true } } impl<'a> Eq for Token<'a> {} pub struct TokenIterator<'a> { previous_whitespace: Option>>>, core_state: TokState<'a>, absolute_indents: Vec<&'a str>, } impl<'a> TokenIterator<'a> { pub fn new(module_text: &'a str, config: &TokConfig) -> Self { Self { previous_whitespace: None, absolute_indents: vec![], core_state: TokState::new(module_text, config), } } } impl<'a> Iterator for TokenIterator<'a> { type Item = Result, TokError<'a>>; fn next(&mut self) -> Option { let next = self.core_state.next(); next.as_ref()?; Some((|| { let tok_type = next.unwrap()?; let relative_indent = match tok_type { TokType::Indent => { let end_idx = self.core_state.text_pos.byte_idx(); let start_idx = end_idx - self.core_state.bol_width; let absolute_indent = &self.core_state.text_pos.text()[start_idx..end_idx]; let relative_indent = if let Some(prev_absolute_indent) = self.absolute_indents.last() { if let Some(ri) = absolute_indent.strip_prefix(prev_absolute_indent) { ri } else { // TODO: return the correct exception type, improve error message return Err(TokError::Dedent); } } else { // there's no previous indent, absolute_indent is relative_indent absolute_indent }; self.absolute_indents.push(absolute_indent); // HACKY: mutate and fixup the previous whitespace state if let Some(ws) = self.previous_whitespace.as_mut() { ws.borrow_mut().absolute_indent = absolute_indent; } Some(relative_indent) } TokType::Dedent => { self.absolute_indents.pop(); // HACKY: mutate and fixup the previous whitespace state if let Some(ws) = self.previous_whitespace.as_mut() { ws.borrow_mut().absolute_indent = self.absolute_indents.last().unwrap_or(&""); } None } _ => None, }; let text_pos = &self.core_state.text_pos; let whitespace_before = self.previous_whitespace.clone().unwrap_or_default(); let whitespace_after = match tok_type { TokType::Indent | TokType::Dedent | TokType::EndMarker => whitespace_before.clone(), _ => Rc::new(RefCell::new(WhitespaceState { line: text_pos.line_number(), column: text_pos.char_column_number(), column_byte: text_pos.byte_column_number(), byte_offset: text_pos.byte_idx(), absolute_indent: self.absolute_indents.last().unwrap_or(&""), is_parenthesized: self.core_state.is_parenthesized(), })), }; self.previous_whitespace = Some(whitespace_after.clone()); Ok(Token { r#type: tok_type, string: text_pos.slice_from_start_pos(&self.core_state.start_pos), start_pos: self.core_state.start_pos.clone(), end_pos: text_pos.into(), whitespace_after: whitespace_after.clone(), whitespace_before: whitespace_before.clone(), relative_indent, }) })()) } } libcst-0.1.0/src/tokenizer/core/string_types.rs000064400000000000000000000060360072674642500177700ustar 00000000000000// This implementation is Copyright (c) Meta Platforms, Inc. and affiliates. // // CPython 3.10.0a5 and the original C code this is based on is // Copyright (c) 2001-2021 Python Software Foundation; All Rights Reserved // // Portions of this module (f-string splitting) are based on parso's tokenize.py, which is also PSF // licensed. /// Helper types for string processing in the core tokenizer. use std::convert::TryFrom; use crate::tokenizer::text_position::TextPositionSnapshot; #[derive(Clone, Copy, Eq, PartialEq)] pub enum StringQuoteSize { Single, Triple, } impl From for usize { fn from(qs: StringQuoteSize) -> Self { match qs { StringQuoteSize::Single => 1, StringQuoteSize::Triple => 3, } } } #[derive(Clone, Copy)] pub enum StringQuoteChar { Apostrophe, DoubleQuote, } impl StringQuoteChar { pub fn triple_str(&self) -> &'static str { match self { Self::Apostrophe => "'''", Self::DoubleQuote => "\"\"\"", } } } impl From for char { fn from(ch: StringQuoteChar) -> Self { match ch { StringQuoteChar::Apostrophe => '\'', StringQuoteChar::DoubleQuote => '"', } } } #[derive(Debug, thiserror::Error)] #[error("{0:?} is not a valid string quote character")] pub struct StringQuoteCharConversionError(Option); impl TryFrom> for StringQuoteChar { type Error = StringQuoteCharConversionError; fn try_from(ch: Option) -> Result { match ch { Some('\'') => Ok(StringQuoteChar::Apostrophe), Some('"') => Ok(StringQuoteChar::DoubleQuote), _ => Err(StringQuoteCharConversionError(ch)), } } } #[derive(Clone)] pub struct FStringNode { pub quote_char: StringQuoteChar, pub quote_size: StringQuoteSize, pub parentheses_count: usize, pub string_start: Option, // In the syntax there can be multiple format_spec's nested: {x:{y:3}} pub format_spec_count: usize, pub is_raw_string: bool, } impl FStringNode { pub fn new( quote_char: StringQuoteChar, quote_size: StringQuoteSize, is_raw_string: bool, ) -> Self { Self { quote_char, quote_size, parentheses_count: 0, string_start: None, format_spec_count: 0, is_raw_string, } } pub fn open_parentheses(&mut self) { self.parentheses_count += 1; } pub fn close_parentheses(&mut self) { if self.is_in_format_spec() { self.format_spec_count -= 1; } self.parentheses_count -= 1; } pub fn allow_multiline(&self) -> bool { self.quote_size == StringQuoteSize::Triple } pub fn is_in_expr(&self) -> bool { self.parentheses_count > self.format_spec_count } pub fn is_in_format_spec(&self) -> bool { !self.is_in_expr() && self.format_spec_count > 0 } } libcst-0.1.0/src/tokenizer/debug_utils.rs000064400000000000000000000007660072674642500166200ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. use std::fmt; /// An empty struct that when writes "..." when using `fmt::Debug`. Useful for omitting fields when /// using `fmt::Formatter::debug_struct`. pub struct EllipsisDebug; impl fmt::Debug for EllipsisDebug { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("...") } } libcst-0.1.0/src/tokenizer/mod.rs000064400000000000000000000005010072674642500150540ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. mod core; mod debug_utils; mod operators; mod text_position; pub mod whitespace_parser; pub use self::core::*; #[cfg(test)] mod tests; libcst-0.1.0/src/tokenizer/operators.rs000064400000000000000000000050220072674642500163160ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. // // Part of this file is derived from the CPython documentation, which is available under the // zero-clause BSD license. That license does not require that derivative works cite the original // code or that we retain the original work's copyright information. // https://docs.python.org/3/license.html#zero-clause-bsd-license-for-code-in-the-python-release-documentation use regex::Regex; /// A list of strings that make up all the possible operators in a specific version of Python. /// Derived from the [CPython's token documentation](https://docs.python.org/3/library/token.html). pub const OPERATORS: &[&str] = &[ "(", // LPAR ")", // RPAR "[", // LSQB "]", // RSQB ":", // COLON ",", // COMMA ";", // SEMI "+", // PLUS "-", // MINUS "*", // STAR "/", // SLASH "|", // VBAR "&", // AMPER "<", // LESS ">", // GREATER "=", // EQUAL ".", // DOT "%", // PERCENT "{", // LBRACE "}", // RBRACE "==", // EQEQUAL "!=", // NOTEQUAL "<=", // LESSEQUAL ">=", // GREATEREQUAL "~", // TILDE "^", // CIRCUMFLEX "<<", // LEFTSHIFT ">>", // RIGHTSHIFT "**", // DOUBLESTAR "+=", // PLUSEQUAL "-=", // MINEQUAL "*=", // STAREQUAL "/=", // SLASHEQUAL "%=", // PERCENTEQUAL "&=", // AMPEREQUAL "|=", // VBAREQUAL "^=", // CIRCUMFLEXEQUAL "<<=", // LEFTSHIFTEQUAL ">>=", // RIGHTSHIFTEQUAL "**=", // DOUBLESTAREQUAL "//", // DOUBLESLASH "//=", // DOUBLESLASHEQUAL "@", // AT "@=", // ATEQUAL "->", // RARROW "...", // ELLIPSIS ":=", // COLONEQUAL // Not a real operator, but needed to support the split_fstring feature "!", // The fake operator added by PEP 401. Technically only valid if used with: // // from __future__ import barry_as_FLUFL "<>", ]; thread_local! { pub static OPERATOR_RE: Regex = { // sort operators so that we try to match the longest ones first let mut sorted_operators: Box<[&str]> = OPERATORS.into(); sorted_operators.sort_unstable_by_key(|op| usize::MAX - op.len()); Regex::new(&format!( r"\A({})", sorted_operators .iter() .map(|op| regex::escape(op)) .collect::>() .join("|") )) .expect("regex") }; } libcst-0.1.0/src/tokenizer/tests.rs000064400000000000000000000555050072674642500154550ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. /// Tests for the functionality in `tokenize::core`. These tests are not part of the `core` module /// because they're not a derivative work of CPython, and are therefore not subject to the PSF /// license. use crate::tokenizer::core::{TokConfig, TokError, TokState, TokType}; fn default_config() -> TokConfig { TokConfig { async_hacks: false, split_fstring: false, } } fn tokenize_with_end_marker<'t>( text: &'t str, config: &TokConfig, ) -> Result, TokError<'t>> { let mut result = Vec::new(); let mut state = TokState::new(text, config); while let Some(tok_type) = state.next() { result.push(( tok_type?, state.text_pos.slice_from_start_pos(&state.start_pos), )); } Ok(result) } fn tokenize_all<'t>( text: &'t str, config: &TokConfig, ) -> Result, TokError<'t>> { let mut result = tokenize_with_end_marker(text, config)?; // Remove the EndMarker, since it's on every non-error token stream. assert_eq!(result.pop().expect("EndMarker").0, TokType::EndMarker); // Also remove fake newline at the end if let Some((TokType::Newline, "")) = result.last() { result.pop(); } Ok(result) } #[test] fn test_indentifier() { assert_eq!( tokenize_all("test input", &default_config()), Ok(vec![(TokType::Name, "test"), (TokType::Name, "input")]) ); assert_eq!( tokenize_all("__with_underscores", &default_config()), Ok(vec![(TokType::Name, "__with_underscores")]) ); assert_eq!( tokenize_all("{ends_with_op}", &default_config()), Ok(vec![ (TokType::Op, "{"), (TokType::Name, "ends_with_op"), (TokType::Op, "}") ]) ); assert_eq!( tokenize_all("\u{0100}\u{0101}\u{0102}unicode", &default_config()), Ok(vec![(TokType::Name, "\u{0100}\u{0101}\u{0102}unicode")]) ); } #[test] fn test_async_await() { // normally async/await are keywords assert_eq!( tokenize_all("async await", &default_config()), Ok(vec![(TokType::Async, "async"), (TokType::Await, "await")]) ); // with async_hacks, async/await are handled as identifiers by default assert_eq!( tokenize_all( "async await", &TokConfig { async_hacks: true, ..default_config() } ), Ok(vec![(TokType::Name, "async"), (TokType::Name, "await")]) ); // with async_hacks, async/await are handled as keywords in functions assert_eq!( tokenize_all( "async def fn():\n await foo\nawait bar", &TokConfig { async_hacks: true, ..default_config() } ), Ok(vec![ // this async is followed by a def, so it's converted to an Async (TokType::Async, "async"), (TokType::Name, "def"), (TokType::Name, "fn"), (TokType::Op, "("), (TokType::Op, ")"), (TokType::Op, ":"), (TokType::Newline, "\n"), (TokType::Indent, ""), // this await is inside a function, and is converted into an Await (TokType::Await, "await"), (TokType::Name, "foo"), (TokType::Newline, "\n"), (TokType::Dedent, ""), // this await is outside the function, and is turned into an identifier (TokType::Name, "await"), (TokType::Name, "bar") ]) ); } #[test] fn test_blankline() { assert_eq!( tokenize_all("\n \n\t\n\x0c\n\n", &default_config()), Ok(vec![]) ); } #[test] fn test_newline() { assert_eq!( tokenize_all("a\nb\rc\r\n", &default_config()), Ok(vec![ (TokType::Name, "a"), (TokType::Newline, "\n"), (TokType::Name, "b"), (TokType::Newline, "\r"), (TokType::Name, "c"), (TokType::Newline, "\r\n") ]) ); } #[test] fn test_indent_dedent() { assert_eq!( tokenize_all("one\n two\n sameindent\n", &default_config()), Ok(vec![ (TokType::Name, "one"), (TokType::Newline, "\n"), (TokType::Indent, ""), (TokType::Name, "two"), (TokType::Newline, "\n"), (TokType::Name, "sameindent"), (TokType::Newline, "\n"), (TokType::Dedent, "") ]) ); assert_eq!( tokenize_all("one\n two\n \tthree\n", &default_config()), Ok(vec![ (TokType::Name, "one"), (TokType::Newline, "\n"), (TokType::Indent, ""), (TokType::Name, "two"), (TokType::Newline, "\n"), (TokType::Indent, ""), (TokType::Name, "three"), (TokType::Newline, "\n"), (TokType::Dedent, ""), (TokType::Dedent, "") ]) ); // indentation decreases to a new (smaller) indentation level that wasn't on the stack assert_eq!( tokenize_all(" one\n two", &default_config()), Err(TokError::Dedent), ); // TabSpace error without change in indentation assert_eq!( tokenize_all(" one\n\ttwo\n", &default_config()), Err(TokError::TabSpace), ); // TabSpace error with increase in indentation assert_eq!( tokenize_all(" one\n\t\ttwo\n", &default_config()), Err(TokError::TabSpace), ); // TabSpace error with decrease in indentation assert_eq!( tokenize_all(" one\n \ttwo\n\tthree\n", &default_config()), Err(TokError::TabSpace), ); // this looks like a TabSpace error, but CPython allows it, so we should too assert!(tokenize_all(" \tone\n\t two\n", &default_config()).is_ok()); } #[test] fn test_integer_decimal() { assert_eq!( tokenize_all("123456789", &default_config()), Ok(vec![(TokType::Number, "123456789")]) ); assert_eq!( tokenize_all("1_2_3", &default_config()), Ok(vec![(TokType::Number, "1_2_3")]) ); // doesn't consume trailing underscores assert_eq!( tokenize_all("123_", &default_config()), Err(TokError::BadDecimal), ); } #[test] fn test_integer_leading_zeros() { assert_eq!( tokenize_all("000", &default_config()), Ok(vec![(TokType::Number, "000")]) ); assert_eq!( tokenize_all("0_0_0", &default_config()), Ok(vec![(TokType::Number, "0_0_0")]) ); assert_eq!( tokenize_all("00123", &default_config()), Err(TokError::BadDecimalLeadingZeros) ); } #[test] fn test_integer_hexadecimal() { assert_eq!( tokenize_all("0x00Aa12Ff", &default_config()), Ok(vec![(TokType::Number, "0x00Aa12Ff")]), ); assert_eq!( tokenize_all("0x_1_2_3", &default_config()), Ok(vec![(TokType::Number, "0x_1_2_3")]), ); assert_eq!( tokenize_all("0x123_", &default_config()), Err(TokError::BadHexadecimal), ); } #[test] fn test_integer_octal() { assert_eq!( tokenize_all("0o001234567", &default_config()), Ok(vec![(TokType::Number, "0o001234567")]), ); assert_eq!( tokenize_all("0o_1_2_3", &default_config()), Ok(vec![(TokType::Number, "0o_1_2_3")]), ); assert_eq!( tokenize_all("0o123_", &default_config()), Err(TokError::BadOctal), ); assert_eq!( tokenize_all("0o789", &default_config()), Err(TokError::BadOctalDigit('8')), ); } #[test] fn test_integer_binary() { assert_eq!( tokenize_all("0b00101011", &default_config()), Ok(vec![(TokType::Number, "0b00101011")]), ); assert_eq!( tokenize_all("0b_0_1_0_1", &default_config()), Ok(vec![(TokType::Number, "0b_0_1_0_1")]), ); assert_eq!( tokenize_all("0b0101_", &default_config()), Err(TokError::BadBinary), ); assert_eq!( tokenize_all("0b0123", &default_config()), Err(TokError::BadBinaryDigit('2')), ); } #[test] fn test_fraction() { // fraction starting with a dot assert_eq!( tokenize_all(".5", &default_config()), Ok(vec![(TokType::Number, ".5")]) ); // fraction starting with a dot using E assert_eq!( tokenize_all(".5e9", &default_config()), Ok(vec![(TokType::Number, ".5e9")]) ); // fraction starting with a dot using J assert_eq!( tokenize_all(".5j", &default_config()), Ok(vec![(TokType::Number, ".5j")]) ); // fraction starting with a zero assert_eq!( tokenize_all("0.5", &default_config()), Ok(vec![(TokType::Number, "0.5")]) ); // fraction starting with a zero using E assert_eq!( tokenize_all("0.5e9", &default_config()), Ok(vec![(TokType::Number, "0.5e9")]) ); // fraction starting with a zero using J assert_eq!( tokenize_all("0.5j", &default_config()), Ok(vec![(TokType::Number, "0.5j")]) ); // fraction with underscores assert_eq!( tokenize_all("1_0.2_5", &default_config()), Ok(vec![(TokType::Number, "1_0.2_5")]) ); // underscores after the fraction are an error assert_eq!( tokenize_all(".5_", &default_config()), Err(TokError::BadDecimal), ); // doesn't consume underscores around the dot assert_eq!( tokenize_all("1_.25", &default_config()), Err(TokError::BadDecimal), ); // doesn't consume underscores around the dot assert_eq!( tokenize_all("1._25", &default_config()), Ok(vec![(TokType::Number, "1."), (TokType::Name, "_25")]) ); } #[test] fn test_string() { // empty, single quote assert_eq!( tokenize_all("''", &default_config()), Ok(vec![(TokType::String, "''")]), ); // empty, double quote assert_eq!( tokenize_all(r#""""#, &default_config()), Ok(vec![(TokType::String, r#""""#)]), ); // simple string assert_eq!( tokenize_all("'test'", &default_config()), Ok(vec![(TokType::String, "'test'")]), ); // mixed quotes assert_eq!( tokenize_all(r#""test'"#, &default_config()), Err(TokError::UnterminatedString), ); // single quoted strings can contain double quotes, double quoted strings can contain single // quotes assert_eq!( tokenize_all( r#"'she said "hey"' "but he'd ignored her""#, &default_config() ), Ok(vec![ (TokType::String, r#"'she said "hey"'"#), (TokType::String, r#""but he'd ignored her""#) ]), ); // escape characters assert_eq!( tokenize_all("'a\\b\\c\\d\\e\\'\\f\\g'", &default_config()), Ok(vec![(TokType::String, "'a\\b\\c\\d\\e\\'\\f\\g'"),]), ); // newline in the middle of a string causes an unterminated string assert_eq!( tokenize_all("'first\nsecond'", &default_config()), Err(TokError::UnterminatedString), ); // newlines can be escaped and are preserved in the output assert_eq!( tokenize_all("'first\\\nsecond\\\r\nthird\\\r'", &default_config()), Ok(vec![(TokType::String, "'first\\\nsecond\\\r\nthird\\\r'"),]), ); } #[test] fn test_string_triple_quoted() { // empty, single quote assert_eq!( tokenize_all("''''''", &default_config()), Ok(vec![(TokType::String, "''''''")]), ); // empty, double quote assert_eq!( tokenize_all(r#""""""""#, &default_config()), Ok(vec![(TokType::String, r#""""""""#)]), ); // simple string with newlines assert_eq!( tokenize_all("'''\nmulti\rline\r\n'''", &default_config()), Ok(vec![(TokType::String, "'''\nmulti\rline\r\n'''")]), ); // unterminated string assert_eq!( tokenize_all( "'''hey'there's''quotes'here, but not '' three'", &default_config() ), Err(TokError::UnterminatedTripleQuotedString), ); } #[test] fn test_string_prefix() { // works with double-quoted string assert_eq!( tokenize_all(r#"b"""#, &default_config()), Ok(vec![(TokType::String, r#"b"""#)]), ); // works with triple-quoted string assert_eq!( tokenize_all("b'''test'''", &default_config()), Ok(vec![(TokType::String, "b'''test'''")]), ); // prefix can be capitalized assert_eq!( tokenize_all("B'' R'' U'' F''", &default_config()), Ok(vec![ (TokType::String, "B''"), (TokType::String, "R''"), (TokType::String, "U''"), (TokType::String, "F''"), ]), ); // valid prefixes assert_eq!( tokenize_all("b'' r'' u'' f'' br'' fr'' rb'' rf''", &default_config()), Ok(vec![ (TokType::String, "b''"), (TokType::String, "r''"), (TokType::String, "u''"), (TokType::String, "f''"), (TokType::String, "br''"), (TokType::String, "fr''"), (TokType::String, "rb''"), (TokType::String, "rf''"), ]), ); // invalid prefixes assert_eq!( tokenize_all("bb'' rr'' uu'' ff'' ur'' ub'' uf'' fb''", &default_config()), Ok(vec![ (TokType::Name, "bb"), (TokType::String, "''"), (TokType::Name, "rr"), (TokType::String, "''"), (TokType::Name, "uu"), (TokType::String, "''"), (TokType::Name, "ff"), (TokType::String, "''"), (TokType::Name, "ur"), (TokType::String, "''"), (TokType::Name, "ub"), (TokType::String, "''"), (TokType::Name, "uf"), (TokType::String, "''"), (TokType::Name, "fb"), (TokType::String, "''"), ]), ); // raw string escapes assert_eq!( tokenize_all("r'\\''", &default_config()), Ok(vec![(TokType::String, "r'\\''")]), ); assert_eq!( tokenize_all(r#"r"\"""#, &default_config()), Ok(vec![(TokType::String, r#"r"\"""#)]), ); assert_eq!( tokenize_all(r#"r'\\'"#, &default_config()), Ok(vec![(TokType::String, r#"r'\\'"#)]), ); let config = TokConfig { split_fstring: true, ..default_config() }; assert_eq!( tokenize_all("rf'\\''", &config), Ok(vec![ (TokType::FStringStart, "rf'"), (TokType::FStringString, "\\'"), (TokType::FStringEnd, "'"), ]), ); assert_eq!( tokenize_all(r#"rf"\"""#, &config), Ok(vec![ (TokType::FStringStart, "rf\""), (TokType::FStringString, r#"\""#), (TokType::FStringEnd, "\""), ]), ); assert_eq!( tokenize_all(r#"rf'\\'"#, &config), Ok(vec![ (TokType::FStringStart, "rf'"), (TokType::FStringString, r#"\\"#), (TokType::FStringEnd, "'"), ]), ); } #[test] fn test_split_fstring() { let config = TokConfig { split_fstring: true, ..default_config() }; assert_eq!( tokenize_all("f''", &config), Ok(vec![ (TokType::FStringStart, "f'"), (TokType::FStringEnd, "'"), ]), ); assert_eq!( tokenize_all("f'{value}'", &config), Ok(vec![ (TokType::FStringStart, "f'"), (TokType::Op, "{"), (TokType::Name, "value"), (TokType::Op, "}"), (TokType::FStringEnd, "'"), ]), ); assert_eq!( tokenize_all("f'{{just a string}}'", &config), Ok(vec![ (TokType::FStringStart, "f'"), (TokType::FStringString, r"{{just a string}}"), (TokType::FStringEnd, "'"), ]), ); assert_eq!( tokenize_all(r"f'\N{Latin Small Letter A}'", &config), Ok(vec![ (TokType::FStringStart, "f'"), (TokType::FStringString, r"\N{Latin Small Letter A}"), (TokType::FStringEnd, "'"), ]), ); // format specifier assert_eq!( tokenize_all("f'result: {value:{width}.{precision}}'", &config), Ok(vec![ (TokType::FStringStart, "f'"), (TokType::FStringString, "result: "), (TokType::Op, "{"), (TokType::Name, "value"), (TokType::Op, ":"), (TokType::Op, "{"), (TokType::Name, "width"), (TokType::Op, "}"), (TokType::FStringString, "."), (TokType::Op, "{"), (TokType::Name, "precision"), (TokType::Op, "}"), (TokType::Op, "}"), (TokType::FStringEnd, "'"), ]), ); // the walrus operator isn't valid unless parenthesized assert_eq!( tokenize_all("f'{a := b}'", &config), Ok(vec![ (TokType::FStringStart, "f'"), (TokType::Op, "{"), (TokType::Name, "a"), (TokType::Op, ":"), (TokType::FStringString, "= b"), (TokType::Op, "}"), (TokType::FStringEnd, "'"), ]), ); // once parenthesized, this is recognized as the walrus operator assert_eq!( tokenize_all("f'{(a := b)}'", &config), Ok(vec![ (TokType::FStringStart, "f'"), (TokType::Op, "{"), (TokType::Op, "("), (TokType::Name, "a"), (TokType::Op, ":="), (TokType::Name, "b"), (TokType::Op, ")"), (TokType::Op, "}"), (TokType::FStringEnd, "'"), ]), ); } #[test] fn test_fstring_escapes() { let config = TokConfig { split_fstring: true, ..default_config() }; assert_eq!( tokenize_all("f'\\{{\\}}'", &config), Ok(vec![ (TokType::FStringStart, "f'"), (TokType::FStringString, "\\{{\\}}"), (TokType::FStringEnd, "'"), ]) ); assert_eq!( tokenize_all(r#"f"regexp_like(path, '.*\{file_type}$')""#, &config), Ok(vec![ (TokType::FStringStart, "f\""), (TokType::FStringString, "regexp_like(path, '.*\\"), (TokType::Op, "{"), (TokType::Name, "file_type"), (TokType::Op, "}"), (TokType::FStringString, "$')"), (TokType::FStringEnd, "\""), ]) ); } #[test] fn test_operator() { assert_eq!( tokenize_all("= == * ** **= -> . .. ...", &default_config()), Ok(vec![ (TokType::Op, "="), (TokType::Op, "=="), (TokType::Op, "*"), (TokType::Op, "**"), (TokType::Op, "**="), (TokType::Op, "->"), (TokType::Op, "."), (TokType::Op, "."), (TokType::Op, "."), (TokType::Op, "...") ]), ); } #[test] fn test_fake_newline() { assert_eq!( tokenize_with_end_marker("foo", &default_config()), Ok(vec![ (TokType::Name, "foo"), (TokType::Newline, ""), (TokType::EndMarker, "") ]) ); } #[test] fn test_fake_newline_when_at_bol() { assert_eq!( tokenize_with_end_marker("(\n \\\n)", &default_config()), Ok(vec![ (TokType::Op, "("), (TokType::Op, ")"), (TokType::Newline, ""), (TokType::EndMarker, "") ]) ) } #[test] fn test_no_fake_newline_for_empty_input() { assert_eq!( tokenize_with_end_marker("", &default_config()), Ok(vec![(TokType::EndMarker, "")]) ); } #[test] fn test_no_fake_newline_for_only_whitespaces() { assert_eq!( tokenize_with_end_marker(" ", &default_config()), Ok(vec![(TokType::EndMarker, "")]) ); } #[test] fn test_add_dedents_after_fake_newline() { assert_eq!( tokenize_with_end_marker("if 1:\n if 2:\n foo", &default_config()), Ok(vec![ (TokType::Name, "if"), (TokType::Number, "1"), (TokType::Op, ":"), (TokType::Newline, "\n"), (TokType::Indent, ""), (TokType::Name, "if"), (TokType::Number, "2"), (TokType::Op, ":"), (TokType::Newline, "\n"), (TokType::Indent, ""), (TokType::Name, "foo"), (TokType::Newline, ""), (TokType::Dedent, ""), (TokType::Dedent, ""), (TokType::EndMarker, "") ]) ); } #[test] fn test_add_dedents_for_dangling_indent() { assert_eq!( tokenize_with_end_marker("if 1:\n if 2:\n ", &default_config()), Ok(vec![ (TokType::Name, "if"), (TokType::Number, "1"), (TokType::Op, ":"), (TokType::Newline, "\n"), (TokType::Indent, ""), (TokType::Name, "if"), (TokType::Number, "2"), (TokType::Op, ":"), (TokType::Newline, "\n"), (TokType::Dedent, ""), (TokType::EndMarker, "") ]) ); } #[test] fn test_add_dedents_for_dangling_indent_with_comment() { assert_eq!( tokenize_with_end_marker("if 1:\n if 2:\n # foo", &default_config()), Ok(vec![ (TokType::Name, "if"), (TokType::Number, "1"), (TokType::Op, ":"), (TokType::Newline, "\n"), (TokType::Indent, ""), (TokType::Name, "if"), (TokType::Number, "2"), (TokType::Op, ":"), (TokType::Newline, "\n"), (TokType::Dedent, ""), (TokType::EndMarker, "") ]) ); } #[test] fn test_inconsistent_indentation_at_eof() { assert_eq!( tokenize_all("if 1:\n pass\n ", &default_config()), Ok(vec![ (TokType::Name, "if"), (TokType::Number, "1"), (TokType::Op, ":"), (TokType::Newline, "\n"), (TokType::Indent, ""), (TokType::Name, "pass"), (TokType::Newline, "\n"), (TokType::Dedent, ""), ]) ) } #[test] fn test_nested_f_string_specs() { let config = TokConfig { split_fstring: true, ..default_config() }; assert_eq!( tokenize_all("f'{_:{_:}{_}}'", &config), Ok(vec![ (TokType::FStringStart, "f'"), (TokType::Op, "{"), (TokType::Name, "_"), (TokType::Op, ":"), (TokType::Op, "{"), (TokType::Name, "_"), (TokType::Op, ":"), (TokType::Op, "}"), (TokType::Op, "{"), (TokType::Name, "_"), (TokType::Op, "}"), (TokType::Op, "}"), (TokType::FStringEnd, "'") ]) ) } libcst-0.1.0/src/tokenizer/text_position/char_width.rs000064400000000000000000000220160072674642500213260ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. use std::str::Chars; #[derive(Debug, Eq, PartialEq)] pub struct CharWidth { pub byte_width: usize, pub char_width: usize, pub character: char, } /// Iterates over characters (unicode codepoints) normalizing `'\r'` and `"\r\n"` to `'\n'`. Also /// gives the width of each character, but `'\r\n'` is counted as 2 bytes and 2 characters instead /// of one even after being normalized to '\n'. #[derive(Clone)] pub struct NewlineNormalizedCharWidths<'t> { iter: Chars<'t>, text: &'t str, idx: usize, } impl<'t> NewlineNormalizedCharWidths<'t> { pub fn new(text: &'t str) -> Self { Self { text, iter: text.chars(), idx: 0, } } pub fn previous(&mut self) -> Option<::Item> { // This function is called infrequently. let mut back_iter = self.text[..self.idx].chars(); let result = match back_iter.next_back() { // Unlikely: \n, normalization *may* be needed Some('\n') => { // Peek at the previous character to see we're a `\r\n` sequence match back_iter.next_back() { Some('\r') => Some(CharWidth { byte_width: '\r'.len_utf8() + '\n'.len_utf8(), char_width: 2, character: '\n', }), _ => Some(CharWidth { byte_width: '\n'.len_utf8(), char_width: 1, character: '\n', }), } } // Unlikely: \r, normalization is needed Some('\r') => Some(CharWidth { byte_width: '\n'.len_utf8(), char_width: 1, character: '\n', }), // Common case: Not \r or \n, so no normalization is needed Some(ch) => Some(CharWidth { byte_width: ch.len_utf8(), char_width: 1, character: ch, }), // Unlikely: EOF None => None, }; if let Some(r) = &result { self.idx -= r.byte_width; self.iter = self.text[self.idx..].chars(); } result } pub fn peek_character(&self) -> Option { // This function is called very frequently. // // We're not using peekable or caching here, since this should be cheap enough on it's own, // though benchmarking might prove otherwise. match self.iter.clone().next() { Some('\r') => Some('\n'), ch => ch, } } } impl<'t> Iterator for NewlineNormalizedCharWidths<'t> { type Item = CharWidth; fn next(&mut self) -> Option { // This function is called very frequently. let result = match self.iter.next() { // Unlikely: \r, normalization is needed Some('\r') => { // Peek at the next character to see if it's '\n'. let mut speculative = self.iter.clone(); match speculative.next() { Some('\n') => { self.iter = speculative; Some(CharWidth { byte_width: '\r'.len_utf8() + '\n'.len_utf8(), char_width: 2, character: '\n', }) } _ => Some(CharWidth { byte_width: '\r'.len_utf8(), char_width: 1, character: '\n', }), } } // Common case: Not \r, so no normalization is needed Some(ch) => Some(CharWidth { byte_width: ch.len_utf8(), char_width: 1, character: ch, }), // Unlikely: EOF None => None, }; if let Some(r) = &result { self.idx += r.byte_width; } result } } #[cfg(test)] mod tests { use super::*; #[test] fn test_ascii_no_newlines() { let mut cw = NewlineNormalizedCharWidths::new("in"); // go forward assert_eq!(cw.peek_character(), Some('i')); assert_eq!( cw.next(), Some(CharWidth { byte_width: 1, char_width: 1, character: 'i' }) ); assert_eq!(cw.peek_character(), Some('n')); assert_eq!( cw.next(), Some(CharWidth { byte_width: 1, char_width: 1, character: 'n' }) ); // end of text assert_eq!(cw.peek_character(), None); assert_eq!(cw.next(), None); // go backwards assert_eq!( cw.previous(), Some(CharWidth { byte_width: 1, char_width: 1, character: 'n' }) ); assert_eq!( cw.previous(), Some(CharWidth { byte_width: 1, char_width: 1, character: 'i' }) ); // beginning of text assert_eq!(cw.previous(), None); // try going foward again assert_eq!(cw.peek_character(), Some('i')); assert_eq!( cw.next(), Some(CharWidth { byte_width: 1, char_width: 1, character: 'i' }) ); } #[test] fn test_unicode_no_newlines() { // "test" with an accented 'e' let mut cw = NewlineNormalizedCharWidths::new("t\u{00e9}st"); // go forward assert_eq!( cw.next(), Some(CharWidth { byte_width: 1, char_width: 1, character: 't' }) ); assert_eq!(cw.peek_character(), Some('\u{00e9}')); assert_eq!( cw.next(), Some(CharWidth { byte_width: 2, char_width: 1, character: '\u{00e9}' }) ); assert_eq!(cw.peek_character(), Some('s')); assert_eq!( cw.next(), Some(CharWidth { byte_width: 1, char_width: 1, character: 's' }) ); // go backwards assert_eq!( cw.previous(), Some(CharWidth { byte_width: 1, char_width: 1, character: 's' }) ); assert_eq!( cw.previous(), Some(CharWidth { byte_width: 2, char_width: 1, character: '\u{00e9}' }) ); assert_eq!( cw.previous(), Some(CharWidth { byte_width: 1, char_width: 1, character: 't' }) ); } #[test] fn test_newlines() { let mut cw = NewlineNormalizedCharWidths::new("\n\r\r\n"); // go forward assert_eq!(cw.peek_character(), Some('\n')); assert_eq!( cw.next(), Some(CharWidth { byte_width: 1, char_width: 1, character: '\n' }) ); assert_eq!(cw.peek_character(), Some('\n')); assert_eq!( cw.next(), Some(CharWidth { byte_width: 1, char_width: 1, character: '\n' }) ); assert_eq!(cw.peek_character(), Some('\n')); assert_eq!( cw.next(), Some(CharWidth { byte_width: 2, char_width: 2, character: '\n' }) ); // end of text assert_eq!(cw.peek_character(), None); assert_eq!(cw.next(), None); // go backwards assert_eq!( cw.previous(), Some(CharWidth { byte_width: 2, char_width: 2, character: '\n' }) ); assert_eq!( cw.previous(), Some(CharWidth { byte_width: 1, char_width: 1, character: '\n' }) ); assert_eq!( cw.previous(), Some(CharWidth { byte_width: 1, char_width: 1, character: '\n' }) ); // beginning of text assert_eq!(cw.previous(), None); } #[test] fn test_empty() { let mut cw = NewlineNormalizedCharWidths::new(""); assert_eq!(cw.peek_character(), None); assert_eq!(cw.next(), None); assert_eq!(cw.previous(), None); } } libcst-0.1.0/src/tokenizer/text_position/mod.rs000064400000000000000000000272530072674642500200010ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. mod char_width; use regex::Regex; use std::fmt; use crate::tokenizer::debug_utils::EllipsisDebug; use char_width::NewlineNormalizedCharWidths; pub trait TextPattern { fn match_len(&self, text: &str) -> Option; } impl TextPattern for &Regex { // make sure to anchor your regex with \A fn match_len(&self, text: &str) -> Option { self.find(text).map(|m| m.end()) } } impl TextPattern for &str { // make sure to anchor your regex with \A fn match_len(&self, text: &str) -> Option { if text.starts_with(self) { Some(self.len()) } else { None } } } // This is Clone, since that's needed to support async_hacks, but you probably don't usually want to // clone. Use TextPositionSnapshot instead. #[derive(Clone)] pub struct TextPosition<'t> { text: &'t str, char_widths: NewlineNormalizedCharWidths<'t>, inner_byte_idx: usize, inner_char_column_number: usize, inner_byte_column_number: usize, inner_line_number: usize, } /// A lightweight immutable version of TextPosition that's slightly /// cheaper to construct/store. Used for storing the start position of tokens. #[derive(Clone, PartialEq, Eq, Debug)] pub struct TextPositionSnapshot { pub inner_byte_idx: usize, pub inner_char_column_number: usize, pub inner_line_number: usize, } impl TextPositionSnapshot { pub fn byte_idx(&self) -> usize { self.inner_byte_idx } pub fn char_column_number(&self) -> usize { self.inner_char_column_number } pub fn line_number(&self) -> usize { self.inner_line_number } } impl<'t> TextPosition<'t> { pub fn new(text: &'t str) -> Self { Self { text, char_widths: NewlineNormalizedCharWidths::new(text), inner_byte_idx: 0, inner_char_column_number: 0, inner_byte_column_number: 0, inner_line_number: 1, } } /// Peeks at the next character. Similar to `std::iter::Peekable`, but doesn't modify our /// internal position counters like wrapping this in `Peekable` would. pub fn peek(&mut self) -> Option<::Item> { self.char_widths.peek_character() } /// Matches, but does not consume TextPattern. /// /// Caution: This does not normalize `'\r'` characters, like `peek()` and `next()` do. pub fn matches(&self, pattern: P) -> bool { let rest_of_text = &self.text[self.inner_byte_idx..]; let match_len = pattern.match_len(rest_of_text); match match_len { Some(match_len) => { assert!( !rest_of_text[..match_len].contains(|x| x == '\r' || x == '\n'), "matches pattern must not match a newline", ); true } None => false, } } /// Moves the iterator back one character. Panics if a newline is encountered or if we try to /// back up past the beginning of the text. pub fn backup_no_newline(&mut self) { if let Some(cw) = self.char_widths.previous() { // If we tried to back up across a newline, we'd have to recompute char_column_number, // which would be expensive, so it's unsupported. self.inner_char_column_number = self .inner_char_column_number .checked_sub(1) .expect("cannot back up past the beginning of a line."); self.inner_byte_column_number = self .inner_byte_column_number .checked_sub(cw.byte_width) .expect("cannot back up past the beginning of a line."); self.inner_byte_idx -= cw.byte_width; } else { panic!("Tried to backup past the beginning of the text.") } } /// Tries to consume the given TextPattern, moving the TextPosition forward. Returns false if no /// match was found. Does not support newlines. /// /// Panics if a newline is consumed as part of the pattern. pub fn consume(&mut self, pattern: P) -> bool { let rest_of_text = &self.text[self.inner_byte_idx..]; if let Some(len) = pattern.match_len(rest_of_text) { let new_byte_idx = self.inner_byte_idx + len; // Call next() a bunch of times to advance the character counters. There's no way to // shortcut this because we don't know how many characters are in a slice of bytes, // though we could use a faster algorithm that inspects multiple characters at once // (e.g. SIMD). while self.inner_byte_idx < new_byte_idx { // We can't support newline normalization in this API without copying the string, so // rather than exposing that (potentially dangerous) behavior, panic if it happens. assert!( self.next() != Some('\n'), "consume pattern must not match a newline", ); } // this shouldn't be possible for the provided implementations of TextPattern debug_assert!( self.inner_byte_idx == new_byte_idx, "pattern ended on a non-character boundary", ); true } else { false } } pub fn text(&self) -> &'t str { self.text } pub fn slice_from_start_pos(&self, start_pos: &TextPositionSnapshot) -> &'t str { &self.text[start_pos.byte_idx()..self.byte_idx()] } /// Returns the number of bytes we've traversed. This is useful for Rust code that needs to /// slice the input source code, since Rust slices operate on bytes and not unicode codepoints. pub fn byte_idx(&self) -> usize { self.inner_byte_idx } /// Returns the column number in terms of number of characters (unicode codepoints) past the /// beginning of the line. Zero-indexed. pub fn char_column_number(&self) -> usize { self.inner_char_column_number } pub fn byte_column_number(&self) -> usize { self.inner_byte_column_number } /// Returns the one-indexed line number. pub fn line_number(&self) -> usize { self.inner_line_number } } impl Iterator for TextPosition<'_> { type Item = char; /// Gets the next character. This has the side-effect of advancing the internal position /// counters. fn next(&mut self) -> Option { if let Some(cw) = self.char_widths.next() { self.inner_byte_idx += cw.byte_width; match cw.character { '\n' => { self.inner_line_number += 1; self.inner_char_column_number = 0; self.inner_byte_column_number = 0; } _ => { self.inner_char_column_number += cw.char_width; self.inner_byte_column_number += cw.byte_width; } } Some(cw.character) } else { None } } } impl fmt::Debug for TextPosition<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("TextPosition") .field("text", &EllipsisDebug) .field("char_widths", &EllipsisDebug) .field("inner_byte_idx", &self.inner_byte_idx) .field("inner_char_column_number", &self.inner_char_column_number) .field("inner_byte_column_number", &self.inner_byte_column_number) .field("inner_line_number", &self.inner_line_number) .finish() } } impl From<&TextPosition<'_>> for TextPositionSnapshot { fn from(tp: &TextPosition) -> Self { Self { inner_byte_idx: tp.inner_byte_idx, inner_char_column_number: tp.inner_char_column_number, inner_line_number: tp.inner_line_number, } } } #[cfg(test)] mod tests { use super::*; #[test] fn test_empty() { let mut pos = TextPosition::new(""); assert_eq!(pos.byte_idx(), 0); assert_eq!(pos.char_column_number(), 0); assert_eq!(pos.line_number(), 1); assert_eq!(pos.peek(), None); assert!(!pos.consume(&Regex::new(r"\Awon't match").unwrap())); assert!(pos.consume(&Regex::new(r"\A").unwrap())); assert_eq!(pos.next(), None); // call next() again to verify that it's fused assert_eq!(pos.next(), None); } #[test] fn test_ascii() { let mut pos = TextPosition::new("abcdefg"); assert_eq!(pos.peek(), Some('a')); assert_eq!(pos.next(), Some('a')); assert_eq!(pos.byte_idx(), 1); assert_eq!(pos.char_column_number(), 1); assert_eq!(pos.line_number(), 1); // consume a few characters with a regex assert!(!pos.consume(&Regex::new(r"\Awon't match").unwrap())); assert!(pos.consume(&Regex::new(r"\Abcd").unwrap())); assert_eq!(pos.byte_idx(), 4); assert_eq!(pos.char_column_number(), 4); assert_eq!(pos.line_number(), 1); // consume the rest of the text assert_eq!(pos.next(), Some('e')); assert_eq!(pos.next(), Some('f')); assert_eq!(pos.next(), Some('g')); assert_eq!(pos.next(), None); assert_eq!(pos.byte_idx(), 7); assert_eq!(pos.char_column_number(), 7); assert_eq!(pos.line_number(), 1); } #[test] fn test_unicode() { let mut pos = TextPosition::new("\u{00e9}abc"); assert_eq!(pos.peek(), Some('\u{00e9}')); assert_eq!(pos.next(), Some('\u{00e9}')); } #[test] fn test_newline_lf() { let mut pos = TextPosition::new("ab\nde"); assert_eq!(pos.next(), Some('a')); assert_eq!(pos.next(), Some('b')); assert_eq!(pos.line_number(), 1); assert_eq!(pos.char_column_number(), 2); assert_eq!(pos.next(), Some('\n')); assert_eq!(pos.line_number(), 2); assert_eq!(pos.char_column_number(), 0); assert_eq!(pos.next(), Some('d')); assert_eq!(pos.next(), Some('e')); assert_eq!(pos.next(), None); assert_eq!(pos.line_number(), 2); assert_eq!(pos.char_column_number(), 2); assert_eq!(pos.byte_idx(), 5); } #[test] fn test_newline_cr() { let mut pos = TextPosition::new("ab\rde"); assert_eq!(pos.next(), Some('a')); assert_eq!(pos.next(), Some('b')); assert_eq!(pos.line_number(), 1); assert_eq!(pos.char_column_number(), 2); assert_eq!(pos.next(), Some('\n')); assert_eq!(pos.line_number(), 2); assert_eq!(pos.char_column_number(), 0); assert_eq!(pos.next(), Some('d')); assert_eq!(pos.next(), Some('e')); assert_eq!(pos.next(), None); assert_eq!(pos.line_number(), 2); assert_eq!(pos.char_column_number(), 2); assert_eq!(pos.byte_idx(), 5); } #[test] fn test_newline_cr_lf() { let mut pos = TextPosition::new("ab\r\nde"); assert_eq!(pos.next(), Some('a')); assert_eq!(pos.next(), Some('b')); assert_eq!(pos.line_number(), 1); assert_eq!(pos.char_column_number(), 2); assert_eq!(pos.next(), Some('\n')); assert_eq!(pos.line_number(), 2); assert_eq!(pos.char_column_number(), 0); assert_eq!(pos.next(), Some('d')); assert_eq!(pos.next(), Some('e')); assert_eq!(pos.next(), None); assert_eq!(pos.line_number(), 2); assert_eq!(pos.char_column_number(), 2); assert_eq!(pos.byte_idx(), 6); } } libcst-0.1.0/src/tokenizer/whitespace_parser.rs000064400000000000000000000336320072674642500200200ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree use crate::nodes::{ Comment, EmptyLine, Fakeness, Newline, ParenthesizableWhitespace, ParenthesizedWhitespace, SimpleWhitespace, TrailingWhitespace, }; use memchr::memchr2_iter; use regex::Regex; use thiserror::Error; use crate::Token; use super::TokType; thread_local! { static SIMPLE_WHITESPACE_RE: Regex = Regex::new(r"\A([ \f\t]|\\(\r\n?|\n))*").expect("regex"); static NEWLINE_RE: Regex = Regex::new(r"\A(\r\n?|\n)").expect("regex"); static COMMENT_RE: Regex = Regex::new(r"\A#[^\r\n]*").expect("regex"); } #[allow(clippy::upper_case_acronyms, clippy::enum_variant_names)] #[derive(Error, Debug, PartialEq, Eq)] pub enum WhitespaceError { #[error("WTF")] WTF, #[error("Internal error while parsing whitespace: {0}")] InternalError(String), #[error("Failed to parse mandatory trailing whitespace")] TrailingWhitespaceError, } type Result = std::result::Result; #[derive(Debug, PartialEq, Eq, Clone)] pub struct State<'a> { pub line: usize, // one-indexed (to match parso's behavior) pub column: usize, // zero-indexed (to match parso's behavior) pub column_byte: usize, pub absolute_indent: &'a str, pub is_parenthesized: bool, pub byte_offset: usize, } impl<'a> Default for State<'a> { fn default() -> Self { Self { line: 1, column: 0, column_byte: 0, absolute_indent: "", is_parenthesized: false, byte_offset: 0, } } } // TODO pub struct Config<'a> { pub input: &'a str, pub lines: Vec<&'a str>, pub default_newline: &'a str, pub default_indent: &'a str, } impl<'a> Config<'a> { pub fn new(input: &'a str, tokens: &[Token<'a>]) -> Self { let mut default_indent = " "; for tok in tokens { if tok.r#type == TokType::Indent { default_indent = tok.relative_indent.unwrap(); break; } } let mut lines = Vec::new(); let mut start = 0; let mut newline_positions = memchr2_iter(b'\n', b'\r', input.as_bytes()); while let Some(newline_position) = newline_positions.next() { let newline_character = input.as_bytes()[newline_position] as char; let len = if newline_character == '\r' && input.as_bytes().get(newline_position + 1) == Some(&b'\n') { // Skip the next '\n' newline_positions.next(); 2 } else { 1 }; let end = newline_position + len; lines.push(&input[start..end]); start = end; } // Push the last line if it isn't terminated by a newline character if start < input.len() { lines.push(&input[start..]); } let default_newline = match lines.first().map(|line| line.as_bytes()).unwrap_or(&[]) { [.., b'\r', b'\n'] => "\r\n", [.., b'\n'] => "\n", [.., b'\r'] => "\r", _ => "\n", }; Self { input, lines, default_newline, default_indent, } } pub fn has_trailing_newline(&self) -> bool { self.input.ends_with('\n') && !self.input.ends_with("\\\n") && !self.input.ends_with("\\\r\n") } fn get_line(&self, line_number: usize) -> Result<&'a str> { let err_fn = || { WhitespaceError::InternalError(format!( "tried to get line {} which is out of range", line_number )) }; self.lines .get(line_number.checked_sub(1).ok_or_else(err_fn)?) .map(|l| &l[..]) .ok_or_else(err_fn) } fn get_line_after_column(&self, line_number: usize, column_index: usize) -> Result<&'a str> { self.get_line(line_number)? .get(column_index..) .ok_or_else(|| { WhitespaceError::InternalError(format!( "Column index {} out of range for line {}", column_index, line_number )) }) } } #[derive(Debug)] enum ParsedEmptyLine<'a> { NoIndent, Line(EmptyLine<'a>), } fn parse_empty_line<'a>( config: &Config<'a>, state: &mut State, override_absolute_indent: Option<&'a str>, ) -> Result> { let mut speculative_state = state.clone(); if let Ok(indent) = parse_indent(config, &mut speculative_state, override_absolute_indent) { let whitespace = parse_simple_whitespace(config, &mut speculative_state)?; let comment = parse_comment(config, &mut speculative_state)?; if let Some(newline) = parse_newline(config, &mut speculative_state)? { *state = speculative_state; return Ok(ParsedEmptyLine::Line(EmptyLine { indent, whitespace, comment, newline, })); } } Ok(ParsedEmptyLine::NoIndent) } fn _parse_empty_lines<'a>( config: &Config<'a>, state: &mut State<'a>, override_absolute_indent: Option<&'a str>, ) -> Result, EmptyLine<'a>)>> { let mut lines = vec![]; loop { let last_state = state.clone(); let parsed_line = parse_empty_line(config, state, override_absolute_indent)?; if *state == last_state { break; } match parsed_line { ParsedEmptyLine::NoIndent => break, ParsedEmptyLine::Line(l) => lines.push((state.clone(), l)), } } Ok(lines) } pub fn parse_empty_lines<'a>( config: &Config<'a>, state: &mut State<'a>, override_absolute_indent: Option<&'a str>, ) -> Result>> { // If override_absolute_indent is Some, then we need to parse all lines up to and including the // last line that is indented at our level. These all belong to the footer and not to the next // line's leading_lines. // // We don't know what the last line with indent=True is, and there could be indent=False lines // interspersed with indent=True lines, so we need to speculatively parse all possible empty // lines, and then unwind to find the last empty line with indent=True. let mut speculative_state = state.clone(); let mut lines = _parse_empty_lines(config, &mut speculative_state, override_absolute_indent)?; if override_absolute_indent.is_some() { // Remove elements from the end until we find an indented line. while let Some((_, empty_line)) = lines.last() { if empty_line.indent { break; } lines.pop(); } } if let Some((final_state, _)) = lines.last() { // update the state to match the last line that we captured *state = final_state.clone(); } Ok(lines.into_iter().map(|(_, e)| e).collect()) } pub fn parse_comment<'a>(config: &Config<'a>, state: &mut State) -> Result>> { let newline_after = config.get_line_after_column(state.line, state.column_byte)?; if let Some(comment_match) = COMMENT_RE.with(|r| r.find(newline_after)) { let comment_str = comment_match.as_str(); advance_this_line( config, state, comment_str.chars().count(), comment_str.len(), )?; return Ok(Some(Comment(comment_str))); } Ok(None) } pub fn parse_newline<'a>(config: &Config<'a>, state: &mut State) -> Result>> { let newline_after = config.get_line_after_column(state.line, state.column_byte)?; if let Some(newline_match) = NEWLINE_RE.with(|r| r.find(newline_after)) { let newline_str = newline_match.as_str(); advance_this_line( config, state, newline_str.chars().count(), newline_str.len(), )?; if state.column_byte != config.get_line(state.line)?.len() { return Err(WhitespaceError::InternalError(format!( "Found newline at ({}, {}) but it's not EOL", state.line, state.column ))); } if state.line < config.lines.len() { advance_to_next_line(config, state)?; } return Ok(Some(Newline( if newline_str == config.default_newline { None } else { Some(newline_str) }, Fakeness::Real, ))); } // If we're at the end of the file but not on BOL, that means this is the fake // newline inserted by the tokenizer. if state.byte_offset == config.input.len() && state.column_byte != 0 { return Ok(Some(Newline(None, Fakeness::Fake))); } Ok(None) } pub fn parse_optional_trailing_whitespace<'a>( config: &Config<'a>, state: &mut State, ) -> Result>> { let mut speculative_state = state.clone(); let whitespace = parse_simple_whitespace(config, &mut speculative_state)?; let comment = parse_comment(config, &mut speculative_state)?; if let Some(newline) = parse_newline(config, &mut speculative_state)? { *state = speculative_state; Ok(Some(TrailingWhitespace { whitespace, comment, newline, })) } else { Ok(None) } } pub fn parse_trailing_whitespace<'a>( config: &Config<'a>, state: &mut State, ) -> Result> { match parse_optional_trailing_whitespace(config, state)? { Some(ws) => Ok(ws), _ => Err(WhitespaceError::TrailingWhitespaceError), } } fn parse_indent<'a>( config: &Config<'a>, state: &mut State, override_absolute_indent: Option<&'a str>, ) -> Result { let absolute_indent = override_absolute_indent.unwrap_or(state.absolute_indent); if state.column_byte != 0 { if state.column_byte == config.get_line(state.line)?.len() && state.line == config.lines.len() { Ok(false) } else { Err(WhitespaceError::InternalError( "Column should not be 0 when parsing an index".to_string(), )) } } else { Ok( if config .get_line_after_column(state.line, state.column_byte)? .starts_with(absolute_indent) { state.column_byte += absolute_indent.len(); state.column += absolute_indent.chars().count(); state.byte_offset += absolute_indent.len(); true } else { false }, ) } } fn advance_to_next_line<'a>(config: &Config<'a>, state: &mut State) -> Result<()> { let cur_line = config.get_line(state.line)?; state.byte_offset += cur_line.len() - state.column_byte; state.column = 0; state.column_byte = 0; state.line += 1; Ok(()) } fn advance_this_line<'a>( config: &Config<'a>, state: &mut State, char_count: usize, offset: usize, ) -> Result<()> { let cur_line = config.get_line(state.line)?; if cur_line.len() < state.column_byte + offset { return Err(WhitespaceError::InternalError(format!( "Tried to advance past line {}'s end", state.line ))); } state.column += char_count; state.column_byte += offset; state.byte_offset += offset; Ok(()) } pub fn parse_simple_whitespace<'a>( config: &Config<'a>, state: &mut State, ) -> Result> { let capture_ws = |line, col| -> Result<&'a str> { let x = config.get_line_after_column(line, col); let x = x?; Ok(SIMPLE_WHITESPACE_RE.with(|r| { r.find(x) .expect("SIMPLE_WHITESPACE_RE supports 0-length matches, so it must always match") .as_str() })) }; let start_offset = state.byte_offset; let mut prev_line: &str; loop { prev_line = capture_ws(state.line, state.column_byte)?; if !prev_line.contains('\\') { break; } advance_to_next_line(config, state)?; } advance_this_line(config, state, prev_line.chars().count(), prev_line.len())?; Ok(SimpleWhitespace( &config.input[start_offset..state.byte_offset], )) } pub fn parse_parenthesizable_whitespace<'a>( config: &Config<'a>, state: &mut State<'a>, ) -> Result> { if state.is_parenthesized { if let Some(ws) = parse_parenthesized_whitespace(config, state)? { return Ok(ParenthesizableWhitespace::ParenthesizedWhitespace(ws)); } } parse_simple_whitespace(config, state).map(ParenthesizableWhitespace::SimpleWhitespace) } pub fn parse_parenthesized_whitespace<'a>( config: &Config<'a>, state: &mut State<'a>, ) -> Result>> { if let Some(first_line) = parse_optional_trailing_whitespace(config, state)? { let empty_lines = _parse_empty_lines(config, state, None)? .into_iter() .map(|(_, line)| line) .collect(); let indent = parse_indent(config, state, None)?; let last_line = parse_simple_whitespace(config, state)?; Ok(Some(ParenthesizedWhitespace { first_line, empty_lines, indent, last_line, })) } else { Ok(None) } } #[cfg(test)] mod tests { use crate::{tokenize, Config, Result}; #[test] fn config_mixed_newlines() -> Result<'static, ()> { let source = "'' % {\n'test1': '',\r 'test2': '',\r\n}"; let tokens = tokenize(source)?; let config = Config::new(source, &tokens); assert_eq!( &config.lines, &["'' % {\n", "'test1': '',\r", " 'test2': '',\r\n", "}"] ); Ok(()) } } libcst-0.1.0/tests/fixtures/big_binary_operator.py000064400000000000000000000020430072674642500205360ustar 00000000000000( # 350 binary operators lets go 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' + 'X' + 'Y' + 'Z' + 'Q' + 'T' ) libcst-0.1.0/tests/fixtures/class_craziness.py000064400000000000000000000005700072674642500177070ustar 00000000000000class Foo: ... class Bar : ... class Old ( ) : gold : int class OO ( Foo ) : ... class OOP ( Foo , Bar, ) : pass class OOPS ( Foo , ) : pass class OOPSI ( Foo, * Bar , metaclass = foo , ): pass class OOPSIE ( list , *args, kw = arg , ** kwargs ) : what : does_this_even = mean def __init__(self) -> None: self.foo: Bar = Bar() libcst-0.1.0/tests/fixtures/comments.py000064400000000000000000000037070072674642500163530ustar 00000000000000#!/usr/bin/env python3 # fmt: on # Some license here. # # Has many lines. Many, many lines. # Many, many, many lines. """Module docstring. Possibly also many, many lines. """ import os.path import sys import a from b.c.d.e import X # some noqa comment try: import fast except ImportError: import slow as fast # Some comment before a function. y = 1 ( # some strings y # type: ignore ) def function(default=None): """Docstring comes first. Possibly many lines. """ # FIXME: Some comment about why this function is crap but still in production. import inner_imports if inner_imports.are_evil(): # Explains why we have this if. # In great detail indeed. x = X() return x.method1() # type: ignore # This return is also commented for some reason. return default # Explains why we use global state. GLOBAL_STATE = {"a": a(1), "b": a(2), "c": a(3)} # Another comment! # This time two lines. class Foo: """Docstring for class Foo. Example from Sphinx docs.""" #: Doc comment for class attribute Foo.bar. #: It can have multiple lines. bar = 1 flox = 1.5 #: Doc comment for Foo.flox. One line only. baz = 2 """Docstring for class attribute Foo.baz.""" def __init__(self): #: Doc comment for instance attribute qux. self.qux = 3 self.spam = 4 """Docstring for instance attribute spam.""" #'

This is pweave!

@fast(really=True) async def wat(): # This comment, for some reason \ # contains a trailing backslash. async with X.open_async() as x: # Some more comments result = await x.method1() # Comment after ending a block. if result: print("A OK", file=sys.stdout) # Comment between things. print() if True: # Hanging comments # because why not pass # Some closing comments. # Maybe Vim or Emacs directives for formatting. # Who knows. libcst-0.1.0/tests/fixtures/comparisons.py000064400000000000000000000007270072674642500170620ustar 00000000000000if not 1: pass if 1 and 1: pass if 1 or 1: pass if not not not 1: pass if not 1 and 1 and 1: pass if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass if 1: pass #x = (1 == 1) if 1 == 1: pass if 1 != 1: pass if 1 < 1: pass if 1 > 1: pass if 1 <= 1: pass if 1 >= 1: pass if x is x: pass #if x is not x: pass #if 1 in (): pass #if 1 not in (): pass if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 in x is x is x: pass #if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in x is x is not x: pass libcst-0.1.0/tests/fixtures/dangling_indent.py000064400000000000000000000000230072674642500176360ustar 00000000000000if 1: pass libcst-0.1.0/tests/fixtures/decorated_function_without_body.py000064400000000000000000000000440072674642500231540ustar 00000000000000@hello @bello def f () : ...libcst-0.1.0/tests/fixtures/dysfunctional_del.py000064400000000000000000000001740072674642500202270ustar 00000000000000# dysfunctional_del.py del a del a[1] del a.b.c del ( a, b , c ) del [ a, b , c ] del a , b, c del a[1] , b [ 2]libcst-0.1.0/tests/fixtures/expr.py000064400000000000000000000234150072674642500155020ustar 00000000000000... "some_string" b"\\xa3" Name None True False 1 1.0 1j True or False True or False or None True and False True and False and None (Name1 and Name2) or Name3 Name1 and Name2 or Name3 Name1 or (Name2 and Name3) Name1 or Name2 and Name3 (Name1 and Name2) or (Name3 and Name4) Name1 and Name2 or Name3 and Name4 Name1 or (Name2 and Name3) or Name4 Name1 or Name2 and Name3 or Name4 v1 << 2 1 >> v2 1 % finished 1 + v2 - v3 * 4 ^ 5 ** v6 / 7 // 8 ((1 + v2) - (v3 * 4)) ^ (((5 ** v6) / 7) // 8) not great ~great +value -1 ~int and not v1 ^ 123 + v2 | True (~int) and (not ((v1 ^ (123 + v2)) | True)) +(really ** -(confusing ** ~(operator ** -precedence))) flags & ~ select.EPOLLIN and waiters.write_task is not None lambda arg: None lambda arg : None lambda a=True: a lambda a=True : a lambda a, b, c=True: a lambda a, b, c=True, *, d=(1 << v2), e='str': a lambda a, b, c=True, *vararg, d=(v1 << 2), e='str', **kwargs: a + b lambda a, b, c=True, *vararg, d=(v1 << 2), e='str', **kwargs : a + b manylambdas = lambda x=lambda y=lambda z=1: z: y(): x() foo = (lambda port_id, ignore_missing: {"port1": port1_resource, "port2": port2_resource}[port_id]) 1 if True else 2 _ if 0else _ str or None if True else str or bytes or None (str or None) if True else (str or bytes or None) str or None if (1 if True else 2) else str or bytes or None (str or None) if (1 if True else 2) else (str or bytes or None) ((super_long_variable_name or None) if (1 if super_long_test_name else 2) else (str or bytes or None)) {'2.7': dead, '3.7': (long_live or die_hard)} {'2.7': dead, '3.7': (long_live or die_hard), **{'3.6': verygood}} {**a, **b, **c} {"2.7", "3.6", "3.7", "3.8", "3.9"} {"2.7", "3.6", "3.7", "3.8", "3.9",} {"2.7", "3.6", "3.7", "3.8", "3.9", ("4.0" if gilectomy else "3.10")} ({"a": "b"}, (True or False), (+value), "string", b"bytes") or None () (1,) (1, 2) (1, 2, 3) [] [ ] [ 1 , ] [1, 2, 3, 4, 5, 6, 7, 8, 9, (10 or A), (11 or B), (12 or C)] [ 1, 2, 3, ] [*a] [*range(10)] [ *a, 4, 5, ] [ 4, *a, 5, ] [ this_is_a_very_long_variable_which_will_force_a_delimiter_split, element, another, *more, ] { } { 1 , } { 1 : 2 , } {i for i in (1, 2, 3)} {(i ** 2) for i in (1, 2, 3)} {(i ** 2) for i, _ in ((1, "a"), (2, "b"), (3, "c"))} {((i ** 2) + j) for i in (1, 2, 3) for j in (1, 2, 3)} [i for i in (1, 2, 3)] [(i ** 2) for i in (1, 2, 3)] [(i ** 2) for i, _ in ((1, "a"), (2, "b"), (3, "c"))] [((i ** 2) + j) for i in (1, 2, 3) for j in (1, 2, 3)] {i: 0 for i in (1, 2, 3)} {i: j for i, j in ((1, "a"), (2, "b"), (3, "c"))} {a: b * 2 for a, b in dictionary.items()} {a: b * -2 for a, b in dictionary.items()} { k: v for k, v in this_is_a_very_long_variable_which_will_cause_a_trailing_comma_which_breaks_the_comprehension } Python3 > Python2 > COBOL Life is Life call() call(arg) call(kwarg="hey") call(arg, kwarg="hey") call(arg, another, kwarg="hey", **kwargs) call( this_is_a_very_long_variable_which_will_force_a_delimiter_split, arg, another, kwarg="hey", **kwargs, ) # note: no trailing comma pre-3.6 call(*gidgets[:2]) call(a, *gidgets[:2]) call(**screen_kwargs) call(b, **screen_kwargs) call()()()()()() call(**self.screen_kwargs) call(b, **self.screen_kwargs) call(a=a, *args) call(a=a, *args,) call(a=a, **kwargs) call(a=a, **kwargs,) lukasz.langa.pl call.me(maybe) 1 .real 1.0 .real ....__class__ list[str] dict[str, int] tuple[str, ...] tuple[str, int, float, dict[str, int]] tuple[ str, int, float, dict[str, int], ] very_long_variable_name_filters: t.List[ t.Tuple[str, t.Union[str, t.List[t.Optional[str]]]], ] xxxx_xxxxx_xxxx_xxx: Callable[..., List[SomeClass]] = classmethod( # type: ignore sync(async_xxxx_xxx_xxxx_xxxxx_xxxx_xxx.__func__) ) xxxx_xxx_xxxx_xxxxx_xxxx_xxx: Callable[..., List[SomeClass]] = classmethod( # type: ignore sync(async_xxxx_xxx_xxxx_xxxxx_xxxx_xxx.__func__) ) xxxx_xxx_xxxx_xxxxx_xxxx_xxx: Callable[..., List[SomeClass]] = classmethod( sync(async_xxxx_xxx_xxxx_xxxxx_xxxx_xxx.__func__) ) # type: ignore (str or None) if (sys.version_info[0] > (3,)) else (str or bytes or None) {"2.7": dead, "3.7": long_live or die_hard} {"2.7", "3.6", "3.7", "3.8", "3.9", "4.0" if gilectomy else "3.10"} [1, 2, 3, 4, 5, 6, 7, 8, 9, 10 or A, 11 or B, 12 or C] (SomeName) SomeName (Good, Bad, Ugly) (i for i in (1, 2, 3)) ((i ** 2) for i in (1, 2, 3)) ((i ** 2) for i, _ in ((1, "a"), (2, "b"), (3, "c"))) (((i ** 2) + j) for i in (1, 2, 3) for j in (1, 2, 3)) (*starred,) { "id": "1", "type": "type", "started_at": now(), "ended_at": now() + timedelta(days=10), "priority": 1, "import_session_id": 1, **kwargs, } a = (1,) b = (1,) c = 1 d = (1,) + a + (2,) e = (1,).count(1) f = 1, *range(10) g = 1, *"ten" what_is_up_with_those_new_coord_names = (coord_names + set(vars_to_create)) + set( vars_to_remove ) what_is_up_with_those_new_coord_names = (coord_names | set(vars_to_create)) - set( vars_to_remove ) result = ( session.query(models.Customer.id) .filter( models.Customer.account_id == account_id, models.Customer.email == email_address ) .order_by(models.Customer.id.asc()) .all() ) result = ( session.query(models.Customer.id) .filter( models.Customer.account_id == account_id, models.Customer.email == email_address ) .order_by( models.Customer.id.asc(), ) .all() ) Ø = set() authors.łukasz.say_thanks() authors.lukasz.say_thanks() mapping = { A: 0.25 * (10.0 / 12), B: 0.1 * (10.0 / 12), C: 0.1 * (10.0 / 12), D: 0.1 * (10.0 / 12), } [ a for [ a , ] in [ [ 1 ] ] ] def gen(): if 1: if 2: if 3: if not is_value_of_type( subkey, type_args[0], # key type is always invariant invariant_check=True, ): return False yield from outside_of_generator a = yield b = yield c = yield async def f(): await some.complicated[0].call(with_args=(True or (1 is not 1))) lambda : None print(*[] or [1]) print(**{1: 3} if False else {x: x for x in range(3)}) print(*lambda x: x) assert not Test, "Short message" assert this is ComplexTest and not requirements.fit_in_a_single_line( force=False ), "Short message" assert parens is TooMany for (x,) in (1,), (2,), (3,): ... for y in (): ... for z in (i for i in (1, 2, 3)): ... for i in call(): ... for j in 1 + (2 + 3): ... else: ... while this and that: ... while this and that: ... else: ... for ( addr_family, addr_type, addr_proto, addr_canonname, addr_sockaddr, ) in socket.getaddrinfo("google.com", "http"): pass a = ( aaaa.bbbb.cccc.dddd.eeee.ffff.gggg.hhhh.iiii.jjjj.kkkk.llll.mmmm.nnnn.oooo.pppp in qqqq.rrrr.ssss.tttt.uuuu.vvvv.xxxx.yyyy.zzzz ) a = ( aaaa.bbbb.cccc.dddd.eeee.ffff.gggg.hhhh.iiii.jjjj.kkkk.llll.mmmm.nnnn.oooo.pppp not in qqqq.rrrr.ssss.tttt.uuuu.vvvv.xxxx.yyyy.zzzz ) a = ( aaaa.bbbb.cccc.dddd.eeee.ffff.gggg.hhhh.iiii.jjjj.kkkk.llll.mmmm.nnnn.oooo.pppp is qqqq.rrrr.ssss.tttt.uuuu.vvvv.xxxx.yyyy.zzzz ) a = ( aaaa.bbbb.cccc.dddd.eeee.ffff.gggg.hhhh.iiii.jjjj.kkkk.llll.mmmm.nnnn.oooo.pppp is not qqqq.rrrr.ssss.tttt.uuuu.vvvv.xxxx.yyyy.zzzz ) if ( threading.current_thread() != threading.main_thread() and threading.current_thread() != threading.main_thread() or signal.getsignal(signal.SIGINT) != signal.default_int_handler ): return True if ( aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ): return True if ( aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa & aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ): return True if ( aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ): return True if ( aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ): return True if ( aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa * aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ): return True if ( aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa / aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ): return True if ( ~aaaa.a + aaaa.b - aaaa.c * aaaa.d / aaaa.e | aaaa.f & aaaa.g % aaaa.h ^ aaaa.i << aaaa.k >> aaaa.l ** aaaa.m // aaaa.n ): return True if ( ~aaaaaaaa.a + aaaaaaaa.b - aaaaaaaa.c @ aaaaaaaa.d / aaaaaaaa.e | aaaaaaaa.f & aaaaaaaa.g % aaaaaaaa.h ^ aaaaaaaa.i << aaaaaaaa.k >> aaaaaaaa.l ** aaaaaaaa.m // aaaaaaaa.n ): return True if ( ~aaaaaaaaaaaaaaaa.a + aaaaaaaaaaaaaaaa.b - aaaaaaaaaaaaaaaa.c * aaaaaaaaaaaaaaaa.d @ aaaaaaaaaaaaaaaa.e | aaaaaaaaaaaaaaaa.f & aaaaaaaaaaaaaaaa.g % aaaaaaaaaaaaaaaa.h ^ aaaaaaaaaaaaaaaa.i << aaaaaaaaaaaaaaaa.k >> aaaaaaaaaaaaaaaa.l ** aaaaaaaaaaaaaaaa.m // aaaaaaaaaaaaaaaa.n ): return True aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa - aaaaaaaaaaaaaaaa * ( aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa ) / (aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa) aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa >> aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa << aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa bbbb >> bbbb * bbbb aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ^ bbbb.a & aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ^ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa a += B a[x] @= foo().bar this.is_not >>= a.monad last_call() # standalone comment at ENDMARKER libcst-0.1.0/tests/fixtures/expr_statement.py000064400000000000000000000002210072674642500175540ustar 000000000000001 1, 2, 3 x = 1 x = 1, 2, 3 x = y = z = 1, 2, 3 x, y, z = 1, 2, 3 abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4) ( ( ( ... ) ) ) a , = blibcst-0.1.0/tests/fixtures/fun_with_func_defs.py000064400000000000000000000037200072674642500203600ustar 00000000000000def f(a, /,): pass def f(a, / ,): pass def f(a, / ): pass def f(a, /, c, d, e): pass def f(a, /, c, *, d, e): pass def f(a, /, c, *, d, e, **kwargs): pass def f(a=1, /,): pass def f(a=1, /, b=2, c=4): pass def f(a=1, /, b=2, *, c=4): pass def f(a=1, /, b=2, *, c): pass def f(a=1, /, b=2, *, c=4, **kwargs): pass def f(a=1, /, b=2, *, c, **kwargs,): pass def g( a, /, ): pass def f(a, /, c, d, e): pass def f(a, /, c, *, d, e): pass def foo(a, * , bar): pass def f( a, /, c, *, d, e, **kwargs, ): pass def f( a=1, /, ): pass def say_hello( self, user: str, / ): print('Hello ' + user) def f(a=1, /, b=2, c=4): pass def f(a=1, /, b=2, *, c=4): pass def f(a=1, /, b=2, *, c): pass def f( a=1, /, b=2, *, c=4, **kwargs, ): pass def f( a=1, /, b=2, *, c, **kwargs, ): pass async def foo ( bar : Baz , ) -> zooooooooom : ... async def foo(bar : Baz = 0 ) : ... async def foo() -> Bar: ... async def outer( foo ) -> Bar : def inner(lol: Lol) -> None: async def core (): await lol def second(inner): pass def stars ( yes : bool = True , / , noes : List[bool] = [ * falses ], * all : The[Rest], but : Wait[Theres[More]] , ** it : ends[now] , ) -> ret: pass def stars ( yes : bool = True , / , noes : List[bool] = [ * falses ], * all : The[Rest], but : Wait[Theres[More]] , ** it : ends[now[without_a_comma]] ) -> ret : pass def foo(bar: (yield)) -> (yield): something: (yield another) def foo( bar: (yield)) -> (yield) : something: (yield another) return 3 # no return # yes def f(): for (yield 1)[1] in [1]: pass @decorators # foo @woohoo def f(): pass @getattr(None, '', lambda a: lambda b: a(b+1)) def f(): ... @a(now_this = lol) def f(): ... libcst-0.1.0/tests/fixtures/global_nonlocal.py000064400000000000000000000000650072674642500176450ustar 00000000000000global a global b , c, d nonlocal a nonlocal a , blibcst-0.1.0/tests/fixtures/import.py000064400000000000000000000006400072674642500160310ustar 00000000000000# 'import' dotted_as_names import sys import time, sys # 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names) from time import time from time import (time) from sys import path, argv from sys import (path, argv) from sys import (path, argv,) from sys import * from a import (b, ) from . import a from .a import b from ... import a from ...a import b from .... import a from ...... import alibcst-0.1.0/tests/fixtures/indents_but_no_eol_before_eof.py000064400000000000000000000000560072674642500225440ustar 00000000000000if 1: if 2: if 3: passlibcst-0.1.0/tests/fixtures/just_a_comment_without_nl.py000064400000000000000000000000420072674642500217760ustar 00000000000000# just a comment without a newlinelibcst-0.1.0/tests/fixtures/malicious_match.py000064400000000000000000000016000072674642500176550ustar 00000000000000 # foo match ( foo ) : #comment # more comments case False : # comment ... case ( True ) : ... case _ : ... case ( _ ) : ... # foo # bar match x: case "StringMatchValue" : pass case [1, 2] : pass case [ 1 , * foo , * _ , ]: pass case [ [ _, ] , *_ ]: pass case {1: _, 2: _}: pass case { "foo" : bar , ** rest } : pass case { 1 : {**rest} , } : pass case Point2D(): pass case Cls ( 0 , ) : pass case Cls ( x=0, y = 2) :pass case Cls ( 0 , 1 , x = 0 , y = 2 ) : pass case [x] as y: pass case [x] as y : pass case (True)as x:pass case Foo:pass case (Foo):pass case ( Foo ) : pass case [ ( Foo ) , ]: pass case Foo|Bar|Baz : pass case Foo | Bar | ( Baz): pass case x,y , * more :pass case y.z: pass libcst-0.1.0/tests/fixtures/mixed_newlines.py000064400000000000000000000000470072674642500175320ustar 00000000000000"" % { 'test1': '', 'test2': '', } libcst-0.1.0/tests/fixtures/pep646.py000064400000000000000000000012450072674642500155450ustar 00000000000000# see https://github.com/python/cpython/pull/31018/files#diff-3f516b60719dd445d33225e4f316b36e85c9c51a843a0147349d11a005c55937 A[*b] A[ * b ] A[ * b , ] A[*b] = 1 del A[*b] A[* b , * b] A[ b, *b] A[* b, b] A[ * b,b, b] A[b, *b, b] A[*A[b, *b, b], b] A[b, ...] A[*A[b, ...]] A[ * ( 1,2,3)] A[ * [ 1,2,3]] A[1:2, *t] A[1:, *t, 1:2] A[:, *t, :] A[*t, :, *t] A[* returns_list()] A[*returns_list(), * returns_list(), b] def f1(*args: *b): pass def f2(*args: *b, arg1): pass def f3(*args: *b, arg1: int): pass def f4(*args: *b, arg1: int = 1): pass def f(*args: *tuple[int, ...]): pass def f(*args: *tuple[int, *Ts]): pass def f() -> tuple[int, *tuple[int, ...]]: passlibcst-0.1.0/tests/fixtures/raise.py000064400000000000000000000001020072674642500156130ustar 00000000000000raise raise foo raise foo from bar raise lol() from f() + 1libcst-0.1.0/tests/fixtures/smol_statements.py000064400000000000000000000001560072674642500177420ustar 00000000000000def f(): pass ; break ; continue ; return ; return foo assert foo , bar ; a += 2 libcst-0.1.0/tests/fixtures/spacious_spaces.py000064400000000000000000000000060072674642500176770ustar 00000000000000 libcst-0.1.0/tests/fixtures/starry_tries.py000064400000000000000000000006020072674642500172470ustar 00000000000000#foo. try : pass # foo except * lol as LOL : pass except * f: # foo pass else : pass finally : foo try: pass except*f: pass finally: pass try: # 1 try: # 2 pass # 3 # 4 finally: # 5 pass # 6 # 7 except *foo: #8 pass #9 libcst-0.1.0/tests/fixtures/suicidal_slices.py000064400000000000000000000006500072674642500176570ustar 00000000000000slice[0] slice[0:1] slice[0:1:2] slice[:] slice[:-1] slice[1:] slice[::-1] slice[d :: d + 1] slice[:c, c - 1] numpy[:, 0:1] numpy[:, :-1] numpy[0, :] numpy[:, i] numpy[0, :2] numpy[:N, 0] numpy[:2, :4] numpy[2:4, 1:5] numpy[4:, 2:] numpy[:, (0, 1, 2, 5)] numpy[0, [0]] numpy[:, [i]] numpy[1 : c + 1, c] numpy[-(c + 1) :, d] numpy[:, l[-2]] numpy[:, ::-1] numpy[np.newaxis, :] ( spaces [:: , a : , a : a : a , ] )libcst-0.1.0/tests/fixtures/super_strings.py000064400000000000000000000012420072674642500174250ustar 00000000000000_ = "" _ = '' _ = """""" _ = '''''' _ = 'a' "string" 'that' r"is" 'concatenated ' b"string " b"and non f" rb'string' ( "parenthesized" "concatenated" """triple quoted """ ) _ = f"string" f"string" "bonanza" f'starts' r"""here""" _ = f"something {{**not** an expression}} {but(this._is)} {{and this isn't.}} end" _(f"ok { expr = !r: aosidjhoi } end") print(f"{self.ERASE_CURRENT_LINE}{self._human_seconds(elapsed_time)} {percent:.{self.pretty_precision}f}% complete, {self.estimate_completion(elapsed_time, finished, left)} estimated for {left} files to go...") f'\{{\}}' f"regexp_like(path, '.*\{file_type}$')" f"\lfoo" f"{_:{_:}{a}}"libcst-0.1.0/tests/fixtures/terrible_tries.py000064400000000000000000000006410072674642500175360ustar 00000000000000#foo. try : bar() finally : pass try : pass # foo except lol as LOL : pass except : # foo pass else : pass finally : foo try: pass except: pass finally: pass try: # 1 try: # 2 pass # 3 # 4 finally: # 5 pass # 6 # 7 except foo: #8 pass #9 libcst-0.1.0/tests/fixtures/trailing_comment_without_nl.py000064400000000000000000000000170072674642500223240ustar 00000000000000 # hehehe >:)libcst-0.1.0/tests/fixtures/trailing_whitespace.py000064400000000000000000000000340072674642500205410ustar 00000000000000 x = 42 print(x) libcst-0.1.0/tests/fixtures/tuple_shenanigans.py000064400000000000000000000006070072674642500202310ustar 00000000000000(1, 2) (1, 2, 3) # alright here we go. () (()) (((())), ()) ( # evil >:) # evil >:( ) # ... (1,) ( * 1 , * 2 ,) *_ = (l,) () = x ( ) = ( x, ) (x) = (x) ( x , ) = x ( x , *y , * z , ) = l ( x , *y , * z , ) = ( x , *y , * z , ) = ( x , *y , * z , x ) ( x , # :) bar, * baz , ) =\ ( (let, *s, ( ) ) , nest , them , ( * t , * u , * p , l , * e , s , ) )libcst-0.1.0/tests/fixtures/type_parameters.py000064400000000000000000000024730072674642500177310ustar 00000000000000# fmt: off type TA = int type TA1[A] = lambda A: A class Outer[A]: type TA1[A] = None type TA1[A, B] = dict[A, B] class Outer[A]: def inner[B](self): type TA1[C] = TA1[A, B] | int return TA1 def more_generic[T, *Ts, **P](): type TA[T2, *Ts2, **P2] = tuple[Callable[P, tuple[T, *Ts]], Callable[P2, tuple[T2, *Ts2]]] return TA type Recursive = Recursive def func[A](A): return A class ClassA: def func[__A](self, __A): return __A class ClassA[A, B](dict[A, B]): ... class ClassA[A]: def funcB[B](self): class ClassC[C]: def funcD[D](self): return lambda: (A, B, C, D) return ClassC class Child[T](Base[lambda: (int, outer_var, T)]): ... type Alias[T: ([T for T in (T, [1])[1]], T)] = [T for T in T.__name__] type Alias[T: [lambda: T for T in (T, [1])[1]]] = [lambda: T for T in T.__name__] class Foo[T: Foo, U: (Foo, Foo)]: pass def func[T](a: T = "a", *, b: T = "b"): return (a, b) def func1[A: str, B: str | int, C: (int, str)](): return (A, B, C) type A [ T , * V ] =foo;type B=A def AAAAAAAAAAAAAAAAAA [ T : int ,*Ts , ** TT ] ():pass class AAAAAAAAAAAAAAAAAA [ T : int ,*Ts , ** TT ] :pass def yikes[A:int,*B,**C](*d:*tuple[A,*B,...])->A:passlibcst-0.1.0/tests/fixtures/vast_emptiness.py000064400000000000000000000000000072674642500175510ustar 00000000000000libcst-0.1.0/tests/fixtures/with_wickedness.py000064400000000000000000000012600072674642500177100ustar 00000000000000# with_wickedness with foo : pass with foo, bar: pass with (foo, bar): pass with (foo, bar,): pass with foo, bar as bar: pass with (foo, bar as bar): pass with (foo, bar as bar,): pass async def f(): async with foo: with bar: pass async with foo : pass async with foo, bar: pass async with (foo, bar): pass async with (foo, bar,): pass async with foo, bar as bar: pass async with (foo, bar as bar): pass async with (foo, bar as bar,): pass async with foo(1+1) as bar , 1 as (a, b, ) , 2 as [a, b] , 3 as a[b] : pass libcst-0.1.0/tests/fixtures/wonky_walrus.py000064400000000000000000000003120072674642500172570ustar 00000000000000( foo := 5 ) any((lastNum := num) == 1 for num in [1, 2, 3]) [(lastNum := num) == 1 for num in [1, 2, 3]] while f := x(): pass if f := x(): pass f(y:=1) f(x, y := 1 ) _[_:=10]libcst-0.1.0/tests/parser_roundtrip.rs000064400000000000000000000031510072674642500162440ustar 00000000000000// Copyright (c) Meta Platforms, Inc. and affiliates. // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree use difference::assert_diff; use itertools::Itertools; use libcst_native::{parse_module, prettify_error, Codegen}; use std::{ iter::once, path::{Component, PathBuf}, }; fn all_fixtures() -> impl Iterator { let mut path = PathBuf::from(file!()); path.pop(); path = path .components() .skip(1) .chain(once(Component::Normal("fixtures".as_ref()))) .collect(); path.read_dir().expect("read_dir").into_iter().map(|file| { let path = file.unwrap().path(); let contents = std::fs::read_to_string(&path).expect("reading file"); (path, contents) }) } #[test] fn roundtrip_fixtures() { for (path, input) in all_fixtures() { let input = if let Some(stripped) = input.strip_prefix('\u{feff}') { stripped } else { &input }; let m = match parse_module(input, None) { Ok(m) => m, Err(e) => panic!("{}", prettify_error(e, format!("{:#?}", path).as_ref())), }; let mut state = Default::default(); m.codegen(&mut state); let generated = state.to_string(); if generated != input { let got = visualize(&generated); let expected = visualize(input); assert_diff!(expected.as_ref(), got.as_ref(), "", 0); } } } fn visualize(s: &str) -> String { s.replace(' ', "▩").lines().join("↩\n") }