RcppEigen/0000755000176200001440000000000014662276752012147 5ustar liggesusersRcppEigen/tests/0000755000176200001440000000000013557014221013270 5ustar liggesusersRcppEigen/tests/tinytest.R0000644000176200001440000000103413557014221015274 0ustar liggesusers if (requireNamespace("tinytest", quietly=TRUE) && utils::packageVersion("tinytest") >= "1.0.0") { ## Set a seed to make the test deterministic set.seed(42) ## R makes us to this Sys.setenv("R_TESTS"="") ## there are several more granular ways to test files in a tinytest directory, ## see its package vignette; tests can also run once the package is installed ## using the same command `test_package(pkgName)`, or by director or file tinytest::test_package("RcppEigen", ncpu=getOption("Ncpus", 1)) } RcppEigen/MD50000644000176200001440000014120014662276752012455 0ustar liggesusersb21f1f37023e730c64fdb412e5452690 *ChangeLog 94e025a8c899609318bbcbf1ccd0ac61 *DESCRIPTION e42bf18a51bf55ebd814dbec277ee8a9 *LICENSE 742cc37f7265d5c5769876355e6dd8a8 *NAMESPACE 309b1ccf1c3c1dbd6cef99df62730204 *R/RcppEigen.package.skeleton.R e7f01094df303fd863645afc66135e22 *R/RcppExports.R c7aef4bfa80277e3de4994e273195d67 *R/SHLIB.R e1fac714cf161eb0e1ff6a2c8b7806d9 *R/fastLm.R 5e9a26fc37e3d6e32effd1906837a58f *R/flags.R cdd004c471ea6b4571a36553715a1310 *R/inline.R f3aac587d990551b32041ecd549adc58 *README.md 80793f039326afcc89dc7b1997439886 *build/vignette.rds 122124e5c83aa22c1a1edd530dfe89d5 *cleanup 85ae8c4845d02f52b357eb46770fa686 *inst/CITATION 8b394fa87a687b21a763754e0891b7af *inst/COPYRIGHTS 538d260aec3afdb31a0e37c243984ee1 *inst/NEWS.Rd be6e7b61cfe053492c3fd78dbb427f3b *inst/doc/RcppEigen-Introduction.R c52f521406b904dee163f3286ba491e8 *inst/doc/RcppEigen-Introduction.Rnw 0353763433d0540acc0ecdd5311c62c4 *inst/doc/RcppEigen-Introduction.pdf a3f7e1e72bbe54ef5e0acaf6e0a90521 *inst/examples/lmBenchmark.R 145af623d4b14c143f728edefb421c90 *inst/include/Eigen/Cholesky f07ba76748be1d30bf6b5f780346ddc9 *inst/include/Eigen/CholmodSupport 81296a0f304f1f4c2d7683c0da4f41e8 *inst/include/Eigen/Core d83e7f96733d79a7a1a38630c4287e85 *inst/include/Eigen/Dense 97913f1cc1bc6793f4d308537075563a *inst/include/Eigen/Eigen 4a101a6893eb59648e608fb780793df4 *inst/include/Eigen/Eigenvalues 16b99e5b068f30d75b0a40ea3a51671c *inst/include/Eigen/Geometry 21f7d4b876b5e53d591f85566a1fc2a4 *inst/include/Eigen/Householder aaeeeb9a7377155d7ff520d71426c8d0 *inst/include/Eigen/IterativeLinearSolvers e57d3f4e9d7cba9138197a2a327d7290 *inst/include/Eigen/Jacobi 773905229dd2c59b07c0195014b15aad *inst/include/Eigen/KLUSupport a79c9abe1514e57f05658202a6e713fc *inst/include/Eigen/LU 003fe3d1e91833e14d42c3f51f540f55 *inst/include/Eigen/MetisSupport 8603fdfa8648b85a1799a7620b66eadd *inst/include/Eigen/OrderingMethods 6cea51382aa721a45cf1a62312072001 *inst/include/Eigen/PaStiXSupport 61734ff861c3a3996dcb7f397d00e7e1 *inst/include/Eigen/PardisoSupport fd275dac6ea9537884730a8c73387602 *inst/include/Eigen/QR 3ec0f5af1ec7173a413270ecfa454a62 *inst/include/Eigen/QtAlignedMalloc 0eecd23b6d4b5153e15ecbfd55fb802f *inst/include/Eigen/SPQRSupport 49589ccc9db3334105e1399803dba5df *inst/include/Eigen/SVD 76a6f277df269701446d49bc9b31f1e4 *inst/include/Eigen/Sparse 1e32650635cb6297b4578d30edddc59d *inst/include/Eigen/SparseCholesky 9e4037b36be07c843c2e58ba233a97a9 *inst/include/Eigen/SparseCore de0cc8dd6ed700c53b839b5e1f39b1c0 *inst/include/Eigen/SparseLU 52150dd6e9daa40ac034086650733e30 *inst/include/Eigen/SparseQR 4cce81671567f95d091ba804dabe5dea *inst/include/Eigen/StdDeque 5532886591f4b7af430ba6a959570f50 *inst/include/Eigen/StdList 8427ab22ff8235eaca7414fe91c28679 *inst/include/Eigen/StdVector e9debf276c69bf6c2069ca30c17aeea7 *inst/include/Eigen/SuperLUSupport 9dd93af950d3f4d6874967fbd71a7b33 *inst/include/Eigen/UmfPackSupport 8c58b327902c449a4c5b766f928d415d *inst/include/Eigen/src/Cholesky/LDLT.h c2ca41302cfd99797ab636105d4e4998 *inst/include/Eigen/src/Cholesky/LLT.h f59c7074c241180a9135878387652c64 *inst/include/Eigen/src/Cholesky/LLT_LAPACKE.h 75647444567fec25b0aa91ee708980d7 *inst/include/Eigen/src/CholmodSupport/CholmodSupport.h 3faa766a3f8465ee3f97763276ad3de0 *inst/include/Eigen/src/Core/ArithmeticSequence.h 1afdafdd1125921f96e01f55bfcce5f8 *inst/include/Eigen/src/Core/Array.h 73aeac050eca259eb57753f8a3d4dbdb *inst/include/Eigen/src/Core/ArrayBase.h fdccb4e49e0b7ab7e6ca96218b390573 *inst/include/Eigen/src/Core/ArrayWrapper.h ee87e13d58d537915bd0cbc466e59425 *inst/include/Eigen/src/Core/Assign.h 6f3db3cd818698c2c19e7334a25bad68 *inst/include/Eigen/src/Core/AssignEvaluator.h c67b385de56849e26fe94d7118fdbdce *inst/include/Eigen/src/Core/Assign_MKL.h 9951bfbd0ebb7388ff86ea488599a577 *inst/include/Eigen/src/Core/BandMatrix.h ca30591356b64fbcb37f72e4e0f4c8c0 *inst/include/Eigen/src/Core/Block.h ce3feecb621af9961e12507b93c78f64 *inst/include/Eigen/src/Core/BooleanRedux.h f12f1b11b61167132d75acdb4604fc85 *inst/include/Eigen/src/Core/CommaInitializer.h f8cd1f21019da8a5de7094b01055b5e2 *inst/include/Eigen/src/Core/ConditionEstimator.h cd264602b6e6d96fa79458364039e4dc *inst/include/Eigen/src/Core/CoreEvaluators.h 2170d1468014c409a911a8d75b08e101 *inst/include/Eigen/src/Core/CoreIterators.h ee3aaa93b6dafabb21b32970eda2c92b *inst/include/Eigen/src/Core/CwiseBinaryOp.h a61a85b707c1bfd91f7ec2aa73353eb9 *inst/include/Eigen/src/Core/CwiseNullaryOp.h d1b300f2db147c44e8efd3965b3e236a *inst/include/Eigen/src/Core/CwiseTernaryOp.h 7d3e4cdd6611853e42fcf7416414e677 *inst/include/Eigen/src/Core/CwiseUnaryOp.h 0a822e911e015093ae5e261a0b067e57 *inst/include/Eigen/src/Core/CwiseUnaryView.h e8eb1ff37805d18d6a26e4f821fe4099 *inst/include/Eigen/src/Core/DenseBase.h 6ce38277d5f5557b2e91d1f5059478a0 *inst/include/Eigen/src/Core/DenseCoeffsBase.h 332c4beb28938a85892ca249f9f71682 *inst/include/Eigen/src/Core/DenseStorage.h 4269cd5446dd3e1b75a7c1da1302b27c *inst/include/Eigen/src/Core/Diagonal.h d6df22ac21add88b37ce586795ddee59 *inst/include/Eigen/src/Core/DiagonalMatrix.h c98bc66c82e201fdf0d224b4b485454d *inst/include/Eigen/src/Core/DiagonalProduct.h bf80acc97cf60b9cf870554000522591 *inst/include/Eigen/src/Core/Dot.h b4c55dd945692ce504da3801a033e7d2 *inst/include/Eigen/src/Core/EigenBase.h 1991064bf2bfc62c08f74a77f7341488 *inst/include/Eigen/src/Core/ForceAlignedAccess.h aff434c5fb66db0698b41899c19c3cac *inst/include/Eigen/src/Core/Fuzzy.h 0f8d025a52ad45a6d95bad802bb1dc3c *inst/include/Eigen/src/Core/GeneralProduct.h 59e500a7ece6bb21359ee4480405f532 *inst/include/Eigen/src/Core/GenericPacketMath.h 179f3d915354a047342692cb3e5f045a *inst/include/Eigen/src/Core/GlobalFunctions.h a7c663a0183539b201679507000b5d71 *inst/include/Eigen/src/Core/IO.h dc24fe7822e71daf6606b8634d02d769 *inst/include/Eigen/src/Core/IndexedView.h d383b45610f5be5fe97085f7c94925c8 *inst/include/Eigen/src/Core/Inverse.h de260d365e311d39526b98cac59db29f *inst/include/Eigen/src/Core/Map.h cba37023ad729001bdc1f44aee8ddaf4 *inst/include/Eigen/src/Core/MapBase.h ad2bd2b246eed7dd0cc6afe0ef9caf80 *inst/include/Eigen/src/Core/MathFunctions.h 8e1afc569876c4a7c093e9c4c7078b44 *inst/include/Eigen/src/Core/MathFunctionsImpl.h 0622be8830768a011865cf92f28f815d *inst/include/Eigen/src/Core/Matrix.h 8c876a9c927aa3c0b86081d9ee1be2a6 *inst/include/Eigen/src/Core/MatrixBase.h 196d298b6e41f0cbe11615a7ae9c5c03 *inst/include/Eigen/src/Core/NestByValue.h 32bf81920e341e78e1a3a191f08d2911 *inst/include/Eigen/src/Core/NoAlias.h 4496eeb0b722906c6a7ae2fc2e6b52cd *inst/include/Eigen/src/Core/NumTraits.h 2e6f187c558a152de2ec80ba965f7952 *inst/include/Eigen/src/Core/PartialReduxEvaluator.h 0cd11cbf270eca32c38f912090b5f694 *inst/include/Eigen/src/Core/PermutationMatrix.h 69a4fcd14ddf17acbf51862df3a76248 *inst/include/Eigen/src/Core/PlainObjectBase.h 22564df60904e7991ebd0b8148a3b2cd *inst/include/Eigen/src/Core/Product.h 5c1e8be423fcfaf719cd0071bf0cada0 *inst/include/Eigen/src/Core/ProductEvaluators.h d98ef910498ce2862a36134065437180 *inst/include/Eigen/src/Core/Random.h 6805acca17aa2ee1a4415f99e1c5e2ad *inst/include/Eigen/src/Core/Redux.h 9fdd22aee2e1405dac11e7c89ef21ea2 *inst/include/Eigen/src/Core/Ref.h 1fe4a31f7d0ae07750f675fde8c50077 *inst/include/Eigen/src/Core/Replicate.h 666109da17a24240328399d3c83a52ad *inst/include/Eigen/src/Core/Reshaped.h 2aff31f4dd9ba727f663018b6fb64b15 *inst/include/Eigen/src/Core/ReturnByValue.h 6dd655897dd69f6dfe0c80a3f15d570d *inst/include/Eigen/src/Core/Reverse.h 380a770ebe58c88373bed39084352fa1 *inst/include/Eigen/src/Core/Select.h 92c84ac9f3d5f082087324f35fd43ca2 *inst/include/Eigen/src/Core/SelfAdjointView.h 897d2ce94b0417abf2123ce3733a8310 *inst/include/Eigen/src/Core/SelfCwiseBinaryOp.h a9138fe508e2c1d3ebde81ea10f346ff *inst/include/Eigen/src/Core/Solve.h 44fc8adb3344a7c66ec79740e6c73e42 *inst/include/Eigen/src/Core/SolveTriangular.h 1728669436f06d2217a1acbc25e0652d *inst/include/Eigen/src/Core/SolverBase.h 6ca07f4ff21a829a66d5f4961b563f41 *inst/include/Eigen/src/Core/StableNorm.h 8f84953a86a0c369dd30ab9ace48a4e4 *inst/include/Eigen/src/Core/StlIterators.h 434caa1c6d2cac678733bdcb56f87955 *inst/include/Eigen/src/Core/Stride.h 4e1eccdaee1bfcd6cc4a18fa83f79d07 *inst/include/Eigen/src/Core/Swap.h c5d3e32c95df0538050042ca5727e1c6 *inst/include/Eigen/src/Core/Transpose.h 7a707e09534882ef199ed88da23743c4 *inst/include/Eigen/src/Core/Transpositions.h 4b9d138e539834ecdddc8b86ce5eb3b2 *inst/include/Eigen/src/Core/TriangularMatrix.h c90e5017218d387169e1de632bafe989 *inst/include/Eigen/src/Core/VectorBlock.h c44a3977b15c671570a9c40f162c16ae *inst/include/Eigen/src/Core/VectorwiseOp.h e3949c304fd0f0b8d818f55d496592eb *inst/include/Eigen/src/Core/Visitor.h eccf708f1cc23919cba245c5f7b26730 *inst/include/Eigen/src/Core/arch/AVX/Complex.h bb62aaffbdaad0e79a35e1295649abeb *inst/include/Eigen/src/Core/arch/AVX/MathFunctions.h 1ef051a7272c6f962ddd46a06adf9b9a *inst/include/Eigen/src/Core/arch/AVX/PacketMath.h 26a07d215c4953e4f65df781c22ac6bd *inst/include/Eigen/src/Core/arch/AVX/TypeCasting.h d5f4d27595402694f2ad29417ccbf454 *inst/include/Eigen/src/Core/arch/AVX512/Complex.h 76adc7d1ac14e572b03170346f1b3a9e *inst/include/Eigen/src/Core/arch/AVX512/MathFunctions.h 3672c75c894e6d4972d01f6da6eca094 *inst/include/Eigen/src/Core/arch/AVX512/PacketMath.h 3b06eaf49c05a3674b06363001cb4ebf *inst/include/Eigen/src/Core/arch/AVX512/TypeCasting.h 57f01fb49d12aab575f3a1d9f9d27eef *inst/include/Eigen/src/Core/arch/AltiVec/Complex.h b1ff1c2373e425bbb5a1d74ca3603669 *inst/include/Eigen/src/Core/arch/AltiVec/MathFunctions.h 244915bdb5006bd4c323336b1f543954 *inst/include/Eigen/src/Core/arch/AltiVec/MatrixProduct.h 1ece1c8c380758be36bde88eefdf4d18 *inst/include/Eigen/src/Core/arch/AltiVec/MatrixProductCommon.h 2ffdf1cf62c4ef756f4ca2c5478ea916 *inst/include/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h 3f2b25fb5cc4cb83bc5287a78d8c7593 *inst/include/Eigen/src/Core/arch/AltiVec/PacketMath.h 48834752aafbf0c1eed78d591ea5ce43 *inst/include/Eigen/src/Core/arch/CUDA/Complex.h b125897b051b64f28a6ad4a24936d6b2 *inst/include/Eigen/src/Core/arch/Default/BFloat16.h e50eb2bf2d022bb3121deca2b32cecc8 *inst/include/Eigen/src/Core/arch/Default/ConjHelper.h 3973af46054bdab3588a455d07794c94 *inst/include/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h 0b153cd08b1e89c4872863687f915560 *inst/include/Eigen/src/Core/arch/Default/GenericPacketMathFunctionsFwd.h b768042c101bf7279ca54ccbe198ce74 *inst/include/Eigen/src/Core/arch/Default/Half.h 1f5048f878e6a99bb02a552b5d64acc6 *inst/include/Eigen/src/Core/arch/Default/Settings.h dbdb97c9e0db96cccc275a9e5d0cedaa *inst/include/Eigen/src/Core/arch/Default/TypeCasting.h 331f06b2a86d9a7e397638641b52f96e *inst/include/Eigen/src/Core/arch/GPU/MathFunctions.h 4a178bdb2141cf9df1e91482ebc62bfc *inst/include/Eigen/src/Core/arch/GPU/PacketMath.h 655f92633b865e13c43b3232d6dcbde1 *inst/include/Eigen/src/Core/arch/GPU/TypeCasting.h 465706ce6839f39d71ce56515627d14d *inst/include/Eigen/src/Core/arch/HIP/hcc/math_constants.h 837da7ecef71c2b87f92226aaa1ee0d6 *inst/include/Eigen/src/Core/arch/MSA/Complex.h 120f702e71eca10ac75ebb5995530a9f *inst/include/Eigen/src/Core/arch/MSA/MathFunctions.h 4ab06a761f8ac6a64ea71d700c859e7f *inst/include/Eigen/src/Core/arch/MSA/PacketMath.h 49542657bc09b2a1e702dec391a79aab *inst/include/Eigen/src/Core/arch/NEON/Complex.h 5f156a8c03925d154882b995722d6863 *inst/include/Eigen/src/Core/arch/NEON/GeneralBlockPanelKernel.h 272c6d3940697697b599a73cc2813533 *inst/include/Eigen/src/Core/arch/NEON/MathFunctions.h 906be4dd71b459ddbc93e1ab4a418c72 *inst/include/Eigen/src/Core/arch/NEON/PacketMath.h 67a19bf44a9e155ec745307c7a2ef537 *inst/include/Eigen/src/Core/arch/NEON/TypeCasting.h 8b516b2428b1cc51620837772ca4e06e *inst/include/Eigen/src/Core/arch/SSE/Complex.h ac0dfbd09583027880e951536f42e09b *inst/include/Eigen/src/Core/arch/SSE/MathFunctions.h 53fae523e6473b2b427ca3b06eff8588 *inst/include/Eigen/src/Core/arch/SSE/PacketMath.h 288defcacab8be5efb5de016099ce69c *inst/include/Eigen/src/Core/arch/SSE/TypeCasting.h 0abdec9e38825f2994a5b04cc658e050 *inst/include/Eigen/src/Core/arch/SVE/MathFunctions.h 3078718a350c99baa7266e53715076eb *inst/include/Eigen/src/Core/arch/SVE/PacketMath.h 73eab75c1c0a1c209bdf90b27b0d0367 *inst/include/Eigen/src/Core/arch/SVE/TypeCasting.h 9494c07dcfe0b91c52ff975314ef21db *inst/include/Eigen/src/Core/arch/SYCL/InteropHeaders.h 29d2c00fcc5424eeb836cecc258a06cc *inst/include/Eigen/src/Core/arch/SYCL/MathFunctions.h 55976f063f0bc6c03a9e4fcc47282fd1 *inst/include/Eigen/src/Core/arch/SYCL/PacketMath.h c9ae7e85fcb856c7a8d9f66cb2cce4fc *inst/include/Eigen/src/Core/arch/SYCL/SyclMemoryModel.h 2b42e5034c211881e453fdb5acabde12 *inst/include/Eigen/src/Core/arch/SYCL/TypeCasting.h 36c5dc8e86d6332dcc1cefa8224cdbf7 *inst/include/Eigen/src/Core/arch/ZVector/Complex.h 875f0c30c8ab7a39fbc03238929354ec *inst/include/Eigen/src/Core/arch/ZVector/MathFunctions.h 7bc05c9230c129ca8dda453d5f0a3170 *inst/include/Eigen/src/Core/arch/ZVector/PacketMath.h 3173559aeda799ea7110b0b395f020b4 *inst/include/Eigen/src/Core/functors/AssignmentFunctors.h 26fdd4b0e1f992a264af84429e92ad2d *inst/include/Eigen/src/Core/functors/BinaryFunctors.h 887548a06d7e5cc00d6874474eb80501 *inst/include/Eigen/src/Core/functors/NullaryFunctors.h 5707388b8989c2ef61207097280a4f6e *inst/include/Eigen/src/Core/functors/StlFunctors.h bcde622f6e8f3fa6a22feccb4907518f *inst/include/Eigen/src/Core/functors/TernaryFunctors.h 35eef13d14f9fb343215728840bc643a *inst/include/Eigen/src/Core/functors/UnaryFunctors.h 6f3af820c5dd67d16dfed3285be0fd57 *inst/include/Eigen/src/Core/products/GeneralBlockPanelKernel.h 462bfeb265eabe911224d5ec0506736d *inst/include/Eigen/src/Core/products/GeneralMatrixMatrix.h 65cebdb846d5c9af2d5bb38f1b0f2745 *inst/include/Eigen/src/Core/products/GeneralMatrixMatrixTriangular.h c05ed3f002a96a84639364e8b0163e92 *inst/include/Eigen/src/Core/products/GeneralMatrixMatrixTriangular_BLAS.h ba2f515e6a5b7b480d2b30fc6993668a *inst/include/Eigen/src/Core/products/GeneralMatrixMatrix_BLAS.h 517e666638d03f2c95a7a3cef9b623b8 *inst/include/Eigen/src/Core/products/GeneralMatrixVector.h cdf5c4721b372237ca45c6792d497fa4 *inst/include/Eigen/src/Core/products/GeneralMatrixVector_BLAS.h ef62740fc4b14576f4ddeecf7b89886e *inst/include/Eigen/src/Core/products/Parallelizer.h 48090a4f6107df8d103747624e7b4449 *inst/include/Eigen/src/Core/products/SelfadjointMatrixMatrix.h 4387b005442d80b4d6d6f8b97ed19a46 *inst/include/Eigen/src/Core/products/SelfadjointMatrixMatrix_BLAS.h f53417133b66fe42c4ed4f8fd04e84f8 *inst/include/Eigen/src/Core/products/SelfadjointMatrixVector.h b3c4de4cc9058b7d198c9573b1825112 *inst/include/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h 72ab2c1e9288bf204fa2c38277df19ec *inst/include/Eigen/src/Core/products/SelfadjointProduct.h 2f82a286df83e9d56a58d27d7a175241 *inst/include/Eigen/src/Core/products/SelfadjointRank2Update.h 2b64023bcb0d4ba24e0545ddf23c05ea *inst/include/Eigen/src/Core/products/TriangularMatrixMatrix.h 16750354416672507c557117b18adc91 *inst/include/Eigen/src/Core/products/TriangularMatrixMatrix_BLAS.h 40e9483b9c1cd7614cce6fe433cf37f5 *inst/include/Eigen/src/Core/products/TriangularMatrixVector.h 2b3c7391c827c88987c76f199d84c839 *inst/include/Eigen/src/Core/products/TriangularMatrixVector_BLAS.h d84f957ec602fd796856a227eaab8196 *inst/include/Eigen/src/Core/products/TriangularSolverMatrix.h 8b85110f876cb655ce064e1d73040a97 *inst/include/Eigen/src/Core/products/TriangularSolverMatrix_BLAS.h d9efe059a2f494eeb743dd3b5dce326d *inst/include/Eigen/src/Core/products/TriangularSolverVector.h 070a882fde6ab2a7ccf944fbd53c07ba *inst/include/Eigen/src/Core/util/BlasUtil.h 54c8950e010de8b8e2e6d976ff327e71 *inst/include/Eigen/src/Core/util/ConfigureVectorization.h 524daabe8b0ad6b04157d460b45dc71c *inst/include/Eigen/src/Core/util/Constants.h d653b434e6e09d354cf77f131d38ba1c *inst/include/Eigen/src/Core/util/DisableStupidWarnings.h 5321f87825811b679ba086d119636a79 *inst/include/Eigen/src/Core/util/ForwardDeclarations.h e8a432ad83dd1e332510fe10e2091b33 *inst/include/Eigen/src/Core/util/IndexedViewHelper.h 7680c8bd0ee18e19577b1b0c7380eda4 *inst/include/Eigen/src/Core/util/IntegralConstant.h 18aae1e5c1824fdd615691e36584195f *inst/include/Eigen/src/Core/util/MKL_support.h 35ca1e12bda85d53a1622684f452cb0f *inst/include/Eigen/src/Core/util/Macros.h 8a980fda9472eee25b88fe8bdf3f47df *inst/include/Eigen/src/Core/util/Memory.h 5091d09f469f1105e8e0b90a03f1e1f3 *inst/include/Eigen/src/Core/util/Meta.h c4ca2e16c302c71df98732240109e7e0 *inst/include/Eigen/src/Core/util/NonMPL2.h be45859b38e024d8c0823bf6e293f22d *inst/include/Eigen/src/Core/util/ReenableStupidWarnings.h 58b03232b4fa1cc3f2538e9cf6290f6c *inst/include/Eigen/src/Core/util/ReshapedHelper.h 3b585e1b5e5da798cafae55fc6d4cb26 *inst/include/Eigen/src/Core/util/StaticAssert.h 90b5ebe2dd38c421f22f0e4346220128 *inst/include/Eigen/src/Core/util/SymbolicIndex.h 32cd41fd1ed3bd3fd615f9ed56b0e1e8 *inst/include/Eigen/src/Core/util/XprHelper.h 30055c308d945119bc025b0811b0f83a *inst/include/Eigen/src/Eigenvalues/ComplexEigenSolver.h af23a6147de0154039ea815d3ed9ac81 *inst/include/Eigen/src/Eigenvalues/ComplexSchur.h 2555626edfca98529090a0997c336abf *inst/include/Eigen/src/Eigenvalues/ComplexSchur_LAPACKE.h baa22346cdf27a59648055aa445edbfe *inst/include/Eigen/src/Eigenvalues/EigenSolver.h b81925277584de764f8d31b0b02fc584 *inst/include/Eigen/src/Eigenvalues/GeneralizedEigenSolver.h b5b7fdfc0837ab65092a36e9861289e9 *inst/include/Eigen/src/Eigenvalues/GeneralizedSelfAdjointEigenSolver.h b29bd73a0fdb3acaa55b90057846bbd0 *inst/include/Eigen/src/Eigenvalues/HessenbergDecomposition.h 89ee9085ead2ae20e35b181408377f83 *inst/include/Eigen/src/Eigenvalues/MatrixBaseEigenvalues.h 544b0af45362ef8848955ff6e6cd4118 *inst/include/Eigen/src/Eigenvalues/RealQZ.h 5624e33cbaa28ee63c46dd7ae764ca22 *inst/include/Eigen/src/Eigenvalues/RealSchur.h 10c5af47ddd5e16ccb7a320dcd600492 *inst/include/Eigen/src/Eigenvalues/RealSchur_LAPACKE.h f5a29f0450d35f396c9001818a6ed431 *inst/include/Eigen/src/Eigenvalues/SelfAdjointEigenSolver.h eaca77e68a4289a23230708222d4d39c *inst/include/Eigen/src/Eigenvalues/SelfAdjointEigenSolver_LAPACKE.h f279721c98dc0136651ee180cdc83864 *inst/include/Eigen/src/Eigenvalues/Tridiagonalization.h df9023c12bca691bf96ccd1f6aa46787 *inst/include/Eigen/src/Geometry/AlignedBox.h a68f17dafd98a9e53d09c2f2cf9a26c7 *inst/include/Eigen/src/Geometry/AngleAxis.h a6a60dec0043515f0fb9e01708a500cb *inst/include/Eigen/src/Geometry/EulerAngles.h 0aeaf85793e7a6d1bec92e755a2228c3 *inst/include/Eigen/src/Geometry/Homogeneous.h 8f73fb2f90452392f75687a82c50d6d3 *inst/include/Eigen/src/Geometry/Hyperplane.h 385c034ba5489035f82962e9900604a0 *inst/include/Eigen/src/Geometry/OrthoMethods.h 6a89a934a1907e21785445aefb6676f4 *inst/include/Eigen/src/Geometry/ParametrizedLine.h 6167d1c4fe02187fce8711029b4300e5 *inst/include/Eigen/src/Geometry/Quaternion.h 9761fa57781c85a08a91d5add80591a7 *inst/include/Eigen/src/Geometry/Rotation2D.h ac623d0fac2a116d9c06b23c8980d0c7 *inst/include/Eigen/src/Geometry/RotationBase.h a2ca635a91c609b7cc04b9f93fb63557 *inst/include/Eigen/src/Geometry/Scaling.h 945a726ad581c2094e9c2df50a0b35b0 *inst/include/Eigen/src/Geometry/Transform.h 1d901c1f895f7deeb3064e2fa40797d9 *inst/include/Eigen/src/Geometry/Translation.h 96e84c0d16e8ab0b86a30a5dc0170a8d *inst/include/Eigen/src/Geometry/Umeyama.h 4a1990f7e43fc51601dc5b45d3588aa9 *inst/include/Eigen/src/Geometry/arch/Geometry_SIMD.h e616b6c79f8585000c341130304fb505 *inst/include/Eigen/src/Householder/BlockHouseholder.h eafe0af2a8a3f6d31c7a9e9289db091c *inst/include/Eigen/src/Householder/Householder.h 9964e4c6105e11ea30800379a9c9c42c *inst/include/Eigen/src/Householder/HouseholderSequence.h bcecbad4f12eadff9c4a8c9e38598238 *inst/include/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h e17a6edbfcdad1e6dc347a1f07ca8378 *inst/include/Eigen/src/IterativeLinearSolvers/BiCGSTAB.h 6d26e8ed13906e3ee69a8a194630d795 *inst/include/Eigen/src/IterativeLinearSolvers/ConjugateGradient.h b8cbc38f82fea3b5f847c4e0ab5275ea *inst/include/Eigen/src/IterativeLinearSolvers/IncompleteCholesky.h 750ce867811ca64e5c99a9969b42755a *inst/include/Eigen/src/IterativeLinearSolvers/IncompleteLUT.h 03227d1d680fb5e1d36105aca9ac2433 *inst/include/Eigen/src/IterativeLinearSolvers/IterativeSolverBase.h ef29a7509ad32bdfb424e93f1f27c2ac *inst/include/Eigen/src/IterativeLinearSolvers/LeastSquareConjugateGradient.h 4c6cebdd36c8399f03fea7d9fedb268a *inst/include/Eigen/src/IterativeLinearSolvers/SolveWithGuess.h e6e33db6c0e9404873f5c4f0cf35affa *inst/include/Eigen/src/Jacobi/Jacobi.h 03f84ae4be7cf89985c79fb0bcc05fbe *inst/include/Eigen/src/KLUSupport/KLUSupport.h 7ffff43e112f6635eb0c455cf39822f1 *inst/include/Eigen/src/LU/Determinant.h 5bd14d969773d1d48178b80d986b2872 *inst/include/Eigen/src/LU/FullPivLU.h 3f2fc7624ca164d4f6884e300e35e37b *inst/include/Eigen/src/LU/InverseImpl.h 2db64e29d2ab949f9e91ee931050d96d *inst/include/Eigen/src/LU/PartialPivLU.h 9e5f31822442bfdb277a0f78b183306f *inst/include/Eigen/src/LU/PartialPivLU_LAPACKE.h c90df35abb13baa4d77cbc2d960b64ab *inst/include/Eigen/src/LU/arch/InverseSize4.h 5dbb0113de10441d8d1f996fb9cd085e *inst/include/Eigen/src/MetisSupport/MetisSupport.h 8de296e44450afd05c27d56e1b9601fd *inst/include/Eigen/src/OrderingMethods/Amd.h 665b2e5eddcb2cc874b9ebc373899d91 *inst/include/Eigen/src/OrderingMethods/Eigen_Colamd.h d72f08e34d076a79587d7ec20dcccb6d *inst/include/Eigen/src/OrderingMethods/Ordering.h 88e3776a970754fa5cc7f38160bd725a *inst/include/Eigen/src/PaStiXSupport/PaStiXSupport.h eccb80bd7c73941aa0ce3bf85736db46 *inst/include/Eigen/src/PardisoSupport/PardisoSupport.h 01312cbe66424f2377dc45fecb0b155b *inst/include/Eigen/src/QR/ColPivHouseholderQR.h 5549163935d111d6ea09c19bb848b696 *inst/include/Eigen/src/QR/ColPivHouseholderQR_LAPACKE.h c2bc6c679aa91f0d0e0e926bea74461c *inst/include/Eigen/src/QR/CompleteOrthogonalDecomposition.h 39d8d91a9197583b3bd08b9f907b66b9 *inst/include/Eigen/src/QR/FullPivHouseholderQR.h 5ee87087db6ee3b64ac05b54424e38c0 *inst/include/Eigen/src/QR/HouseholderQR.h 5a6fb56c5132fd49451bd9c2da752833 *inst/include/Eigen/src/QR/HouseholderQR_LAPACKE.h fca07660088d80bae414808ff49e8553 *inst/include/Eigen/src/SPQRSupport/SuiteSparseQRSupport.h 1f1de7a316f6deefc12ef268353c33a1 *inst/include/Eigen/src/SVD/BDCSVD.h 87c2dc197df35a2d43d76e5a11bbd967 *inst/include/Eigen/src/SVD/JacobiSVD.h 80266065e69cd87a54ab72cf61fd8c75 *inst/include/Eigen/src/SVD/JacobiSVD_LAPACKE.h fd337c9be2444b6da9d69dcaffbb7b14 *inst/include/Eigen/src/SVD/SVDBase.h 30588fee4028392fada31352f75aadfc *inst/include/Eigen/src/SVD/UpperBidiagonalization.h ee59e519e968229b1ccbabf532ca2cac *inst/include/Eigen/src/SparseCholesky/SimplicialCholesky.h c1876efaf959f6a7f66d0bf8bd572eee *inst/include/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h c2a3e3e4dca66c99172c172cafcbeffc *inst/include/Eigen/src/SparseCore/AmbiVector.h f0b3947776d1f81fceb142e77f4cc96a *inst/include/Eigen/src/SparseCore/CompressedStorage.h eb0e2921618f7a0d03df1b8e56cfd6e9 *inst/include/Eigen/src/SparseCore/ConservativeSparseSparseProduct.h 0884d5b081e7f51dc171dd1800202384 *inst/include/Eigen/src/SparseCore/MappedSparseMatrix.h c97932af3160f16b0952d0a0b064e73a *inst/include/Eigen/src/SparseCore/SparseAssign.h d2517b9b4359dffcb5d72ad1f90e36b1 *inst/include/Eigen/src/SparseCore/SparseBlock.h b2f2001a8c8a897cffd4bdab6a633af0 *inst/include/Eigen/src/SparseCore/SparseColEtree.h 4f9df1f9f74d642abcae0fefb9d80c33 *inst/include/Eigen/src/SparseCore/SparseCompressedBase.h d02bdcfbacec3837b3b38702e6ee23ad *inst/include/Eigen/src/SparseCore/SparseCwiseBinaryOp.h 50b39b36aa74a4e8bb575769714a782d *inst/include/Eigen/src/SparseCore/SparseCwiseUnaryOp.h 51daae3ce550d9acbf4ac00a53c5f41a *inst/include/Eigen/src/SparseCore/SparseDenseProduct.h a5b0bea9c6fba7de90d0d9d13ccc1ba8 *inst/include/Eigen/src/SparseCore/SparseDiagonalProduct.h 70c41e25cddfc6f434c3b538f1408107 *inst/include/Eigen/src/SparseCore/SparseDot.h 3286f132a4926940c987390f47be3d7a *inst/include/Eigen/src/SparseCore/SparseFuzzy.h df51c9241f0b09618e91429df9152562 *inst/include/Eigen/src/SparseCore/SparseMap.h 44a3e99834d97901640edf78f7a2dd27 *inst/include/Eigen/src/SparseCore/SparseMatrix.h b969205a8ee249ff6630ed45bf98ab73 *inst/include/Eigen/src/SparseCore/SparseMatrixBase.h fbc26e89dfb89c64081ce2ce645d399c *inst/include/Eigen/src/SparseCore/SparsePermutation.h c9f504ba9d93a52833f8d33e499d4de0 *inst/include/Eigen/src/SparseCore/SparseProduct.h a158561b3c68f135e55a18df9a07e52c *inst/include/Eigen/src/SparseCore/SparseRedux.h cdc958c4134408cfba40648faa92b900 *inst/include/Eigen/src/SparseCore/SparseRef.h 97f7b4a6934f9c75bf782f4c849d7cc3 *inst/include/Eigen/src/SparseCore/SparseSelfAdjointView.h c97f6e2a1e4958ad9e0be90050bcdcfc *inst/include/Eigen/src/SparseCore/SparseSolverBase.h a755b2ac228e2eb0a0fa97773100df3f *inst/include/Eigen/src/SparseCore/SparseSparseProductWithPruning.h f79edcfc05ae044bd9c3ff584b0a8951 *inst/include/Eigen/src/SparseCore/SparseTranspose.h b0cd41339687152b78311edce045fece *inst/include/Eigen/src/SparseCore/SparseTriangularView.h 21ef1846ac0f10b18eb5b25a7853c996 *inst/include/Eigen/src/SparseCore/SparseUtil.h fa560763cd7a5bb20d5878698a4c6243 *inst/include/Eigen/src/SparseCore/SparseVector.h 2bd75163656dd29db5bab551f6890913 *inst/include/Eigen/src/SparseCore/SparseView.h 1ba1ec01dcd995c58e4b4c0f08c80c97 *inst/include/Eigen/src/SparseCore/TriangularSolver.h 0e066407863da1f2b319925190dbcf39 *inst/include/Eigen/src/SparseLU/SparseLU.h cbfff2f704bcf8ca688daf09ff50947c *inst/include/Eigen/src/SparseLU/SparseLUImpl.h 6d0afb3241a3d73846d164c342954b29 *inst/include/Eigen/src/SparseLU/SparseLU_Memory.h 3d4309877f3eb4325af2643aab1e23a9 *inst/include/Eigen/src/SparseLU/SparseLU_Structs.h 706ae5c3d9293e71ab54729ce6d998d8 *inst/include/Eigen/src/SparseLU/SparseLU_SupernodalMatrix.h 8aaa285321fd675fea2b4d31754e0f95 *inst/include/Eigen/src/SparseLU/SparseLU_Utils.h 5dff2c3c59cb15e6df2da1b0e66145a7 *inst/include/Eigen/src/SparseLU/SparseLU_column_bmod.h 30dab09f618e660469bfa3ee64adad41 *inst/include/Eigen/src/SparseLU/SparseLU_column_dfs.h 551299f37e2ee9640ff95769358f79b2 *inst/include/Eigen/src/SparseLU/SparseLU_copy_to_ucol.h 594d09438b2bada027b73e9bd734944a *inst/include/Eigen/src/SparseLU/SparseLU_gemm_kernel.h e3c46328e66d4f410da2474155a9874f *inst/include/Eigen/src/SparseLU/SparseLU_heap_relax_snode.h c1e457785aca12c411e4d4f1604982fb *inst/include/Eigen/src/SparseLU/SparseLU_kernel_bmod.h 28a13019f675d1221e1743af48382a10 *inst/include/Eigen/src/SparseLU/SparseLU_panel_bmod.h 76b774de3134b8136b8b869f2c18a9b5 *inst/include/Eigen/src/SparseLU/SparseLU_panel_dfs.h e40b8ca24a6f9a1cd81995818acff05a *inst/include/Eigen/src/SparseLU/SparseLU_pivotL.h 98a63ced95b43ba2ef87d654555d6a11 *inst/include/Eigen/src/SparseLU/SparseLU_pruneL.h cfba7ff37fee21d1a94d8b5e86d06be0 *inst/include/Eigen/src/SparseLU/SparseLU_relax_snode.h 37c8bddfc7afd91acfb2377195237725 *inst/include/Eigen/src/SparseQR/SparseQR.h 3ed7e7cc49cc30d11ddffe2140c35f11 *inst/include/Eigen/src/StlSupport/StdDeque.h 5f37a7a9ca8db822562aee07b9f0e91a *inst/include/Eigen/src/StlSupport/StdList.h a18c9208acd9db9ecc9643eebe051c74 *inst/include/Eigen/src/StlSupport/StdVector.h 4ca2b8ce970639b16e3607774bd0e0c1 *inst/include/Eigen/src/StlSupport/details.h 64f57bfac851f3a8f748a0283abc2688 *inst/include/Eigen/src/SuperLUSupport/SuperLUSupport.h 883329bb82305c48528a1084a43ff399 *inst/include/Eigen/src/UmfPackSupport/UmfPackSupport.h c850487641ece48cb76f0ed6eb162b55 *inst/include/Eigen/src/misc/Image.h 204aec7b6cbd46a40fecb67549751f32 *inst/include/Eigen/src/misc/Kernel.h 62b9105c4ceb13dd8f7747adb6019a3a *inst/include/Eigen/src/misc/RealSvd2x2.h 62b34544558e47616d613f96b05b2e5b *inst/include/Eigen/src/misc/blas.h 6b21f68118e06a562cf64f3e385d5e0d *inst/include/Eigen/src/misc/lapack.h a048ab6772f93f28d18bfef8f98d0f7d *inst/include/Eigen/src/misc/lapacke.h 78a253bca660904aae9b45474fbc88bb *inst/include/Eigen/src/misc/lapacke_mangling.h 42429baaf47ae7b289ebe6889baefaa6 *inst/include/Eigen/src/plugins/ArrayCwiseBinaryOps.h 6deec1ad021076ea6ab50d8455f9305e *inst/include/Eigen/src/plugins/ArrayCwiseUnaryOps.h 85e04e86e084a6ab310701e08ef738db *inst/include/Eigen/src/plugins/BlockMethods.h 0a4e2987ce97a2988f02e202d8d721dd *inst/include/Eigen/src/plugins/CommonCwiseBinaryOps.h 31b1395d3465cb91f22f4abd59e10ff2 *inst/include/Eigen/src/plugins/CommonCwiseUnaryOps.h 04321efbca30d74a62e9f793ce76ee35 *inst/include/Eigen/src/plugins/IndexedViewMethods.h 57a3069a059090096d36532f1c7f5be1 *inst/include/Eigen/src/plugins/MatrixCwiseBinaryOps.h c2c96628d2be989f2809f8d90919c483 *inst/include/Eigen/src/plugins/MatrixCwiseUnaryOps.h 029331e21a26ca36f285500c6e7c0d2c *inst/include/Eigen/src/plugins/ReshapedMethods.h ce0bc155c278e78c32ae2ba83875dc8c *inst/include/RcppEigen.h 867e96baa1693665406ff7722386f2c1 *inst/include/RcppEigenCholmod.h af3eafe4bdce7a2a731af54f076d78b1 *inst/include/RcppEigenForward.h 2ff59c7f138b4e564e77c59f140761ca *inst/include/RcppEigenStubs.cpp 6fe9907c0c098413251d6aa4d0331d3e *inst/include/RcppEigenWrap.h 24ded1f60f4ce0ed3818f66cadf5adc8 *inst/include/unsupported/Eigen/AdolcForward e99e523bbb6c2dc85615f392a568627f *inst/include/unsupported/Eigen/AlignedVector3 e961d90876b34128336eb25e9344705a *inst/include/unsupported/Eigen/ArpackSupport f9b4acb68a0603f67222c3e48cd64b57 *inst/include/unsupported/Eigen/AutoDiff 2ab4f5b6ca6547bded20a5b7859e56f1 *inst/include/unsupported/Eigen/BVH 9898870dac5a80296ac469ea9ebfd3cb *inst/include/unsupported/Eigen/CXX11/Tensor 2e55a9ee72ff54c6b84b86d9c456a7ce *inst/include/unsupported/Eigen/CXX11/TensorSymmetry c8ca91a3a4623e4ef3d2770631826e61 *inst/include/unsupported/Eigen/CXX11/ThreadPool 72834521c8c429d3e59be510cef34c13 *inst/include/unsupported/Eigen/CXX11/src/Tensor/README.md 2ad2419cbab9f2db679d5c0e286ae4e4 *inst/include/unsupported/Eigen/CXX11/src/Tensor/Tensor.h b20dd2bcb79295ca469d46a6f8eae872 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorArgMax.h e9f80fbef1898da59f0dc9f8af0ae4b5 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorAssign.h 50fb858dd166f85773bc84eaf4ce2a8d *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorBase.h 341d36c4e71b4cbaf70de16d11127738 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorBlock.h d22ed87f0d91893e4498a7a67bf5eb9e *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorBroadcasting.h 3059c9c6732e156926c22f1d42018cab *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorChipping.h ebe0fb5ae5f6116dd46e1786cc973f01 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorConcatenation.h baad1e7051fc5ff4d558a0273c860a43 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorContraction.h 9605c70ee69b8fe6dc9eee591ee34a77 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorContractionBlocking.h d8a47e6835de083022a09cb80b5ecf7c *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorContractionCuda.h c0981d2f7c139348facd4b84b4acbfb7 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorContractionGpu.h 1b50fa39db16999e88f4a42177082c75 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorContractionMapper.h c7a184b1ff0c2eea46b5ee4ba3a72570 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorContractionSycl.h f55836eb433eb9c52770af5b1e28d126 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h d9e9a66f6b5a3f582a584a2cbba463fb *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorConversion.h c43f7bf1924a1e49c73e67254d1e62ad *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorConvolution.h 92551b80060266683d82d9eaedca0a63 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorConvolutionSycl.h b84589099123afb4e557456325dc1876 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorCostModel.h fad761ed911a3cadd3b0d6f56be3ed2b *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorCustomOp.h 59a83feb1475732537ea3a14999e5b13 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorDevice.h e9dab2e49bec95a42e5de9fd439465bf *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceCuda.h 753de1d90165c4d2e20bc941a2b2b9a8 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceDefault.h b35cb44ff9d1391af5639974ba177d0b *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceGpu.h be0fc410f8644f0a958c4fb38ca4e10c *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceSycl.h cd1265f55d196f3752f2e2d1d294bd03 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h 857b647f53809b09ad1c899a708d36b5 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorDimensionList.h 49b0a87a14f3f7b2133bef53a9c3b228 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorDimensions.h 59d5e9d341db08f4c239eab7c17e9612 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorEvalTo.h aa7891f16542853daa328070347199a6 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorEvaluator.h bf10253eca39f1226e1f870716af5881 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorExecutor.h b0d35804998980bbcae849f460908c6c *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorExpr.h fecc8bfce4334f9ed5ec3f074b58b869 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorFFT.h 5afe7b5622e1a44a0114854b07af2ed7 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorFixedSize.h acdb14779e7fe7d2554fa4eb5128e53a *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorForcedEval.h 0f1b2555a9eb2f112f02e7368a715273 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorForwardDeclarations.h b2a3f7c4b2b5c25a3f25fe36bb7a893e *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorFunctors.h ce1a927a7639b74066444665c31471a0 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorGenerator.h 54d3a782295491a6eccbe0eeb372ea7b *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorGlobalFunctions.h 283c86d12a861d4bf771bbacbd976044 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h f683315262a2d02680801567ad4c4922 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaUndefines.h 758cdf0ded97dde946991d7defb652c4 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorIO.h 6acb57fbae8fcf48c79dea5ace827b87 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorImagePatch.h 34d78d2dbd9e9d21673f1ca14bfc53f2 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorIndexList.h 33bd3d5b7968992c565992a8e544be35 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorInflation.h cb90de121809bc1942168c1ceec0d27f *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorInitializer.h aa2e92edf6e9bed3c9c9b1e2bf5b1d2b *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorIntDiv.h 2c5c07bb910c8da488cf23017cfa59d4 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorLayoutSwap.h 2d517757ba82dc213b90aa035870983b *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorMacros.h 5ccfa8a1d9c554c164ba1ea19b5bc650 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorMap.h e63919a1c6e5e84190d020c01751a67c *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorMeta.h 3305a0b609ae2c46755182fbb1fa1caf *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorMorphing.h 5694ccb891d11c4bd3be9e6df6e69f03 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorPadding.h 29d15195b83f13e267daa115d1977648 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorPatch.h 2b43d06e57e732602ca8335da5331262 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorRandom.h cb63b27f17492e5b08679304b392597c *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorReduction.h 4c282ab2f0237c96f1976535b6d22852 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorReductionCuda.h d0400712aaeede05b108a1f7ee67a63a *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h fb4e175446163f5df3d821979ca27f32 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorReductionSycl.h 1b2df14e0f7ff8c8a2f28a782c60ebc8 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorRef.h fda02c0b40acef943a0f9257e195a4cd *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorReverse.h 8bb7e61b5da97fe6e65f72cdf76b70cb *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorScan.h 72dc6d3c8694b68013ba6e19c2dec369 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorScanSycl.h d43b96ac8a1744dd1fae1b930fd2c135 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorShuffling.h c2eab785c9222f6a955377f8df858254 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorStorage.h 40a277a774380ff50e9058a69e4afca8 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorStriding.h 9c5ebca86272b2c3a05fb7153e4c023f *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorTrace.h 34d4213eb0a18e72b582286583f6e537 *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorTraits.h db02acae1da6d2d06c666d6c22b4740f *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorUInt128.h ccc96193cbb99ac0cdf13e9e36464b1a *inst/include/unsupported/Eigen/CXX11/src/Tensor/TensorVolumePatch.h 6f54705b485291c4673e3a00071422c2 *inst/include/unsupported/Eigen/CXX11/src/TensorSymmetry/DynamicSymmetry.h 984e6e1291e9fe783dd226c8754288af *inst/include/unsupported/Eigen/CXX11/src/TensorSymmetry/StaticSymmetry.h 605ff85dc24af5c998a19d7bfe49b583 *inst/include/unsupported/Eigen/CXX11/src/TensorSymmetry/Symmetry.h 22b36025d9ecfd6e85038a4303a07f86 *inst/include/unsupported/Eigen/CXX11/src/TensorSymmetry/util/TemplateGroupTheory.h 1c6baa178d843ec8960212a46581f084 *inst/include/unsupported/Eigen/CXX11/src/ThreadPool/Barrier.h 34d22af36853a3c71ce3da3f3e0d565d *inst/include/unsupported/Eigen/CXX11/src/ThreadPool/EventCount.h e60ab3f4920ffdfe2497c337ae04a396 *inst/include/unsupported/Eigen/CXX11/src/ThreadPool/NonBlockingThreadPool.h 99bd359fc00e1e160eea19a121d840ef *inst/include/unsupported/Eigen/CXX11/src/ThreadPool/RunQueue.h 99c1519739630dc91690432451033ee8 *inst/include/unsupported/Eigen/CXX11/src/ThreadPool/ThreadCancel.h 77ba70869910de96c93fd4b535526834 *inst/include/unsupported/Eigen/CXX11/src/ThreadPool/ThreadEnvironment.h 4a673fd804c8ad4519c9b7676eed6b51 *inst/include/unsupported/Eigen/CXX11/src/ThreadPool/ThreadLocal.h be92216cff4a7978a175f0e7374bf03a *inst/include/unsupported/Eigen/CXX11/src/ThreadPool/ThreadPoolInterface.h 265bd1619a45db5e1aa7b41c66dcab53 *inst/include/unsupported/Eigen/CXX11/src/ThreadPool/ThreadYield.h eec720d2717d77abd9be8fe98f871d3d *inst/include/unsupported/Eigen/CXX11/src/util/CXX11Meta.h 9ec42bffe00578aceca970344a507864 *inst/include/unsupported/Eigen/CXX11/src/util/CXX11Workarounds.h 1e12f04d429aa7539da4e08d1b105753 *inst/include/unsupported/Eigen/CXX11/src/util/EmulateArray.h 2106c71ab7c12b1df8f21cd7de91900e *inst/include/unsupported/Eigen/CXX11/src/util/MaxSizeVector.h e1aad13f436794eb03abbf9cb9c8f329 *inst/include/unsupported/Eigen/EulerAngles 54eed7040f45334ac17e8ce52052657f *inst/include/unsupported/Eigen/FFT 900031a26ead4c754ca9a5ec4fb4fd60 *inst/include/unsupported/Eigen/IterativeSolvers 2ec3b8ae1f6f8cf0c06cdd26e4c92b19 *inst/include/unsupported/Eigen/KroneckerProduct 7db524ba7997a0e026986947a7681485 *inst/include/unsupported/Eigen/LevenbergMarquardt fe82f55590ef4e7e8746d42849b36c16 *inst/include/unsupported/Eigen/MPRealSupport 7d203b31fdc9e2a16ef3a84c857bb098 *inst/include/unsupported/Eigen/MatrixFunctions b6bb2d06c8ecf9e5ee1c1b1ffece2c64 *inst/include/unsupported/Eigen/MoreVectorization 21cce7ad666181669753c90327e01321 *inst/include/unsupported/Eigen/NonLinearOptimization 5ad7a0ec7dc670c56313da681c282242 *inst/include/unsupported/Eigen/NumericalDiff 3bbda3333cce7212781177633429242e *inst/include/unsupported/Eigen/OpenGLSupport fc6cade564d8203a15ded8d3230e8939 *inst/include/unsupported/Eigen/Polynomials 5eb76b0bf6d7158082b9f53ce8952c1f *inst/include/unsupported/Eigen/Skyline fa2b95a8a9d6d32a7620da61c209d187 *inst/include/unsupported/Eigen/SparseExtra 36ad389632e775ed0afc72b87caa2fcc *inst/include/unsupported/Eigen/SpecialFunctions a9e681a260f3e7f0229ad01c0c74b0cd *inst/include/unsupported/Eigen/Splines a87cd9c4876c33612bd474881d479bf6 *inst/include/unsupported/Eigen/src/AutoDiff/AutoDiffJacobian.h 93d82a06e0a1b8ddef50c5719c549dda *inst/include/unsupported/Eigen/src/AutoDiff/AutoDiffScalar.h 92ad88e993f5118143fd5e9a0a01d772 *inst/include/unsupported/Eigen/src/AutoDiff/AutoDiffVector.h 48837c998e8b3eab654adbd4fa663e84 *inst/include/unsupported/Eigen/src/BVH/BVAlgorithms.h 3e01f21ae682bb50741a0e045f348ba0 *inst/include/unsupported/Eigen/src/BVH/KdBVH.h be69b918915c2b18c89b0b5c4bded54f *inst/include/unsupported/Eigen/src/Eigenvalues/ArpackSelfAdjointEigenSolver.h 7641cc97dd35ba62dd0774321f68904e *inst/include/unsupported/Eigen/src/EulerAngles/EulerAngles.h 67c26fa8d2b4c0bc9e5435b9f93398a8 *inst/include/unsupported/Eigen/src/EulerAngles/EulerSystem.h 91c40c9d1a1c2f82a5bc928760f00809 *inst/include/unsupported/Eigen/src/FFT/ei_fftw_impl.h 883ba96410570d9432c3479ca7b38767 *inst/include/unsupported/Eigen/src/FFT/ei_kissfft_impl.h 816b889bab1493935c8b57a011788e08 *inst/include/unsupported/Eigen/src/IterativeSolvers/ConstrainedConjGrad.h 39398818ce951f1a7eab9588250e348f *inst/include/unsupported/Eigen/src/IterativeSolvers/DGMRES.h 14e0f3b7c5d4dc6c24a0c93bc9d8490a *inst/include/unsupported/Eigen/src/IterativeSolvers/GMRES.h a42bc8531a6fe28e8fe3b70166cc7d2d *inst/include/unsupported/Eigen/src/IterativeSolvers/IDRS.h 27ac1505b341316adbc72365b9367ae8 *inst/include/unsupported/Eigen/src/IterativeSolvers/IncompleteLU.h 4cfae3281bf13bc9ed1a952ecd3a34ba *inst/include/unsupported/Eigen/src/IterativeSolvers/IterationController.h 134d6eacba23e34a48e930bc2de55f20 *inst/include/unsupported/Eigen/src/IterativeSolvers/MINRES.h 700f58fae8ecc28726582df3ff76783d *inst/include/unsupported/Eigen/src/IterativeSolvers/Scaling.h fe3008f64d440042b992dfcbdb1c43fb *inst/include/unsupported/Eigen/src/KroneckerProduct/KroneckerTensorProduct.h d9ff86916e5e5a1e3d74273ff947dab7 *inst/include/unsupported/Eigen/src/LevenbergMarquardt/CopyrightMINPACK.txt d81eff550898b40c7d51e86be39a5eee *inst/include/unsupported/Eigen/src/LevenbergMarquardt/LMcovar.h eb5aab42e6b1755ad6ed6a3d64c85849 *inst/include/unsupported/Eigen/src/LevenbergMarquardt/LMonestep.h 370bfa601a00a753effd58dd2acab1eb *inst/include/unsupported/Eigen/src/LevenbergMarquardt/LMpar.h b335d907e4283aaa1fd76ea468e5b955 *inst/include/unsupported/Eigen/src/LevenbergMarquardt/LMqrsolv.h c87a5e2ae14f64cf56439d08236891ce *inst/include/unsupported/Eigen/src/LevenbergMarquardt/LevenbergMarquardt.h d0731f6733298683ba4303908de31e7b *inst/include/unsupported/Eigen/src/MatrixFunctions/MatrixExponential.h 946b65b5dabe6dd1cab9bcec99be0365 *inst/include/unsupported/Eigen/src/MatrixFunctions/MatrixFunction.h e39bf3d9ef17a3c5bf01100d5f74ce94 *inst/include/unsupported/Eigen/src/MatrixFunctions/MatrixLogarithm.h b0c03782cb14949ef52526a841ebef1e *inst/include/unsupported/Eigen/src/MatrixFunctions/MatrixPower.h c2233ea27daa28237dc585513f9e59be *inst/include/unsupported/Eigen/src/MatrixFunctions/MatrixSquareRoot.h ad137f2e6b96d77cce5fd3000dde38b6 *inst/include/unsupported/Eigen/src/MatrixFunctions/StemFunction.h e3e564572a4abd4270811bfd55eefc0b *inst/include/unsupported/Eigen/src/MoreVectorization/MathFunctions.h ef5bfb36ead3d59b0740262c42f35e9c *inst/include/unsupported/Eigen/src/NonLinearOptimization/HybridNonLinearSolver.h 916bb8c904d76c8f5221c354d380b1d3 *inst/include/unsupported/Eigen/src/NonLinearOptimization/LevenbergMarquardt.h 2dddb5bda1aa9c3ec00c14fe57d9b036 *inst/include/unsupported/Eigen/src/NonLinearOptimization/chkder.h 8fa8effaf7427df38efe187d79f38977 *inst/include/unsupported/Eigen/src/NonLinearOptimization/covar.h 2bc5fad8c5e8bc445a152cca73b55695 *inst/include/unsupported/Eigen/src/NonLinearOptimization/dogleg.h 7c9dbda9021e771f78218266f7bd7d36 *inst/include/unsupported/Eigen/src/NonLinearOptimization/fdjac1.h 39c65c9a6ea3e0a03b30f838f4a4e434 *inst/include/unsupported/Eigen/src/NonLinearOptimization/lmpar.h 0c31ca70c017a1caa330afb6e7f7d613 *inst/include/unsupported/Eigen/src/NonLinearOptimization/qrsolv.h 383cf032d01af4567d1f02d11897fb7d *inst/include/unsupported/Eigen/src/NonLinearOptimization/r1mpyq.h c930b6771d0c4a8e13c1c6b77b93d273 *inst/include/unsupported/Eigen/src/NonLinearOptimization/r1updt.h bc16831605dac24fd37a87e1cc1bce38 *inst/include/unsupported/Eigen/src/NonLinearOptimization/rwupdt.h 3d568e244f7e3e7fba0d70ed07e3263a *inst/include/unsupported/Eigen/src/NumericalDiff/NumericalDiff.h e112dec4dd76e520ba0bad302364fe79 *inst/include/unsupported/Eigen/src/Polynomials/Companion.h 12b0293d31e7c4b24f024b7a1603f07d *inst/include/unsupported/Eigen/src/Polynomials/PolynomialSolver.h de9aaaebae2081ceedd850ec2f439827 *inst/include/unsupported/Eigen/src/Polynomials/PolynomialUtils.h 79ca7405d8b8ad2fe36e10b8fee4148b *inst/include/unsupported/Eigen/src/Skyline/SkylineInplaceLU.h 824c9da2a9592016b537be34e54e4dfe *inst/include/unsupported/Eigen/src/Skyline/SkylineMatrix.h 9f5a3fae279cd008f68e22f1b3f68e0d *inst/include/unsupported/Eigen/src/Skyline/SkylineMatrixBase.h a2ef069d1b248a87813de8b901693684 *inst/include/unsupported/Eigen/src/Skyline/SkylineProduct.h bae2cb0cb4a531ae9b1f47c4c280cacd *inst/include/unsupported/Eigen/src/Skyline/SkylineStorage.h 3fde3a91d55eb2fafb974de05722a571 *inst/include/unsupported/Eigen/src/Skyline/SkylineUtil.h 442ac0f752eb3cc76178316070d1f627 *inst/include/unsupported/Eigen/src/SparseExtra/BlockOfDynamicSparseMatrix.h 66cda544a242ec51e4230b3caab4b96f *inst/include/unsupported/Eigen/src/SparseExtra/BlockSparseMatrix.h 9de57da7656f706385d421f9b7e2fcb2 *inst/include/unsupported/Eigen/src/SparseExtra/DynamicSparseMatrix.h dc752616501332f15020a721ec6d7ebf *inst/include/unsupported/Eigen/src/SparseExtra/MarketIO.h 5159861264eb780296f6ea62b1c885c5 *inst/include/unsupported/Eigen/src/SparseExtra/MatrixMarketIterator.h a59440ef2127a48d2d990c3e350c57aa *inst/include/unsupported/Eigen/src/SparseExtra/RandomSetter.h cbd20ac144f08c97ee471cb1f3ca2a54 *inst/include/unsupported/Eigen/src/SpecialFunctions/BesselFunctionsArrayAPI.h 4621520b36892f2903eb151b804a68ab *inst/include/unsupported/Eigen/src/SpecialFunctions/BesselFunctionsBFloat16.h a63cd8aafb6d61e8f4ac6035fdb93017 *inst/include/unsupported/Eigen/src/SpecialFunctions/BesselFunctionsFunctors.h 9bdad930bf90868e1ee9d09801c6f77b *inst/include/unsupported/Eigen/src/SpecialFunctions/BesselFunctionsHalf.h 54a0647aa83f10e267777acbabb84889 *inst/include/unsupported/Eigen/src/SpecialFunctions/BesselFunctionsImpl.h 95e2404ede0aa0be7028b928b767fdcb *inst/include/unsupported/Eigen/src/SpecialFunctions/BesselFunctionsPacketMath.h ce0ae07a1b2f7f5bedd9e31d5b8d0134 *inst/include/unsupported/Eigen/src/SpecialFunctions/HipVectorCompatibility.h 0fd1fab71470296959461ff0b0cc0e62 *inst/include/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsArrayAPI.h 80336ac8a4cf3b1372e1ca989ef2f5d5 *inst/include/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsBFloat16.h e126a9c2faab38c4d8552af22ffa9464 *inst/include/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsFunctors.h 870773cb741c1bfb74c626c668ca0976 *inst/include/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsHalf.h 764c502155970dfd44bc27230b8855b2 *inst/include/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h 031cc02523f69e04c4321a42b6dc59a8 *inst/include/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsPacketMath.h a896db19e3f753e87edb0c855f6611a6 *inst/include/unsupported/Eigen/src/SpecialFunctions/arch/AVX/BesselFunctions.h 7d5c3e84f0af0555cdeb0a90395a161a *inst/include/unsupported/Eigen/src/SpecialFunctions/arch/AVX/SpecialFunctions.h 4183ece3dd27af3bf108c8aa1ca8947e *inst/include/unsupported/Eigen/src/SpecialFunctions/arch/AVX512/BesselFunctions.h 34efb0d9403f8e589cdda3cd1e73ad99 *inst/include/unsupported/Eigen/src/SpecialFunctions/arch/AVX512/SpecialFunctions.h 7e042ddaa7e463b95c487e9eef5d7093 *inst/include/unsupported/Eigen/src/SpecialFunctions/arch/GPU/SpecialFunctions.h e32b6e9a7992d683ed5e5258516e69e5 *inst/include/unsupported/Eigen/src/SpecialFunctions/arch/NEON/BesselFunctions.h 0ce08e0456e47a1640c1777f2cf38450 *inst/include/unsupported/Eigen/src/SpecialFunctions/arch/NEON/SpecialFunctions.h 6cf9e021f1bc3e964b85f3b436337d4a *inst/include/unsupported/Eigen/src/Splines/Spline.h a7a8b63bca04532ec7b30e6134764558 *inst/include/unsupported/Eigen/src/Splines/SplineFitting.h bc75ab5a8c3ca729f8dcb08c309b6f47 *inst/include/unsupported/Eigen/src/Splines/SplineFwd.h b4f0b014ec53404e843dbc718fabf57e *inst/skeleton/Makevars b4f0b014ec53404e843dbc718fabf57e *inst/skeleton/Makevars.win 094a35f66ff4b08c54a4fb8d6d1f4393 *inst/skeleton/rcppeigen_hello_world.Rd b83bee5d5ad946dbfa8f1e410c90d745 *inst/skeleton/rcppeigen_hello_world.cpp b81206a4c831ced7a37fd8f7a0066830 *inst/tinytest/cpp/rcppeigen.cpp be6cfb634100b1f08a3e08b30e929fd7 *inst/tinytest/cpp/solution.cpp a62e42e20e8381d578b0f3af53fb11c6 *inst/tinytest/cpp/sparse.cpp 2bba9269b536376c10e821c2e9dcadc7 *inst/tinytest/cpp/transform.cpp 1c06c0fac5f692d057b35d2584001afd *inst/tinytest/cpp/wrap.cpp 76f6f32151c7d47de981270fe5e97613 *inst/tinytest/test_RcppEigen.R a67648e012cd905a275ea48d5da0e6b0 *inst/tinytest/test_fastLm.R eb291ceff757f9e041cb865f09fc8428 *inst/tinytest/test_misc.R 637b2794a96520b25fe867d2a2974cc0 *inst/tinytest/test_solution.R 856cbc90058c8f2c24d05c4afc6e8174 *inst/tinytest/test_sparse.R 0c87d121b0a4a7499b01e07146a1dcf0 *inst/tinytest/test_transform.R 5971761ce901a48ca3651c1293aaf284 *inst/tinytest/test_wrap.R 9b1fd1fa838f03ed944e8b724dbafe98 *man/RcppEigen-package.Rd 7d22109cab99baae3f89a00a23fdd0bc *man/RcppEigen.package.skeleton.Rd 3f68ed0b145014b9e9e4bc769970b915 *man/fastLm.Rd 410976344986b045687d43424a05deb3 *src/Makevars 410976344986b045687d43424a05deb3 *src/Makevars.win d7bc0ddde486ae6dccc76d29f1a5f297 *src/RcppEigen.cpp 66c7c106ce076772af617e639df41042 *src/RcppExports.cpp 35b458bb1fd3d10375ae693f196d073b *src/fastLm.cpp 16915b4298cfa6d0ba90c23fd5e0c633 *src/fastLm.h 24d69df3f629ce3447ec7d7c81ab775c *tests/tinytest.R c52f521406b904dee163f3286ba491e8 *vignettes/RcppEigen-Introduction.Rnw RcppEigen/R/0000755000176200001440000000000014456322731012336 5ustar liggesusersRcppEigen/R/RcppExports.R0000644000176200001440000000066514456320632014757 0ustar liggesusers# Generated by using Rcpp::compileAttributes() -> do not edit by hand # Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393 eigen_version <- function(single) { .Call(`_RcppEigen_eigen_version`, single) } Eigen_SSE <- function() { .Call(`_RcppEigen_Eigen_SSE`) } EigenNbThreads <- function() { .Call(`_RcppEigen_EigenNbThreads`) } fastLm_Impl <- function(X, y, type) { .Call(`_RcppEigen_fastLm_Impl`, X, y, type) } RcppEigen/R/flags.R0000644000176200001440000000164112253717461013560 0ustar liggesusers## Copyright (C) 2012 Douglas Bates, Dirk Eddelbuettel and Romain Francois ## ## This file is part of RcppEigen. ## ## RcppEigen is free software: you can redistribute it and/or modify it ## under the terms of the GNU General Public License as published by ## the Free Software Foundation, either version 2 of the License, or ## (at your option) any later version. ## ## RcppEigen is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with RcppEigen. If not, see . RcppEigenCxxFlags <- function() { paste('-I"', system.file("include", package="RcppEigen"), '"', sep="") } CxxFlags <- function() { cat(RcppEigenCxxFlags()) } RcppEigen/R/fastLm.R0000644000176200001440000000716714456322731013722 0ustar liggesusers## fastLm.R: Rcpp/Eigen implementation of lm() ## ## Copyright (C) 2011 - 2023 Douglas Bates, Dirk Eddelbuettel and Romain Francois ## ## This file is part of RcppEigen. ## ## RcppEigen is free software: you can redistribute it and/or modify it ## under the terms of the GNU General Public License as published by ## the Free Software Foundation, either version 2 of the License, or ## (at your option) any later version. ## ## RcppEigen is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with RcppEigen. If not, see . fastLmPure <- function(X, y, method = 0L) { stopifnot(is.matrix(X), is.numeric(y), NROW(y)==nrow(X)) fastLm_Impl(X, y, method) } fastLm <- function(X, ...) UseMethod("fastLm") fastLm.default <- function(X, y, method = 0L, ...) { X <- as.matrix(X) y <- as.numeric(y) res <- fastLmPure(X, y, method) res$call <- match.call() res$intercept <- any(apply(X, 2, function(x) all(x == x[1]))) class(res) <- "fastLm" res } print.fastLm <- function(x, ...) { cat("\nCall:\n") print(x$call) cat("\nCoefficients:\n") print(x$coefficients, digits=5) } summary.fastLm <- function(object, ...) { coef <- object$coefficients se <- object$se tval <- coef/se object$coefficients <- cbind(Estimate = coef, "Std. Error" = se, "t value" = tval, "Pr(>|t|)" = 2*pt(-abs(tval), df=object$df.residual)) ## cf src/stats/R/lm.R and case with no weights and an intercept f <- object$fitted.values r <- object$residuals #mss <- sum((f - mean(f))^2) mss <- if (object$intercept) sum((f - mean(f))^2) else sum(f^2) rss <- sum(r^2) object$r.squared <- mss/(mss + rss) df.int <- if (object$intercept) 1L else 0L n <- length(f) rdf <- object$df.residual object$adj.r.squared <- 1 - (1 - object$r.squared) * ((n - df.int)/rdf) class(object) <- "summary.fastLm" object } print.summary.fastLm <- function(x, ...) { cat("\nCall:\n") print(x$call) cat("\nResiduals:\n") digits <- max(3, getOption("digits") - 3) print(summary(x$residuals, digits=digits)[-4]) cat("\n") printCoefmat(x$coefficients, P.values=TRUE, has.Pvalue=TRUE, ...) cat("\nResidual standard error: ", formatC(x$s, digits=digits), " on ", formatC(x$df.residual), " degrees of freedom\n", sep="") cat("Multiple R-squared: ", formatC(x$r.squared, digits=digits), ",\tAdjusted R-squared: ",formatC(x$adj.r.squared, digits=digits), "\n", sep="") invisible(x) } fastLm.formula <- function(formula, data=list(), method = 0L, ...) { mf <- model.frame(formula=formula, data=data) X <- model.matrix(attr(mf, "terms"), data=mf) y <- model.response(mf) res <- fastLm.default(X, y, method=method, ...) res$call <- match.call() ## I think this is redundant. The formula is available as res$call$formula res$formula <- formula res$intercept <- attr(attr(mf, "terms"), "intercept") res } predict.fastLm <- function(object, newdata=NULL, ...) { if (is.null(newdata)) { y <- fitted(object) } else { if (!is.null(object$formula)) { x <- model.matrix(object$formula, newdata) } else { x <- newdata # #nocov } y <- as.vector(x %*% coef(object)) } y } RcppEigen/R/RcppEigen.package.skeleton.R0000644000176200001440000001125314402632114017542 0ustar liggesusers## RcppEigen.package.skeleton.R: makes a skeleton for a package that wants to use RcppEigen ## ## Copyright (C) 2011 - 2023 Douglas Bates, Dirk Eddelbuettel and Romain Francois ## ## This file is part of RcppEigen. ## ## RcppEigen is free software: you can redistribute it and/or modify it ## under the terms of the GNU General Public License as published by ## the Free Software Foundation, either version 2 of the License, or ## (at your option) any later version. ## ## RcppEigen is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with RcppEigen. If not, see . RcppEigen.package.skeleton <- function(name= "anRpackage", list = character(), environment = .GlobalEnv, path = ".", force = FALSE, code_files = character(), example_code = TRUE) { env <- parent.frame(1) # #nocov start if (!length(list)) { fake <- TRUE assign("Rcpp.fake.fun", function() {}, envir = env) } else { fake <- FALSE } haveKitten <- requireNamespace("pkgKitten", quietly=TRUE) skelFunUsed <- ifelse(haveKitten, pkgKitten::kitten, package.skeleton) skelFunName <- ifelse(haveKitten, "kitten", "package.skeleton") message("\nCalling ", skelFunName, " to create basic package.") ## first let the traditional version do its business call <- match.call() call[[1]] <- skelFunUsed if ("example_code" %in% names(call)) { call[["example_code"]] <- NULL # remove the example_code argument } if (! haveKitten) { # in the package.skeleton() case if (fake) { call[["list"]] <- "Rcpp.fake.fun" } } else { if (force) { call[["force"]] <- NULL } } tryCatch(eval(call, envir = env), error = function(e) { cat(paste(e, "\n")) # print error stop(paste("error while calling `", skelFunName, "`", sep="")) }) message("\nAdding RcppEigen settings") ## now pick things up root <- file.path(path, name) ## Add Rcpp to the DESCRIPTION DESCRIPTION <- file.path(root, "DESCRIPTION") if (file.exists(DESCRIPTION)) { x <- cbind(read.dcf(DESCRIPTION), "Imports" = sprintf("Rcpp (>= %s)", packageDescription("Rcpp")[["Version"]]), "LinkingTo" = "Rcpp, RcppEigen") write.dcf(x, file = DESCRIPTION) message(" >> added Imports: Rcpp") message(" >> added LinkingTo: Rcpp, RcppEigen") } ## add a useDynLib to NAMESPACE, NAMESPACE <- file.path(root, "NAMESPACE") lines <- readLines(NAMESPACE) if (!any(grepl("useDynLib", lines))) { lines <- c(sprintf("useDynLib(%s)", name), "importFrom(Rcpp, evalCpp)", ## ensures Rcpp instantiation lines) writeLines(lines, con = NAMESPACE) message(" >> added useDynLib and importFrom directives to NAMESPACE") } ## lay things out in the src directory src <- file.path(root, "src") if (!file.exists(src)) { dir.create(src) } man <- file.path(root, "man") if (!file.exists(man)) { dir.create(man) } skeleton <- system.file("skeleton", package = "RcppEigen") Makevars <- file.path(src, "Makevars") if (!file.exists(Makevars)) { file.copy(file.path(skeleton, "Makevars"), Makevars) message(" >> added Makevars file with RcppEigen settings") } Makevars.win <- file.path(src, "Makevars.win") if (!file.exists(Makevars.win)) { file.copy(file.path(skeleton, "Makevars.win"), Makevars.win) message(" >> added Makevars.win file with RcppEigen settings") } if (example_code) { file.copy(file.path(skeleton, "rcppeigen_hello_world.cpp"), src) message(" >> added example src file using Eigen classes") file.copy(file.path(skeleton, "rcppeigen_hello_world.Rd"), man) message(" >> added example Rd file for using Eigen classes") Rcpp::compileAttributes(root) message(" >> invoked Rcpp::compileAttributes to create wrappers") } if (fake) { rm("Rcpp.fake.fun", envir = env) unlink(file.path(root, "R" , "Rcpp.fake.fun.R")) unlink(file.path(root, "man", "Rcpp.fake.fun.Rd")) } invisible(NULL) # #nocov end } RcppEigen/R/inline.R0000644000176200001440000000203413204322073013723 0ustar liggesusers## Copyright (C) 2011 - 2017 Douglas Bates, Dirk Eddelbuettel and Romain Francois ## ## This file is part of RcppEigen. ## ## RcppEigen is free software: you can redistribute it and/or modify it ## under the terms of the GNU General Public License as published by ## the Free Software Foundation, either version 2 of the License, or ## (at your option) any later version. ## ## RcppEigen is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with RcppEigen. If not, see . inlineCxxPlugin <- Rcpp::Rcpp.plugin.maker(include.before = "#include ", package = "RcppEigen" # , LinkingTo = c("RcppEigen", "Rcpp") ) RcppEigen/R/SHLIB.R0000644000176200001440000000153412264631730013322 0ustar liggesusers## Copyright (C) 2011 Douglas Bates, Dirk Eddelbuettel and Romain Francois ## ## This file is part of RcppEigen. ## ## RcppEigen is free software: you can redistribute it and/or modify it ## under the terms of the GNU General Public License as published by ## the Free Software Foundation, either version 2 of the License, or ## (at your option) any later version. ## ## RcppEigen is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with RcppEigen. If not, see . #SHLIB <- Rcpp:::SHLIB.maker( # env = list( # PKG_LIBS = Rcpp:::RcppLdFlags(), # ) #) RcppEigen/cleanup0000755000176200001440000000031014662150313013476 0ustar liggesusers#!/bin/sh rm -f src/*.o src/*.so \ inst/doc/RcppEigen-unitTests.out \ inst/doc/RcppEigen-unitTests.aux \ inst/doc/RcppEigen-unitTests.log \ */*~ *~ rm -rf autom4te.cache RcppEigen/vignettes/0000755000176200001440000000000014662150312014136 5ustar liggesusersRcppEigen/vignettes/RcppEigen-Introduction.Rnw0000644000176200001440000024401313471777756021215 0ustar liggesusers\documentclass[shortnames,article,nojss]{jss} \usepackage{booktabs,bm,amsmath,thumbpdf} %\VignetteIndexEntry{RcppEigen-intro} %\VignetteKeywords{linear algebra, template programming, C++, R, Rcpp} %\VignettePackage{RcppEigen} %% VIGNETTE <>= pkgVersion <- packageDescription("RcppEigen")$Version pkgDate <- packageDescription("RcppEigen")$Date prettyDate <- format(Sys.Date(), "%B %e, %Y") #require("RcppEigen") #eigenVersion <- paste(unlist(.Call("eigen_version", FALSE)), collapse=".") @ \author{Douglas Bates\\University of Wisconsin-Madison \And Dirk Eddelbuettel\\Debian Project} \Plainauthor{Douglas Bates, Dirk Eddelbuettel} \title{Fast and Elegant Numerical Linear Algebra Using the \pkg{RcppEigen} Package} \Plaintitle{Fast and Elegant Numerical Linear Algebra Using the RcppEigen Package} \Shorttitle{Fast and Elegant Numerical Linear Algebra with \pkg{RcppEigen}} \Abstract{ The \pkg{RcppEigen} package provides access from \proglang{R} \citep{R:Main} to the \pkg{Eigen} \citep*{Eigen:Web} \proglang{C++} template library for numerical linear algebra. \pkg{Rcpp} \citep{CRAN:Rcpp,Eddelbuettel:2013:Rcpp} classes and specializations of the \proglang{C++} templated functions \code{as} and \code{wrap} from \pkg{Rcpp} provide the ``glue'' for passing objects from \proglang{R} to \proglang{C++} and back. Several introductory examples are presented. This is followed by an in-depth discussion of various available approaches for solving least-squares problems, including rank-revealing methods, concluding with an empirical run-time comparison. Last but not least, sparse matrix methods are discussed. } \Keywords{linear algebra, template programming, \proglang{R}, \proglang{C++}, \pkg{Rcpp}} \Plainkeywords{linear algebra, template programmig, R, C++, Rcpp} \Address{ Douglas Bates \\ Department of Statistics \\ University of Wisconsin-Madison \\ Madison, WI, United States of America \\ E-mail: \email{bates@stat.wisc.edu} \\ URL: \url{http://www.stat.wisc.edu/~bates/}\\ Dirk Eddelbuettel \\ Debian Project \\ River Forest, IL, United States of America\\ E-mail: \email{edd@debian.org}\\ URL: \url{http://dirk.eddelbuettel.com}\\ } \usepackage{Sweave} \newcommand{\argmin}{\operatorname{argmin}\displaylimits} \newcommand{\rank}{\operatorname{rank}} %% highlights macros %% Style definition file generated by highlight 2.7, http://www.andre-simon.de/ % Highlighting theme definition: \newcommand{\hlstd}[1]{\textcolor[rgb]{0,0,0}{#1}} \newcommand{\hlnum}[1]{\textcolor[rgb]{0,0,0}{#1}} \newcommand{\hlopt}[1]{\textcolor[rgb]{0,0,0}{#1}} \newcommand{\hlesc}[1]{\textcolor[rgb]{0.74,0.55,0.55}{#1}} \newcommand{\hlstr}[1]{\textcolor[rgb]{0.90,0.15,0.15}{#1}} \newcommand{\hldstr}[1]{\textcolor[rgb]{0.74,0.55,0.55}{#1}} \newcommand{\hlslc}[1]{\textcolor[rgb]{0.67,0.13,0.13}{\it{#1}}} \newcommand{\hlcom}[1]{\textcolor[rgb]{0.67,0.13,0.13}{\it{#1}}} \newcommand{\hldir}[1]{\textcolor[rgb]{0,0,0}{#1}} \newcommand{\hlsym}[1]{\textcolor[rgb]{0,0,0}{#1}} \newcommand{\hlline}[1]{\textcolor[rgb]{0.33,0.33,0.33}{#1}} \newcommand{\hlkwa}[1]{\textcolor[rgb]{0.61,0.13,0.93}{\bf{#1}}} \newcommand{\hlkwb}[1]{\textcolor[rgb]{0.13,0.54,0.13}{#1}} \newcommand{\hlkwc}[1]{\textcolor[rgb]{0,0,1}{#1}} \newcommand{\hlkwd}[1]{\textcolor[rgb]{0,0,0}{#1}} \definecolor{bgcolor}{rgb}{1,1,1} % ------------------------------------------------------------------------ \begin{document} \SweaveOpts{engine=R,eps=FALSE} \begin{quote} \footnotesize This vignette corresponds to a \href{http://www.jstatsoft.org/v52/i05/}{paper published} in the \textsl{Journal of Statistical Software}. Currently still identical to the paper, this vignette version may over time receive minor updates. For citations, please use \citet{JSS:RcppEigen} as provided by \code{citation("RcppEigen")}. This version corresponds to \pkg{RcppEigen} version \Sexpr{pkgVersion} and was typeset on \Sexpr{prettyDate}. \end{quote} \section{Introduction} \label{sec:intro} Linear algebra is an essential building block of statistical computing. Operations such as matrix decompositions, linear program solvers, and eigenvalue/eigenvector computations are used in many estimation and analysis routines. As such, libraries supporting linear algebra have long been provided by statistical programmers for different programming languages and environments. Because it is object-oriented, \proglang{C++}, one of the central modern languages for numerical and statistical computing, is particularly effective at representing matrices, vectors and decompositions, and numerous class libraries providing linear algebra routines have been written over the years. As both the \proglang{C++} language and standards have evolved \citep{Meyers:2005:EffectiveC++,Meyers:1995:MoreEffectiveC++,Cpp11}, so have the compilers implementing the language. Relatively modern language constructs such as template meta-programming are particularly useful because they provide overloading of operations (allowing expressive code in the compiled language similar to what can be done in scripting languages) and can shift some of the computational burden from the run-time to the compile-time. (A more detailed discussion of template meta-programming is, however, beyond the scope of this paper). \cite{Veldhuizen:1998:Blitz} provided an early and influential implementation of numerical linear algebra classes that already demonstrated key features of this approach. Its usage was held back at the time by the limited availability of compilers implementing all the necessary features of the \proglang{C++} language. This situation has greatly improved over the last decade, and many more libraries have been made available. One such \proglang{C++} library is \pkg{Eigen} by \citet*{Eigen:Web} which started as a sub-project to KDE (a popular Linux desktop environment), initially focussing on fixed-size matrices to represent projections in a visualization application. \pkg{Eigen} grew from there and has over the course of about a decade produced three major releases with \pkg{Eigen}3 being the current major version. To check the minor and patch version numbers, load the \pkg{RcppEigen} package and call this (internal) helper function: \begin{CodeInput} R> RcppEigen:::eigen_version(FALSE) \end{CodeInput} \begin{CodeOutput} major minor patch 3 3 5 \end{CodeOutput} \pkg{Eigen} is of interest as the \proglang{R} system for statistical computation and graphics \citep{R:Main} is itself easily extensible. This is particular true via the \proglang{C} language that most of \proglang{R}'s compiled core parts are written in, but also for the \proglang{C++} language which can interface with \proglang{C}-based systems rather easily. The manual ``Writing \proglang{R} Extensions'' \citep{R:Extensions} is the basic reference for extending \proglang{R} with either \proglang{C} or \proglang{C++}. The \pkg{Rcpp} package by \citet{JSS:Rcpp,CRAN:Rcpp} facilitates extending \proglang{R} with \proglang{C++} code by providing seamless object mapping between both languages. % As stated in the \pkg{Rcpp} \citep{CRAN:Rcpp} vignette, ``Extending \pkg{Rcpp}'' as well as in \citet{Eddelbuettel:2013:Rcpp} \begin{quote} \pkg{Rcpp} facilitates data interchange between \proglang{R} and \proglang{C++} through the templated functions \code{Rcpp::as} (for conversion of objects from \proglang{R} to \proglang{C++}) and \code{Rcpp::wrap} (for conversion from \proglang{C++} to \proglang{R}). \end{quote} The \pkg{RcppEigen} package provides the header files composing the \pkg{Eigen} \proglang{C++} template library, along with implementations of \code{Rcpp::as} and \code{Rcpp::wrap} for the \proglang{C++} classes defined in \pkg{Eigen}. The \pkg{Eigen} classes themselves provide high-performance, versatile and comprehensive representations of dense and sparse matrices and vectors, as well as decompositions and other functions to be applied to these objects. The next section introduces some of these classes and shows how to interface to them from \proglang{R}. \section[Eigen classes]{\pkg{Eigen} classes} \label{sec:eclasses} \pkg{Eigen} is a \proglang{C++} template library providing classes for many forms of matrices, vectors, arrays and decompositions. These classes are flexible and comprehensive allowing for both high performance and well structured code representing high-level operations. \proglang{C++} code based on \pkg{Eigen} is often more like \proglang{R} code, working on the ``whole object'', than like compiled code in other languages where operations often must be coded in loops. As in many \proglang{C++} template libraries using template meta-programming \citep{Abrahams+Gurtovoy:2004:TemplateMetaprogramming}, the templates themselves can be very complicated. However, \pkg{Eigen} provides \code{typedef}s for common classes that correspond to \proglang{R} matrices and vectors, as shown in Table~\ref{tab:REigen}, and this paper will use these \code{typedef}s. \begin{table}[t!] \centering \begin{tabular}{l l} \hline \multicolumn{1}{c}{\proglang{R} object type} & \multicolumn{1}{c}{\pkg{Eigen} class typedef}\\ \hline numeric matrix & \code{MatrixXd}\\ integer matrix & \code{MatrixXi}\\ complex matrix & \code{MatrixXcd}\\ numeric vector & \code{VectorXd}\\ integer vector & \code{VectorXi}\\ complex vector & \code{VectorXcd}\\ \code{Matrix::dgCMatrix} \phantom{XXX} & \code{SparseMatrix}\\ \hline \end{tabular} \caption{Correspondence between \proglang{R} matrix and vector types and classes in the \pkg{Eigen} namespace. \label{tab:REigen}} \end{table} Here, \code{Vector} and \code{Matrix} describe the dimension of the object. The \code{X} signals that these are dynamically-sized objects (as opposed to fixed-size matrices such as $3 \times 3$ matrices also available in \pkg{Eigen}). Lastly, the trailing characters \code{i}, \code{d} and \code{cd} denote, respectively, storage types \code{integer}, \code{double} and \code{complex double}. The \proglang{C++} classes shown in Table~\ref{tab:REigen} are in the \pkg{Eigen} namespace, which means that they must be written as \code{Eigen::MatrixXd}. However, if one prefaces the use of these class names with a declaration like \begin{quote} \noindent \ttfamily \hlstd{}\hlkwa{using\ }\hlstd{Eigen}\hlopt{::}\hlstd{MatrixXd}\hlopt{;}\hlstd{}\hspace*{\fill}\\ \mbox{} \normalfont \normalsize \end{quote} \vspace*{-0.4cm} then one can use these names without the namespace qualifier. \subsection[Mapped matrices in Eigen]{Mapped matrices in \pkg{Eigen}} \label{sec:mapped} Storage for the contents of matrices from the classes shown in Table~\ref{tab:REigen} is allocated and controlled by the class constructors and destructors. Creating an instance of such a class from an \proglang{R} object involves copying its contents. An alternative is to have the contents of the \proglang{R} matrix or vector mapped to the contents of the object from the \pkg{Eigen} class. For dense matrices one can use the \pkg{Eigen} templated class \code{Map}, and for sparse matrices one can deploy the \pkg{Eigen} templated class \code{MappedSparseMatrix}. One must, of course, be careful not to modify the contents of the \proglang{R} object in the \proglang{C++} code. A recommended practice is always to declare mapped objects as {\ttfamily\hlkwb{const}\normalfont}. \subsection[Arrays in Eigen]{Arrays in \pkg{Eigen}} \label{sec:arrays} For matrix and vector classes \pkg{Eigen} overloads the \code{`*'} operator as matrix multiplication. Occasionally component-wise operations instead of matrix operations are desired, for which the \code{Array} templated classes are used in \pkg{Eigen}. Switching back and forth between matrices or vectors and arrays is usually done via the \code{array()} method for matrix or vector objects and the \code{matrix()} method for arrays. \subsection[Structured matrices in Eigen]{Structured matrices in \pkg{Eigen}} \label{sec:structured} \pkg{Eigen} provides classes for matrices with special structure such as symmetric matrices, triangular matrices and banded matrices. For dense matrices, these special structures are described as ``views'', meaning that the full dense matrix is stored but only part of the matrix is used in operations. For a symmetric matrix one must specify whether the lower triangle or the upper triangle is to be used as the contents, with the other triangle defined by the implicit symmetry. \section{Some simple examples} \label{sec:simple} \proglang{C++} functions to perform simple operations on matrices or vectors can follow a pattern of: \begin{enumerate} \item Map the \proglang{R} objects passed as arguments into \pkg{Eigen} objects. \item Create the result. \item Return \code{Rcpp::wrap} applied to the result. \end{enumerate} An idiom for the first step is % using Eigen::Map; % using Eigen::MatrixXd; % using Rcpp::as; % % const Map A(as >(AA)); %\end{lstlisting} \begin{quote} \noindent \ttfamily \hlstd{}\hlkwa{using\ }\hlstd{Eigen}\hlsym{::}\hlstd{Map}\hlsym{;}\hspace*{\fill}\\ \hlstd{}\hlkwa{using\ }\hlstd{Eigen}\hlsym{::}\hlstd{MatrixXd}\hlsym{;}\hspace*{\fill}\\ \hlstd{}\hlkwa{using\ }\hlstd{Rcpp}\hlsym{::}\hlstd{as}\hlsym{;}\hspace*{\fill}\\ \hlstd{}\hspace*{\fill}\\ \hlkwb{const\ }\hlstd{Map}\hlsym{$<$}\hlstd{MatrixXd}\hlsym{$>$}\hlstd{\ \ }\hlsym{}\hlstd{}\hlkwd{A}\hlstd{}\hlsym{(}\hlstd{as}\hlsym{$<$}\hlstd{Map}\hlsym{$<$}\hlstd{MatrixXd}\hlsym{$>$\ $>$(}\hlstd{AA}\hlsym{));}\hlstd{}\hspace*{\fill}\\ \mbox{} \normalfont \end{quote} \vspace*{-0.4cm} where \code{AA} is the name of the \proglang{R} object (of type \code{SEXP} in \proglang{C} and \proglang{C++}) passed to the \proglang{C++} function. An alternative to the \code{using} declarations is to declare a \code{typedef} as in % typedef Eigen::Map MapMatd; % const MapMatd A(Rcpp::as(AA)); \begin{quote} \noindent \ttfamily \hlstd{}\hlkwc{typedef\ }\hlstd{Eigen}\hlopt{::}\hlstd{Map}\hlopt{$<$}\hlstd{Eigen}\hlopt{::}\hlstd{MatrixXd}\hlopt{$>$}\hlstd{\ \ \ }\hlopt{}\hlstd{MapMatd}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{MapMatd}\hlstd{\ \ \ \ \ \ \ \ \ \ }\hlstd{}\hlkwd{A}\hlstd{}\hlopt{(}\hlstd{Rcpp}\hlopt{::}\hlstd{as}\hlopt{$<$}\hlstd{MapMatd}\hlopt{$>$(}\hlstd{AA}\hlopt{));}\hlstd{}\hspace*{\fill}\\ \mbox{} \normalfont \normalsize \end{quote} \vspace*{-0.4cm} The \code{cxxfunction} function from the \pkg{inline} package \citep*{CRAN:inline} for \proglang{R} and its \pkg{RcppEigen} plugin provide a convenient method of developing and debugging the \proglang{C++} code. For actual production code one generally incorporates the \proglang{C++} source code files in a package and includes the line \code{LinkingTo: Rcpp, RcppEigen} in the package's \code{DESCRIPTION} file. The \code{RcppEigen.package.skeleton} function provides a quick way of generating the skeleton of a package that will use \pkg{RcppEigen}. The \code{cxxfunction} with the \code{"Rcpp"} or \code{"RcppEigen"} plugins has the \code{as} and \code{wrap} functions already defined as \code{Rcpp::as} and \code{Rcpp::wrap}. In the examples below these declarations are omitted. It is important to remember that they are needed in actual \proglang{C++} source code for a package. The first few examples are simply for illustration as the operations shown could be more effectively performed directly in \proglang{R}. Finally, the results from \pkg{Eigen} are compared to those from the direct \proglang{R} results. \subsection{Transpose of an integer matrix} \label{sec:transpose} The next \proglang{R} code snippet creates a simple matrix of integers \begin{CodeInput} R> (A <- matrix(1:6, ncol = 2)) \end{CodeInput} \begin{CodeOutput} [,1] [,2] [1,] 1 4 [2,] 2 5 [3,] 3 6 \end{CodeOutput} \begin{CodeInput} R> str(A) \end{CodeInput} \begin{CodeOutput} int [1:3, 1:2] 1 2 3 4 5 6 \end{CodeOutput} and, in Figure~\ref{trans}, the \code{transpose()} method for the \code{Eigen::MatrixXi} class is used to return the transpose of the supplied matrix. The \proglang{R} matrix in the \code{SEXP} \code{AA} is first mapped to an \code{Eigen::MatrixXi} object, and then the matrix \code{At} is constructed from its transpose and returned to \proglang{R}. \begin{figure}[t!] \hrule \smallskip \noindent \ttfamily \hlstd{}\hlkwa{using\ }\hlstd{Eigen}\hlsym{::}\hlstd{Map}\hlsym{;}\hspace*{\fill}\\ \hlstd{}\hlkwa{using\ }\hlstd{Eigen}\hlsym{::}\hlstd{MatrixXi}\hlsym{;}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{}\hlslc{//\ Map\ the\ integer\ matrix\ AA\ from\ R}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{Map}\hlsym{$<$}\hlstd{MatrixXi}\hlsym{$>$}\hlstd{\ \ }\hlsym{}\hlstd{}\hlkwd{A}\hlstd{}\hlsym{(}\hlstd{as}\hlsym{$<$}\hlstd{Map}\hlsym{$<$}\hlstd{MatrixXi}\hlsym{$>$\ $>$(}\hlstd{AA}\hlsym{));}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{}\hlslc{//\ evaluate\ and\ return\ the\ transpose\ of\ A}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{MatrixXi}\hlstd{\ \ \ \ \ \ }\hlstd{}\hlkwd{At}\hlstd{}\hlsym{(}\hlstd{A}\hlsym{.}\hlstd{}\hlkwd{transpose}\hlstd{}\hlsym{());}\hspace*{\fill}\\ \hlstd{}\hlkwa{return\ }\hlstd{}\hlkwd{wrap}\hlstd{}\hlsym{(}\hlstd{At}\hlsym{);}\hlstd{}\hspace*{\fill} \normalfont \hrule \caption{\code{transCpp}: Transpose a matrix of integers. \label{trans}} \end{figure} The \proglang{R} snippet below compiles and links the \proglang{C++} code segment. The actual work is done by the function \code{cxxfunction} from the \pkg{inline} package which compiles, links and loads code written in \proglang{C++} and supplied as a character variable. This frees the user from having to know about compiler and linker details and options, which makes ``exploratory programming'' much easier. Here the program piece to be compiled is stored as a character variable named \code{transCpp}, and \code{cxxfunction} creates an executable function which is assigned to \code{ftrans}. This new function is used on the matrix $\bm A$ shown above, and one can check that it works as intended by comparing the output to an explicit transpose of the matrix argument. \begin{CodeInput} R> ftrans <- cxxfunction(signature(AA = "matrix"), transCpp, + plugin = "RcppEigen") R> (At <- ftrans(A)) \end{CodeInput} \begin{CodeOutput} [,1] [,2] [,3] [1,] 1 2 3 [2,] 4 5 6 \end{CodeOutput} \begin{CodeInput} R> stopifnot(all.equal(At, t(A))) \end{CodeInput} For numeric or integer matrices the \code{adjoint()} method is equivalent to the \code{transpose()} method. For complex matrices, the adjoint is the conjugate of the transpose. In keeping with the conventions in the \pkg{Eigen} documentation the \code{adjoint()} method is used in what follows to create the transpose of numeric or integer matrices. \subsection{Products and cross-products} \label{sec:products} As mentioned in Section~\ref{sec:arrays}, the \code{`*'} operator is overloaded as matrix multiplication for the various matrix and vector classes in \pkg{Eigen}. The \proglang{C++} code in Figure~\ref{prod} produces a list containing both the product and cross-product (in the sense of the \proglang{R} function call \code{crossproduct(A, B)} evaluating $\bm A^\top\bm B$) of its two arguments % \begin{figure}[t!] \hrule \smallskip \noindent \ttfamily \hlstd{}\hlkwc{typedef\ }\hlstd{Eigen}\hlopt{::}\hlstd{Map}\hlopt{$<$}\hlstd{Eigen}\hlopt{::}\hlstd{MatrixXi}\hlopt{$>$}\hlstd{\ \ \ }\hlopt{}\hlstd{MapMati}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{MapMati}\hlstd{\ \ \ \ }\hlstd{}\hlkwd{B}\hlstd{}\hlopt{(}\hlstd{as}\hlopt{$<$}\hlstd{MapMati}\hlopt{$>$(}\hlstd{BB}\hlopt{));}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{MapMati}\hlstd{\ \ \ \ }\hlstd{}\hlkwd{C}\hlstd{}\hlopt{(}\hlstd{as}\hlopt{$<$}\hlstd{MapMati}\hlopt{$>$(}\hlstd{CC}\hlopt{));}\hspace*{\fill}\\ \hlstd{}\hlkwa{return\ }\hlstd{List}\hlopt{::}\hlstd{}\hlkwd{create}\hlstd{}\hlopt{(}\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"B\ \%{*}\%\ C"}\hlstd{}\hlopt{{)}}\hlstd{\ \ \ \ \ \ \ \ \ }\hlopt{=\ }\hlstd{B\ }\hlopt{{*}\ }\hlstd{C}\hlopt{,}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"crossprod(B,\ C)"}\hlstd{}\hlopt{{)}\ =\ }\hlstd{B}\hlopt{.}\hlstd{}\hlkwd{adjoint}\hlstd{}\hlopt{()\ {*}\ }\hlstd{C}\hlopt{);}\hlstd{}\hspace*{\fill}\\ \mbox{} \normalfont \normalsize \hrule \caption{\code{prodCpp}: Product and cross-product of two matrices. \label{prod}} \end{figure} % \begin{CodeInput} R> fprod <- cxxfunction(signature(BB = "matrix", CC = "matrix"), + prodCpp, "RcppEigen") R> B <- matrix(1:4, ncol = 2) R> C <- matrix(6:1, nrow = 2) R> str(fp <- fprod(B, C)) \end{CodeInput} \begin{CodeOutput} List of 2 $ B %*% C : int [1:2, 1:3] 21 32 13 20 5 8 $ crossprod(B, C): int [1:2, 1:3] 16 38 10 24 4 10 \end{CodeOutput} \begin{CodeInput} R> stopifnot(all.equal(fp[[1]], B %*% C), all.equal(fp[[2]], crossprod(B, C))) \end{CodeInput} % Note that the \code{create} class method for \code{Rcpp::List} implicitly applies \code{Rcpp::wrap} to its arguments. \subsection{Crossproduct of a single matrix} \label{sec:crossproduct} As shown in the last example, the \proglang{R} function \code{crossprod} calculates the product of the transpose of its first argument with its second argument. The single argument form, \code{crossprod(X)}, evaluates $\bm X^\top\bm X$. One could, of course, calculate this product as \begin{verbatim} t(X) %*% X \end{verbatim} but \code{crossprod(X)} is roughly twice as fast because the result is known to be symmetric and only one triangle needs to be calculated. The function \code{tcrossprod} evaluates \code{crossprod(t(X))} without actually forming the transpose. To express these calculations in \pkg{Eigen}, a \code{SelfAdjointView}---which is a dense matrix of which only one triangle is used, the other triangle being inferred from the symmetry---is created. (The characterization ``self-adjoint'' is equivalent to symmetric for non-complex matrices.) The \pkg{Eigen} class name is \code{SelfAdjointView}. The method for general matrices that produces such a view is called \code{selfadjointView}. Both require specification of either the \code{Lower} or \code{Upper} triangle. For triangular matrices the class is \code{TriangularView} and the method is \code{triangularView}. The triangle can be specified as \code{Lower}, \code{UnitLower}, \code{StrictlyLower}, \code{Upper}, \code{UnitUpper} or \code{StrictlyUpper}. For self-adjoint views the \code{rankUpdate} method adds a scalar multiple of $\bm A\bm A^\top$ to the current symmetric matrix. The scalar multiple defaults to 1. The code in Figure~\ref{crossprod} produces both $\bm A^\top\bm A$ and $\bm A\bm A^\top$ from a matrix $\bm A$. \begin{figure}[t!] \hrule \smallskip \noindent \ttfamily \hlstd{}\hlkwa{using\ }\hlstd{Eigen}\hlopt{::}\hlstd{Map}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwa{using\ }\hlstd{Eigen}\hlopt{::}\hlstd{MatrixXi}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwa{using\ }\hlstd{Eigen}\hlopt{::}\hlstd{Lower}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hspace*{\fill}\\ \hlkwb{const\ }\hlstd{Map}\hlopt{$<$}\hlstd{MatrixXi}\hlopt{$>$\ }\hlstd{}\hlkwd{A}\hlstd{}\hlopt{(}\hlstd{as}\hlopt{$<$}\hlstd{Map}\hlopt{$<$}\hlstd{MatrixXi}\hlopt{$>$\ $>$(}\hlstd{AA}\hlopt{));}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ int}\hlstd{\ \ \ \ \ \ \ \ \ \ \ }\hlkwb{}\hlstd{}\hlkwd{m}\hlstd{}\hlopt{(}\hlstd{A}\hlopt{.}\hlstd{}\hlkwd{rows}\hlstd{}\hlopt{()),\ }\hlstd{}\hlkwd{n}\hlstd{}\hlopt{(}\hlstd{A}\hlopt{.}\hlstd{}\hlkwd{cols}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{MatrixXi}\hlstd{\ \ \ \ \ \ \ \ \ \ }\hlstd{}\hlkwd{AtA}\hlstd{}\hlopt{(}\hlstd{}\hlkwd{MatrixXi}\hlstd{}\hlopt{(}\hlstd{n}\hlopt{,\ }\hlstd{n}\hlopt{).}\hlstd{}\hlkwd{setZero}\hlstd{}\hlopt{().}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{selfadjointView}\hlopt{$<$}\hlstd{Lower}\hlopt{$>$().}\hlstd{}\hlkwd{rankUpdate}\hlstd{}\hlopt{(}\hlstd{A}\hlopt{.}\hlstd{}\hlkwd{adjoint}\hlstd{}\hlopt{()));}\hspace*{\fill}\\ \hlstd{MatrixXi}\hlstd{\ \ \ \ \ \ \ \ \ \ }\hlstd{}\hlkwd{AAt}\hlstd{}\hlopt{(}\hlstd{}\hlkwd{MatrixXi}\hlstd{}\hlopt{(}\hlstd{m}\hlopt{,\ }\hlstd{m}\hlopt{).}\hlstd{}\hlkwd{setZero}\hlstd{}\hlopt{().}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{selfadjointView}\hlopt{$<$}\hlstd{Lower}\hlopt{$>$().}\hlstd{}\hlkwd{rankUpdate}\hlstd{}\hlopt{(}\hlstd{A}\hlopt{));}\hspace*{\fill}\\ \hlstd{}\hspace*{\fill}\\ \hlkwa{return\ }\hlstd{List}\hlopt{::}\hlstd{}\hlkwd{create}\hlstd{}\hlopt{(}\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"crossprod(A)"}\hlstd{}\hlopt{{)}}\hlstd{\ \ }\hlopt{=\ }\hlstd{AtA}\hlopt{,}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"tcrossprod(A)"}\hlstd{}\hlopt{{)}\ =\ }\hlstd{AAt}\hlopt{);}\hlstd{}\hspace*{\fill} \normalfont \normalsize \hrule \caption{\code{crossprodCpp}: Cross-product and transposed cross-product of a single matrix. \label{crossprod}} \end{figure} \begin{CodeInput} R> fcprd <- cxxfunction(signature(AA = "matrix"), crossprodCpp, "RcppEigen") R> str(crp <- fcprd(A)) \end{CodeInput} \begin{CodeOutput} List of 2 $ crossprod(A) : int [1:2, 1:2] 14 32 32 77 $ tcrossprod(A): int [1:3, 1:3] 17 22 27 22 29 36 27 36 45 \end{CodeOutput} \begin{CodeInput} R> stopifnot(all.equal(crp[[1]], crossprod(A)), + all.equal(crp[[2]], tcrossprod(A))) \end{CodeInput} To some, the expressions in Figure~\ref{crossprod} to construct \code{AtA} and \code{AAt} are compact and elegant. To others they are hopelessly confusing. If you find yourself in the latter group, you just need to read the expression left to right. So, for example, we construct \code{AAt} by creating a general integer matrix of size $m\times m$ (where $\bm A$ is $m\times n$), ensuring that all its elements are zero, regarding it as a self-adjoint (i.e., symmetric) matrix using the elements in the lower triangle, then adding $\bm A\bm A^\top$ to it and converting back to a general matrix form (i.e., the strict lower triangle is copied into the strict upper triangle). In more detail: \begin{enumerate} \item \code{MatrixXi(n, n)} creates an $n\times n$ integer matrix with arbitrary contents \item \code{.setZero()} zeros out the contents of the matrix \item \code{.selfAdjointView()} causes what follows to treat the matrix as a symmetric matrix in which only the lower triangle is used, the strict upper triangle being inferred by symmetry \item \code{.rankUpdate(A)} forms the sum $\bm B+\bm A\bm A^\top$ where $\bm B$ is the symmetric matrix of zeros created in the previous steps. \end{enumerate} The assignment of this symmetric matrix to the (general) \code{MatrixXi} object \code{AAt} causes the result to be symmetrized during the assignment. For these products one could define the symmetric matrix from either the lower triangle or the upper triangle as the result will be symmetrized before it is returned. To cut down on repetition of \code{using} statements we gather them in a character variable, \code{incl}, that will be given as the \code{includes} argument in the calls to \code{cxxfunction}. We also define a utility function, \code{AtA}, that returns the crossproduct matrix as shown in Figure~\ref{fig:incl} \begin{figure}[t!] \hrule \smallskip \noindent \ttfamily \hlstd{}\hlkwa{using}\hlstd{\ \ \ }\hlkwa{}\hlstd{Eigen}\hlopt{::}\hlstd{LLT}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwa{using}\hlstd{\ \ \ }\hlkwa{}\hlstd{Eigen}\hlopt{::}\hlstd{Lower}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwa{using}\hlstd{\ \ \ }\hlkwa{}\hlstd{Eigen}\hlopt{::}\hlstd{Map}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwa{using}\hlstd{\ \ \ }\hlkwa{}\hlstd{Eigen}\hlopt{::}\hlstd{MatrixXd}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwa{using}\hlstd{\ \ \ }\hlkwa{}\hlstd{Eigen}\hlopt{::}\hlstd{MatrixXi}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwa{using}\hlstd{\ \ \ }\hlkwa{}\hlstd{Eigen}\hlopt{::}\hlstd{Upper}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwa{using}\hlstd{\ \ \ }\hlkwa{}\hlstd{Eigen}\hlopt{::}\hlstd{VectorXd}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwc{typedef\ }\hlstd{Map}\hlopt{$<$}\hlstd{MatrixXd}\hlopt{$>$}\hlstd{\ \ }\hlopt{}\hlstd{MapMatd}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwc{typedef\ }\hlstd{Map}\hlopt{$<$}\hlstd{MatrixXi}\hlopt{$>$}\hlstd{\ \ }\hlopt{}\hlstd{MapMati}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwc{typedef\ }\hlstd{Map}\hlopt{$<$}\hlstd{VectorXd}\hlopt{$>$}\hlstd{\ \ }\hlopt{}\hlstd{MapVecd}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwc{inline\ }\hlstd{MatrixXd\ }\hlkwd{AtA}\hlstd{}\hlopt{(}\hlstd{}\hlkwb{const\ }\hlstd{MapMatd}\hlopt{\&\ }\hlstd{A}\hlopt{)\ \{}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{}\hlkwb{int}\hlstd{\ \ \ \ }\hlkwb{}\hlstd{}\hlkwd{n}\hlstd{}\hlopt{(}\hlstd{A}\hlopt{.}\hlstd{}\hlkwd{cols}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{}\hlkwa{return}\hlstd{\ \ \ }\hlkwa{}\hlstd{}\hlkwd{MatrixXd}\hlstd{}\hlopt{(}\hlstd{n}\hlopt{,}\hlstd{n}\hlopt{).}\hlstd{}\hlkwd{setZero}\hlstd{}\hlopt{().}\hlstd{selfadjointView}\hlopt{$<$}\hlstd{Lower}\hlopt{$>$()}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{}\hlopt{.}\hlstd{}\hlkwd{rankUpdate}\hlstd{}\hlopt{(}\hlstd{A}\hlopt{.}\hlstd{}\hlkwd{adjoint}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlopt{\}}\hlstd{}\hspace*{\fill}\\ \mbox{} \normalfont \normalsize \hrule \caption{The contents of the character vector, \code{incl}, that will preface \proglang{C++} code segments that follow. \label{fig:incl}} \end{figure} \subsection{Cholesky decomposition of the crossprod} \label{sec:chol} The Cholesky decomposition of the positive-definite, symmetric matrix, $\bm A$, can be written in several forms. Numerical analysts define the ``LLt'' form as the lower triangular matrix, $\bm L$, such that $\bm A=\bm L\bm L^\top$ and the ``LDLt'' form as a unit lower triangular matrix $\bm L$ and a diagonal matrix $\bm D$ with positive diagonal elements such that $\bm A=\bm L\bm D\bm L^\top$. Statisticians often write the decomposition as $\bm A=\bm R^\top\bm R$ where $\bm R$ is an upper triangular matrix. Of course, this $\bm R$ is simply the transpose of $\bm L$ from the ``LLt'' form. The templated \pkg{Eigen} classes for the LLt and LDLt forms are called \code{LLT} and \code{LDLT} and the corresponding methods are \code{.llt()} and \code{.ldlt()}. Because the Cholesky decomposition involves taking square roots, we pass a numeric matrix, $\bm A$, not an integer matrix. % \begin{figure}[t!] \hrule \smallskip \noindent \ttfamily \hlstd{}\hlkwb{const}\hlstd{\ \ }\hlkwb{}\hlstd{LLT}\hlopt{$<$}\hlstd{MatrixXd}\hlopt{$>$\ }\hlstd{}\hlkwd{llt}\hlstd{}\hlopt{(}\hlstd{}\hlkwd{AtA}\hlstd{}\hlopt{(}\hlstd{as}\hlopt{$<$}\hlstd{MapMatd}\hlopt{$>$(}\hlstd{AA}\hlopt{)));}\hspace*{\fill}\\ \hlstd{}\hlkwa{return\ }\hlstd{List}\hlopt{::}\hlstd{}\hlkwd{create}\hlstd{}\hlopt{(}\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"L"}\hlstd{}\hlopt{{)}\ =\ }\hlstd{}\hlkwd{MatrixXd}\hlstd{}\hlopt{(}\hlstd{llt}\hlopt{.}\hlstd{}\hlkwd{matrixL}\hlstd{}\hlopt{()),}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"R"}\hlstd{}\hlopt{{)}\ =\ }\hlstd{}\hlkwd{MatrixXd}\hlstd{}\hlopt{(}\hlstd{llt}\hlopt{.}\hlstd{}\hlkwd{matrixU}\hlstd{}\hlopt{()));}\hlstd{}\hspace*{\fill}\\ \mbox{} \normalfont \normalsize \hrule \caption{\code{cholCpp}: Cholesky decomposition of a cross-product. \label{chol}} \end{figure} % \begin{CodeInput} R> storage.mode(A) <- "double" R> fchol <- cxxfunction(signature(AA = "matrix"), cholCpp, "RcppEigen", incl) R> (ll <- fchol(A)) \end{CodeInput} \begin{CodeOutput} $L [,1] [,2] [1,] 3.74166 0.00000 [2,] 8.55236 1.96396 $R [,1] [,2] [1,] 3.74166 8.55236 [2,] 0.00000 1.96396 \end{CodeOutput} \begin{CodeInput} R> stopifnot(all.equal(ll[[2]], chol(crossprod(A)))) \end{CodeInput} \subsection{Determinant of the cross-product matrix} \label{sec:determinant} The ``D-optimal'' criterion for experimental design chooses the design that maximizes the determinant, $|\bm X^\top\bm X|$, for the $n\times p$ model matrix (or Jacobian matrix), $\bm X$. The determinant, $|\bm L|$, of the $p\times p$ lower Cholesky factor $\bm L$, defined so that $\bm L\bm L^\top=\bm X^\top\bm X$, is the product of its diagonal elements, as is the case for any triangular matrix. By the properties of determinants, \begin{displaymath} |\bm X^\top\bm X|=|\bm L\bm L^\top|=|\bm L|\,|\bm L^\top|=|\bm L|^2 \end{displaymath} Alternatively, if using the ``LDLt'' decomposition, $\bm L\bm D\bm L^\top=\bm X^\top\bm X$ where $\bm L$ is unit lower triangular and $\bm D$ is diagonal then $|\bm X^\top\bm X|$ is the product of the diagonal elements of $\bm D$. Because it is known that the diagonal elements of $\bm D$ must be non-negative, one often evaluates the logarithm of the determinant as the sum of the logarithms of the diagonal elements of $\bm D$. Several options are shown in Figure~\ref{cholDet}. % \begin{figure}[t!] \hrule \smallskip \noindent \ttfamily \hlstd{}\hlkwb{const\ }\hlstd{MatrixXd}\hlstd{\ \ }\hlstd{}\hlkwd{ata}\hlstd{}\hlopt{(}\hlstd{}\hlkwd{AtA}\hlstd{}\hlopt{(}\hlstd{as}\hlopt{$<$}\hlstd{MapMatd}\hlopt{$>$(}\hlstd{AA}\hlopt{)));}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ double}\hlstd{\ \ \ }\hlkwb{}\hlstd{}\hlkwd{detL}\hlstd{}\hlopt{(}\hlstd{}\hlkwd{MatrixXd}\hlstd{}\hlopt{(}\hlstd{ata}\hlopt{.}\hlstd{}\hlkwd{llt}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{matrixL}\hlstd{}\hlopt{()).}\hlstd{}\hlkwd{diagonal}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{prod}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{VectorXd\ }\hlkwd{Dvec}\hlstd{}\hlopt{(}\hlstd{ata}\hlopt{.}\hlstd{}\hlkwd{ldlt}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{vectorD}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlkwa{return\ }\hlstd{List}\hlopt{::}\hlstd{}\hlkwd{create}\hlstd{}\hlopt{(}\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"d1"}\hlstd{}\hlopt{{)}\ =\ }\hlstd{detL\ }\hlopt{{*}\ }\hlstd{detL}\hlopt{,}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"d2"}\hlstd{}\hlopt{{)}\ =\ }\hlstd{Dvec}\hlopt{.}\hlstd{}\hlkwd{prod}\hlstd{}\hlopt{(),}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"ld"}\hlstd{}\hlopt{{)}\ =\ }\hlstd{Dvec}\hlopt{.}\hlstd{}\hlkwd{array}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{log}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{sum}\hlstd{}\hlopt{());}\hlstd{}\hspace*{\fill}\\ \mbox{} \normalfont \normalsize \hrule \caption{\code{cholDetCpp}: Determinant of a cross-product using the ``LLt'' and ``LDLt'' forms of the Cholesky decomposition.} \label{cholDet} \end{figure} % \begin{CodeInput} R> fdet <- cxxfunction(signature(AA = "matrix"), cholDetCpp, "RcppEigen", + incl) R> unlist(ll <- fdet(A)) \end{CodeInput} \begin{CodeOutput} d1 d2 ld 54.00000 54.00000 3.98898 \end{CodeOutput} % Note the use of the \code{array()} method in the calculation of the log-determinant. Because the \code{log()} method applies to arrays, not to vectors or matrices, an array must be created from \code{Dvec} before applying the \code{log()} method. \section{Least squares solutions} \label{sec:leastSquares} A common operation in statistical computing is calculating a least squares solution, $\widehat{\bm\beta}$, defined as \begin{displaymath} \widehat{\bm\beta}=\argmin_{\bm \beta}\|\bm y-\bm X\bm\beta\|^2 \end{displaymath} where the model matrix, $\bm X$, is $n\times p$ ($n\ge p$) and $\bm y$ is an $n$-dimensional response vector. There are several ways, based on matrix decompositions, to determine such a solution. Earlier, two forms of the Cholesky decomposition were discussed: ``LLt'' and ``LDLt'', which can both be used to solve for $\widehat{\bm\beta}$. Other decompositions that can be used are the QR decomposition, with or without column pivoting, the singular value decomposition and the eigendecomposition of a symmetric matrix. Determining a least squares solution is relatively straightforward. However, statistical computing often requires additional information, such as the standard errors of the coefficient estimates. Calculating these involves evaluating the diagonal elements of $\left(\bm X^\top\bm X\right)^{-1}$ and the residual sum of squares, $\|\bm y-\bm X\widehat{\bm\beta}\|^2$. \subsection{Least squares using the ``LLt'' Cholesky} \label{sec:LLtLeastSquares} \begin{figure}[t!] \hrule \smallskip \noindent \ttfamily \hlstd{}\hlkwb{const\ }\hlstd{MapMatd}\hlstd{\ \ \ \ \ \ \ \ \ }\hlstd{}\hlkwd{X}\hlstd{}\hlopt{(}\hlstd{as}\hlopt{$<$}\hlstd{MapMatd}\hlopt{$>$(}\hlstd{XX}\hlopt{));}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{MapVecd}\hlstd{\ \ \ \ \ \ \ \ \ }\hlstd{}\hlkwd{y}\hlstd{}\hlopt{(}\hlstd{as}\hlopt{$<$}\hlstd{MapVecd}\hlopt{$>$(}\hlstd{yy}\hlopt{));}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ int}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ }\hlkwb{}\hlstd{}\hlkwd{n}\hlstd{}\hlopt{(}\hlstd{X}\hlopt{.}\hlstd{}\hlkwd{rows}\hlstd{}\hlopt{()),\ }\hlstd{}\hlkwd{p}\hlstd{}\hlopt{(}\hlstd{X}\hlopt{.}\hlstd{}\hlkwd{cols}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{LLT}\hlopt{$<$}\hlstd{MatrixXd}\hlopt{$>$\ }\hlstd{}\hlkwd{llt}\hlstd{}\hlopt{(}\hlstd{}\hlkwd{AtA}\hlstd{}\hlopt{(}\hlstd{X}\hlopt{));}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{VectorXd}\hlstd{\ \ }\hlstd{}\hlkwd{betahat}\hlstd{}\hlopt{(}\hlstd{llt}\hlopt{.}\hlstd{}\hlkwd{solve}\hlstd{}\hlopt{(}\hlstd{X}\hlopt{.}\hlstd{}\hlkwd{adjoint}\hlstd{}\hlopt{()\ {*}\ }\hlstd{y}\hlopt{));}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{VectorXd}\hlstd{\ \ \ }\hlstd{}\hlkwd{fitted}\hlstd{}\hlopt{(}\hlstd{X\ }\hlopt{{*}\ }\hlstd{betahat}\hlopt{);}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{VectorXd}\hlstd{\ \ \ \ }\hlstd{}\hlkwd{resid}\hlstd{}\hlopt{(}\hlstd{y\ }\hlopt{{-}\ }\hlstd{fitted}\hlopt{);}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ int}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ }\hlkwb{}\hlstd{}\hlkwd{df}\hlstd{}\hlopt{(}\hlstd{n\ }\hlopt{{-}\ }\hlstd{p}\hlopt{);}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ double}\hlstd{\ \ \ \ \ \ \ \ \ \ }\hlkwb{}\hlstd{}\hlkwd{s}\hlstd{}\hlopt{(}\hlstd{resid}\hlopt{.}\hlstd{}\hlkwd{norm}\hlstd{}\hlopt{()\ /\ }\hlstd{std}\hlopt{::}\hlstd{}\hlkwd{sqrt}\hlstd{}\hlopt{(}\hlstd{}\hlkwb{double}\hlstd{}\hlopt{(}\hlstd{df}\hlopt{)));}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{VectorXd}\hlstd{\ \ \ \ \ \ \ }\hlstd{}\hlkwd{se}\hlstd{}\hlopt{(}\hlstd{s\ }\hlopt{{*}\ }\hlstd{llt}\hlopt{.}\hlstd{}\hlkwd{matrixL}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{solve}\hlstd{}\hlopt{(}\hlstd{MatrixXd}\hlopt{::}\hlstd{}\hlkwd{Identity}\hlstd{}\hlopt{(}\hlstd{p}\hlopt{,\ }\hlstd{p}\hlopt{))}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{}\hlopt{.}\hlstd{}\hlkwd{colwise}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{norm}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlkwa{return}\hlstd{\ \ \ \ \ }\hlkwa{}\hlstd{List}\hlopt{::}\hlstd{}\hlkwd{create}\hlstd{}\hlopt{(}\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"coefficients"}\hlstd{}\hlopt{{)}}\hlstd{\ \ \ }\hlopt{=\ }\hlstd{betahat}\hlopt{,}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"fitted.values"}\hlstd{}\hlopt{{)}}\hlstd{\ \ }\hlopt{=\ }\hlstd{fitted}\hlopt{,}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"residuals"}\hlstd{}\hlopt{{)}}\hlstd{\ \ \ \ \ \ }\hlopt{=\ }\hlstd{resid}\hlopt{,}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"s"}\hlstd{}\hlopt{{)}}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlopt{=\ }\hlstd{s}\hlopt{,}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"df.residual"}\hlstd{}\hlopt{{)}}\hlstd{\ \ \ \ }\hlopt{=\ }\hlstd{df}\hlopt{,}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"rank"}\hlstd{}\hlopt{{)}}\hlstd{\ \ \ \ \ \ \ \ \ \ \ }\hlopt{=\ }\hlstd{p}\hlopt{,}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"Std.\ Error"}\hlstd{}\hlopt{{)}}\hlstd{\ \ \ \ \ }\hlopt{=\ }\hlstd{se}\hlopt{);}\hlstd{}\hspace*{\fill}\\ \mbox{} \normalfont \normalsize \hrule \caption{\code{lltLSCpp}: Least squares using the Cholesky decomposition. \label{lltLS}} \end{figure} Figure~\ref{lltLS} shows a calculation of the least squares coefficient estimates (\code{betahat}) and the standard errors (\code{se}) through an ``LLt'' Cholesky decomposition of the crossproduct of the model matrix, $\bm X$. Next, the results from this calculation are compared to those from the \code{lm.fit} function in \proglang{R} (\code{lm.fit} is the workhorse function called by \code{lm} once the model matrix and response have been evaluated). \begin{CodeInput} R> lltLS <- cxxfunction(signature(XX = "matrix", yy = "numeric"), lltLSCpp, + "RcppEigen", incl) R> data("trees", package = "datasets") R> str(lltFit <- with(trees, lltLS(cbind(1, log(Girth)), log(Volume)))) \end{CodeInput} \begin{CodeOutput} List of 7 $ coefficients : num [1:2] -2.35 2.2 $ fitted.values: num [1:31] 2.3 2.38 2.43 2.82 2.86 ... $ residuals : num [1:31] 0.0298 -0.0483 -0.1087 -0.0223 0.0727 ... $ s : num 0.115 $ df.residual : int 29 $ rank : int 2 $ Std. Error : num [1:2] 0.2307 0.0898 R> str(lmFit <- with(trees, lm.fit(cbind(1, log(Girth)), log(Volume)))) List of 8 $ coefficients : Named num [1:2] -2.35 2.2 ..- attr(*, "names")= chr [1:2] "x1" "x2" $ residuals : num [1:31] 0.0298 -0.0483 -0.1087 -0.0223 0.0727 ... $ effects : Named num [1:31] -18.2218 2.8152 -0.1029 -0.0223 0.0721 ... ..- attr(*, "names")= chr [1:31] "x1" "x2" "" "" ... $ rank : int 2 $ fitted.values: num [1:31] 2.3 2.38 2.43 2.82 2.86 ... $ assign : NULL $ qr :List of 5 ..$ qr : num [1:31, 1:2] -5.57 0.18 0.18 0.18 0.18 ... ..$ qraux: num [1:2] 1.18 1.26 ..$ pivot: int [1:2] 1 2 ..$ tol : num 1e-07 ..$ rank : int 2 ..- attr(*, "class")= chr "qr" $ df.residual : int 29 \end{CodeOutput} \begin{CodeInput} R> for(nm in c("coefficients", "residuals", "fitted.values", "rank")) + stopifnot(all.equal(lltFit[[nm]], unname(lmFit[[nm]]))) R> stopifnot(all.equal(lltFit[["Std. Error"]], + unname(coef(summary(lm(log(Volume) ~ log(Girth), trees)))[,2]))) \end{CodeInput} There are several aspects of the \proglang{C++} code in Figure~\ref{lltLS} worth mentioning. The \code{solve} method for the \code{LLT} object evaluates, in this case, $\left(\bm X^\top\bm X\right)^{-1}\bm X^\top\bm y$ but without actually evaluating the inverse. The calculation of the residuals, $\bm y-\widehat{\bm y}$, can be written, as in \proglang{R}, as \code{y - fitted}. (But note that \pkg{Eigen} classes do not have a ``recycling rule'' as in \proglang{R}. That is, the two vector operands must have the same length.) The \code{norm()} method evaluates the square root of the sum of squares of the elements of a vector. Although one does not explicitly evaluate $\left(\bm X^\top\bm X\right)^{-1}$ one does evaluate $\bm L^{-1}$ to obtain the standard errors. Note also the use of the \code{colwise()} method in the evaluation of the standard errors. It applies a method to the columns of a matrix, returning a vector. The \pkg{Eigen} \code{colwise()} and \code{rowwise()} methods are similar in effect to the \code{apply} function in \proglang{R}. In the descriptions of other methods for solving least squares problems, much of the code parallels that shown in Figure~\ref{lltLS}. The redundant parts are omitted, and only the evaluation of the coefficients, the rank and the standard errors is shown. Actually, the standard errors are calculated only up to the scalar multiple of $s$, the residual standard error, in these code fragments. The calculation of the residuals and $s$ and the scaling of the coefficient standard errors is the same for all methods. (See the files \code{fastLm.h} and \code{fastLm.cpp} in the \pkg{RcppEigen} source package for details.) \subsection{Least squares using the unpivoted QR decomposition} \label{sec:QR} A QR decomposition has the form \begin{displaymath} \bm X=\bm Q\bm R=\bm Q_1\bm R_1 \end{displaymath} where $\bm Q$ is an $n\times n$ orthogonal matrix, which means that $\bm Q^\top\bm Q=\bm Q\bm Q^\top=\bm I_n$, and the $n\times p$ matrix $\bm R$ is zero below the main diagonal. The $n\times p$ matrix $\bm Q_1$ is the first $p$ columns of $\bm Q$ and the $p\times p$ upper triangular matrix $\bm R_1$ is the top $p$ rows of $\bm R$. There are three \pkg{Eigen} classes for the QR decomposition: \code{HouseholderQR} provides the basic QR decomposition using Householder transformations, \code{ColPivHouseholderQR} incorporates column pivots and \code{FullPivHouseholderQR} incorporates both row and column pivots. Figure~\ref{QRLS} shows a least squares solution using the unpivoted QR decomposition. The calculations in Figure~\ref{QRLS} are quite similar to those in Figure~\ref{lltLS}. In fact, if one had extracted the upper triangular factor (the \code{matrixU()} method) from the \code{LLT} object in Figure~\ref{lltLS}, the rest of the code would be nearly identical. \begin{figure}[t!] \hrule \smallskip \noindent \ttfamily \hlstd{}\hlkwa{using\ }\hlstd{Eigen}\hlopt{::}\hlstd{HouseholderQR}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hspace*{\fill}\\ \hlkwb{const\ }\hlstd{HouseholderQR}\hlopt{$<$}\hlstd{MatrixXd}\hlopt{$>$\ }\hlstd{}\hlkwd{QR}\hlstd{}\hlopt{(}\hlstd{X}\hlopt{);}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{VectorXd\ }\hlkwd{betahat}\hlstd{}\hlopt{(}\hlstd{QR}\hlopt{.}\hlstd{}\hlkwd{solve}\hlstd{}\hlopt{(}\hlstd{y}\hlopt{));}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{VectorXd}\hlstd{\ \ }\hlstd{}\hlkwd{fitted}\hlstd{}\hlopt{(}\hlstd{X\ }\hlopt{{*}\ }\hlstd{betahat}\hlopt{);}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ int}\hlstd{\ \ \ \ \ \ \ \ \ \ \ }\hlkwb{}\hlstd{}\hlkwd{df}\hlstd{}\hlopt{(}\hlstd{n\ }\hlopt{{-}\ }\hlstd{p}\hlopt{);}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{VectorXd}\hlstd{\ \ \ \ \ \ }\hlstd{}\hlkwd{se}\hlstd{}\hlopt{(}\hlstd{QR}\hlopt{.}\hlstd{}\hlkwd{matrixQR}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{topRows}\hlstd{}\hlopt{(}\hlstd{p}\hlopt{).}\hlstd{triangularView}\hlopt{$<$}\hlstd{Upper}\hlopt{$>$()}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{}\hlopt{.}\hlstd{}\hlkwd{solve}\hlstd{}\hlopt{(}\hlstd{MatrixXd}\hlopt{::}\hlstd{}\hlkwd{Identity}\hlstd{}\hlopt{(}\hlstd{p}\hlopt{,}\hlstd{p}\hlopt{)).}\hlstd{}\hlkwd{rowwise}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{norm}\hlstd{}\hlopt{());}\hlstd{}\hspace*{\fill}\\ \mbox{} \normalfont \normalsize \hrule \caption{\code{QRLSCpp}: Least squares using the unpivoted QR decomposition. \label{QRLS}} \end{figure} \subsection{Handling the rank-deficient case} \label{sec:rankdeficient} One important consideration when determining least squares solutions is whether $\rank(\bm X)$ is $p$, a situation described by saying that $\bm X$ has ``full column rank''. When $\bm X$ does not have full column rank it is said to be ``rank deficient''. Although the theoretical rank of a matrix is well-defined, its evaluation in practice is not. At best one can compute an effective rank according to some tolerance. Decompositions that allow to estimation of the rank of the matrix in this way are said to be ``rank-revealing''. Because the \code{model.matrix} function in \proglang{R} does a considerable amount of symbolic analysis behind the scenes, one usually ends up with full-rank model matrices. The common cases of rank-deficiency, such as incorporating both a constant term and a full set of indicators columns for the levels of a factor, are eliminated. Other, more subtle, situations will not be detected at this stage, however. A simple example occurs when there is a ``missing cell'' in a two-way layout and the interaction of the two factors is included in the model. \begin{CodeInput} R> dd <- data.frame(f1 = gl(4, 6, labels = LETTERS[1:4]), + f2 = gl(3, 2, labels = letters[1:3]))[-(7:8), ] R> xtabs(~ f2 + f1, dd) \end{CodeInput} \begin{CodeOutput} f1 f2 A B C D a 2 0 2 2 b 2 2 2 2 c 2 2 2 2 \end{CodeOutput} \begin{CodeInput} R> mm <- model.matrix(~ f1 * f2, dd) R> kappa(mm) \end{CodeInput} \begin{CodeOutput} [1] 4.30923e+16 \end{CodeOutput} This yields a large condition number, indicating rank deficiency. Alternatively, the reciprocal condition number can be evaluated. \begin{CodeInput} R> rcond(mm) \end{CodeInput} \begin{CodeOutput} [1] 2.3206e-17 \end{CodeOutput} \begin{CodeInput} R> (c(rank = qr(mm)$rank, p = ncol(mm))) \end{CodeInput} \begin{CodeOutput} rank p 11 12 \end{CodeOutput} \begin{CodeInput} R> set.seed(1) R> dd$y <- mm %*% seq_len(ncol(mm)) + rnorm(nrow(mm), sd = 0.1) R> fm1 <- lm(y ~ f1 * f2, dd) R> writeLines(capture.output(print(summary(fm1), + signif.stars = FALSE))[9:22]) \end{CodeInput} \begin{CodeOutput} Coefficients: (1 not defined because of singularities) Estimate Std. Error t value Pr(>|t|) (Intercept) 0.9779 0.0582 16.8 3.4e-09 f1B 12.0381 0.0823 146.3 < 2e-16 f1C 3.1172 0.0823 37.9 5.2e-13 f1D 4.0685 0.0823 49.5 2.8e-14 f2b 5.0601 0.0823 61.5 2.6e-15 f2c 5.9976 0.0823 72.9 4.0e-16 f1B:f2b -3.0148 0.1163 -25.9 3.3e-11 f1C:f2b 7.7030 0.1163 66.2 1.2e-15 f1D:f2b 8.9643 0.1163 77.1 < 2e-16 f1B:f2c NA NA NA NA f1C:f2c 10.9613 0.1163 94.2 < 2e-16 f1D:f2c 12.0411 0.1163 103.5 < 2e-16 \end{CodeOutput} The \code{lm} function for fitting linear models in \proglang{R} uses a rank-revealing form of the QR decomposition. When the model matrix is determined to be rank deficient, according to the threshold used in \proglang{R}'s \code{qr} function, the model matrix is reduced to $\rank{(\bm X)}$ columns by pivoting selected columns (those that are apparently linearly dependent on columns to their left) to the right hand side of the matrix. A solution for this reduced model matrix is determined and the coefficients and standard errors for the redundant columns are flagged as missing. An alternative approach is to evaluate the ``pseudo-inverse'' of $\bm X$ from the singular value decomposition (SVD) of $\bm X$ or the eigendecomposition of $\bm X^\top\bm X$. The SVD is of the form \begin{displaymath} \bm X=\bm U\bm D\bm V^\top=\bm U_1\bm D_1\bm V^\top \end{displaymath} where $\bm U$ is an orthogonal $n\times n$ matrix and $\bm U_1$ is its leftmost $p$ columns, $\bm D$ is $n\times p$ and zero off the main diagonal so that $\bm D_1$ is a $p\times p$ diagonal matrix with non-increasing, non-negative diagonal elements, and $\bm V$ is a $p\times p$ orthogonal matrix. The pseudo-inverse of $\bm D_1$, written $\bm D_1^+$ is a $p\times p$ diagonal matrix whose first $r=\rank(\bm X)$ diagonal elements are the inverses of the corresponding diagonal elements of $\bm D_1$ and whose last $p-r$ diagonal elements are zero. The tolerance for determining if an element of the diagonal of $\bm D_1$ is considered to be (effectively) zero is a multiple of the largest singular value (i.e., the $(1,1)$ element of $\bm D$). The pseudo-inverse, $\bm X^+$, of $\bm X$ is defined as \begin{displaymath} \bm X^+=\bm V\bm D_1^+\bm U_1^\top . \end{displaymath} \begin{figure}[t!] \hrule \smallskip \noindent \ttfamily \hlstd{}\hlkwc{inline\ }\hlstd{ArrayXd\ }\hlkwd{Dplus}\hlstd{}\hlopt{(}\hlstd{}\hlkwb{const\ }\hlstd{ArrayXd}\hlopt{\&\ }\hlstd{d}\hlopt{)\ \{}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{ArrayXd}\hlstd{\ \ \ }\hlstd{}\hlkwd{di}\hlstd{}\hlopt{(}\hlstd{d}\hlopt{.}\hlstd{}\hlkwd{size}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{}\hlkwb{double}\hlstd{\ \ }\hlkwb{}\hlstd{}\hlkwd{comp}\hlstd{}\hlopt{(}\hlstd{d}\hlopt{.}\hlstd{}\hlkwd{maxCoeff}\hlstd{}\hlopt{()\ {*}\ }\hlstd{}\hlkwd{threshold}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{}\hlkwa{for\ }\hlstd{}\hlopt{(}\hlstd{}\hlkwb{int\ }\hlstd{j\ }\hlopt{=\ }\hlstd{}\hlnum{0}\hlstd{}\hlopt{;\ }\hlstd{j\ }\hlopt{$<$\ }\hlstd{d}\hlopt{.}\hlstd{}\hlkwd{size}\hlstd{}\hlopt{();\ ++}\hlstd{j}\hlopt{)\ }\hlstd{di}\hlopt{{[}}\hlstd{j}\hlopt{{]}\ =\ (}\hlstd{d}\hlopt{{[}}\hlstd{j}\hlopt{{]}\ $<$\ }\hlstd{comp}\hlopt{)\ }\hlstd{?\ }\hlnum{0}\hlstd{}\hlopt{.\ :\ }\hlstd{}\hlnum{1}\hlstd{}\hlopt{./}\hlstd{d}\hlopt{{[}}\hlstd{j}\hlopt{{]};}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{}\hlkwa{return\ }\hlstd{di}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlopt{\}}\hlstd{}\hspace*{\fill}\\ \mbox{} \normalfont \normalsize \hrule \caption{\code{DplusCpp}: Create the diagonal $\bm d^+$ of the pseudo-inverse, $\bm D_1^+$, from the array of singular values, $\bm d$. \label{Dplus}} \end{figure} In Figure~\ref{Dplus} a utility function, \code{Dplus}, is defined to return the diagonal of the pseudo-inverse, $\bm D_1^+$, as an array, given the singular values (the diagonal of $\bm D_1$) as an array. Calculation of the maximum element of $\bm d$ (the method is called \code{.maxCoeff()}) and the use of a \code{threshold()} function provides greater generality for the function. It can be used on the eigenvalues of $\bm X^\top\bm X$, as shown in Section~\ref{sec:eigendecomp}, even though these are returned in increasing order, as opposed to the decreasing order of the singular values. \subsection{Least squares using the SVD} \label{sec:SVDls} \begin{figure}[b!] \hrule \smallskip \noindent \ttfamily \hlkwb{const\ }\hlstd{Eigen}\hlopt{::}\hlstd{JacobiSVD}\hlopt{$<$}\hlstd{MatrixXd}\hlopt{$>$}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ }\hlstd{}\hlkwd{UDV}\hlstd{}\hlopt{(}\hlstd{X}\hlopt{.}\hlstd{}\hlkwd{jacobiSvd}\hlstd{}\hlopt{(}\hlstd{Eigen}\hlopt{::}\hlstd{ComputeThinU}\hlopt{\textbar }\hlstd{Eigen}\hlopt{::}\hlstd{ComputeThinV}\hlopt{));}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{ArrayXd}\hlstd{\ \ \ \ \ \ \ }\hlstd{}\hlkwd{Dp}\hlstd{}\hlopt{(}\hlstd{}\hlkwd{Dplus}\hlstd{}\hlopt{(}\hlstd{UDV}\hlopt{.}\hlstd{}\hlkwd{singularValues}\hlstd{}\hlopt{()));}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ int}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ }\hlkwb{}\hlstd{}\hlkwd{r}\hlstd{}\hlopt{((}\hlstd{Dp\ }\hlopt{$>$\ }\hlstd{}\hlnum{0}\hlstd{}\hlopt{).}\hlstd{}\hlkwd{count}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{MatrixXd}\hlstd{\ \ \ \ \ }\hlstd{}\hlkwd{VDp}\hlstd{}\hlopt{(}\hlstd{UDV}\hlopt{.}\hlstd{}\hlkwd{matrixV}\hlstd{}\hlopt{()\ {*}\ }\hlstd{Dp}\hlopt{.}\hlstd{}\hlkwd{matrix}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{asDiagonal}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{VectorXd\ }\hlkwd{betahat}\hlstd{}\hlopt{(}\hlstd{VDp\ }\hlopt{{*}\ }\hlstd{UDV}\hlopt{.}\hlstd{}\hlkwd{matrixU}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{adjoint}\hlstd{}\hlopt{()\ {*}\ }\hlstd{y}\hlopt{);}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{VectorXd}\hlstd{\ \ \ \ \ \ }\hlstd{}\hlkwd{se}\hlstd{}\hlopt{(}\hlstd{s\ }\hlopt{{*}\ }\hlstd{VDp}\hlopt{.}\hlstd{}\hlkwd{rowwise}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{norm}\hlstd{}\hlopt{());}\hlstd{}\hspace*{\fill}\\ \mbox{} \normalfont \normalsize \hrule \caption{\code{SVDLSCpp}: Least squares using the SVD. \label{SVDLS}} \end{figure} With these definitions the code for least squares using the singular value decomposition can be written as in Figure~\ref{SVDLS}. Even in the rank-deficient case this code will produce a complete set of coefficients and their standard errors. The user must be careful to check if the computed rank is less than $p$, the number of columns in $\bm X$, in which case the estimated coefficients are just one of an infinite number of coefficient vectors that produce the same fitted values. It happens that the solution from this pseudo-inverse is the minimum norm solution (in the sense that $\|\bm\beta\|^2$ is minimized among those $\bm\beta$ that minimize $\|\bm y-\bm X\bm\beta\|^2$). The standard errors of the coefficient estimates in the rank-deficient case must be interpreted carefully. The solution with one or more missing coefficients, as returned by the \code{lm.fit} function in \proglang{R} and by the column-pivoted QR decomposition described in Section~\ref{sec:colPivQR}, does not provide standard errors for the missing coefficients. That is, both the coefficient and its standard error are returned as \code{NA} because the least squares solution is performed on a reduced model matrix. It is also true that the solution returned by the SVD method is with respect to a reduced model matrix but the $p$ coefficient estimates and their $p$ standard errors don't show this. They are, in fact, linear combinations of a set of $r$ coefficient estimates and their standard errors. \pagebreak \subsection{Least squares using the eigendecomposition} \label{sec:eigendecomp} The eigendecomposition of $\bm X^\top\bm X$ is defined as \begin{displaymath} \bm X^\top\bm X=\bm V\bm\Lambda\bm V^\top \end{displaymath} where $\bm V$, the matrix of eigenvectors, is a $p\times p$ orthogonal matrix and $\bm\Lambda$ is a $p\times p$ diagonal matrix with non-decreasing, non-negative diagonal elements, called the eigenvalues of $\bm X^\top\bm X$. When the eigenvalues are distinct, this $\bm V$ is the same (up to reordering of the columns) as that in the SVD and the eigenvalues of $\bm X^\top\bm X$ are the (reordered) squares of the singular values of $\bm X$. With these definitions one can adapt much of the code from the SVD method for the eigendecomposition, as shown in Figure~\ref{SymmEigLS}. \begin{figure}[t!] \hrule \smallskip \noindent \ttfamily \hlkwb{const\ }\hlstd{Eigen}\hlopt{::}\hlstd{SelfAdjointEigenSolver}\hlopt{$<$}\hlstd{MatrixXd}\hlopt{$>$\ }\hlstd{}\hlkwd{VLV}\hlstd{}\hlopt{(}\hlstd{}\hlkwd{AtA}\hlstd{}\hlopt{(}\hlstd{X}\hlopt{));}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{ArrayXd}\hlstd{\ \ \ \ \ \ \ }\hlstd{}\hlkwd{Dp}\hlstd{}\hlopt{(}\hlstd{}\hlkwd{Dplus}\hlstd{}\hlopt{(}\hlstd{eig}\hlopt{.}\hlstd{}\hlkwd{eigenvalues}\hlstd{}\hlopt{()).}\hlstd{}\hlkwd{sqrt}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ int}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ }\hlkwb{}\hlstd{}\hlkwd{r}\hlstd{}\hlopt{((}\hlstd{Dp\ }\hlopt{$>$\ }\hlstd{}\hlnum{0}\hlstd{}\hlopt{).}\hlstd{}\hlkwd{count}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{MatrixXd}\hlstd{\ \ \ \ \ }\hlstd{}\hlkwd{VDp}\hlstd{}\hlopt{(}\hlstd{VLV}\hlopt{.}\hlstd{}\hlkwd{eigenvectors}\hlstd{}\hlopt{()\ {*}\ }\hlstd{Dp}\hlopt{.}\hlstd{}\hlkwd{matrix}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{asDiagonal}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{VectorXd\ }\hlkwd{betahat}\hlstd{}\hlopt{(}\hlstd{VDp\ }\hlopt{{*}\ }\hlstd{VDp}\hlopt{.}\hlstd{}\hlkwd{adjoint}\hlstd{}\hlopt{()\ {*}\ }\hlstd{X}\hlopt{.}\hlstd{}\hlkwd{adjoint}\hlstd{}\hlopt{()\ {*}\ }\hlstd{y}\hlopt{);}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{VectorXd}\hlstd{\ \ \ \ \ \ }\hlstd{}\hlkwd{se}\hlstd{}\hlopt{(}\hlstd{s\ }\hlopt{{*}\ }\hlstd{VDp}\hlopt{.}\hlstd{}\hlkwd{rowwise}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{norm}\hlstd{}\hlopt{());}\hlstd{}\hspace*{\fill}\\ \mbox{} \normalfont \normalsize \hrule \caption{\code{SymmEigLSCpp}: Least squares using the eigendecomposition. \label{SymmEigLS}} \end{figure} \subsection{Least squares using the column-pivoted QR decomposition} \label{sec:colPivQR} The column-pivoted QR decomposition provides results similar to those from \proglang{R} in both the full-rank and the rank-deficient cases. The decomposition is of the form \begin{displaymath} \bm X\bm P=\bm Q\bm R=\bm Q_1\bm R_1 \end{displaymath} where, as before, $\bm Q$ is $n\times n$ and orthogonal and $\bm R$ is $n\times p$ and upper triangular. The $p\times p$ matrix $\bm P$ is a permutation matrix. That is, its columns are a permutation of the columns of $\bm I_p$. It serves to reorder the columns of $\bm X$ so that the diagonal elements of $\bm R$ are non-increasing in magnitude. An instance of the class \code{Eigen::ColPivHouseholderQR} has a \code{rank()} method returning the computational rank of the matrix. When $\bm X$ is of full rank one can use essentially the same code as in the unpivoted decomposition except that one must reorder the standard errors. When $\bm X$ is rank-deficient, the coefficients and standard errors are evaluated for the leading $r$ columns of $\bm X\bm P$ only. In the rank-deficient case the straightforward calculation of the fitted values, as $\bm X\widehat{\bm\beta}$, cannot be used because some of the estimated coefficients, $\widehat{\bm\beta}$, are \code{NA} so the product will be a vector of \code{NA}'s. One could do some complicated rearrangement of the columns of X and the coefficient estimates but it is conceptually (and computationally) easier to employ the relationship \begin{displaymath} \widehat{\bm y} = \bm Q_1\bm Q_1^\top\bm y=\bm Q \begin{bmatrix} \bm I_r & \bm 0\\ \bm 0 & \bm 0 \end{bmatrix} \bm Q^\top\bm y \end{displaymath} The vector $\bm Q^\top\bm y$ is called the ``effects'' vector in \proglang{R}. \begin{figure}[t!] \hrule \smallskip \noindent \ttfamily \hlstd{}\hlkwc{typedef\ }\hlstd{Eigen}\hlopt{::}\hlstd{ColPivHouseholderQR}\hlopt{$<$}\hlstd{MatrixXd}\hlopt{$>$}\hlstd{\ \ }\hlopt{}\hlstd{CPivQR}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwc{typedef\ }\hlstd{CPivQR}\hlopt{::}\hlstd{PermutationType}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{Permutation}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hspace*{\fill}\\ \hlkwb{const\ }\hlstd{CPivQR}\hlstd{\ \ \ \ \ \ \ }\hlstd{}\hlkwd{PQR}\hlstd{}\hlopt{(}\hlstd{X}\hlopt{);}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{Permutation\ }\hlkwd{Pmat}\hlstd{}\hlopt{(}\hlstd{PQR}\hlopt{.}\hlstd{}\hlkwd{colsPermutation}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ int}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ }\hlkwb{}\hlstd{}\hlkwd{r}\hlstd{}\hlopt{(}\hlstd{PQR}\hlopt{.}\hlstd{}\hlkwd{rank}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{VectorXd}\hlstd{\ \ \ \ \ \ \ }\hlstd{betahat}\hlopt{,\ }\hlstd{fitted}\hlopt{,\ }\hlstd{se}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwa{if\ }\hlstd{}\hlopt{(}\hlstd{r\ }\hlopt{==\ }\hlstd{X}\hlopt{.}\hlstd{}\hlkwd{cols}\hlstd{}\hlopt{())\ \{\ }\hlstd{}\hlslc{//\ full\ rank\ case}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{betahat\ }\hlopt{=\ }\hlstd{PQR}\hlopt{.}\hlstd{}\hlkwd{solve}\hlstd{}\hlopt{(}\hlstd{y}\hlopt{);}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{fitted}\hlstd{\ \ }\hlstd{}\hlopt{=\ }\hlstd{X\ }\hlopt{{*}\ }\hlstd{betahat}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{se}\hlstd{\ \ \ \ \ \ }\hlstd{}\hlopt{=\ }\hlstd{Pmat\ }\hlopt{{*}\ }\hlstd{PQR}\hlopt{.}\hlstd{}\hlkwd{matrixQR}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{topRows}\hlstd{}\hlopt{(}\hlstd{p}\hlopt{).}\hlstd{triangularView}\hlopt{$<$}\hlstd{Upper}\hlopt{$>$()}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{}\hlopt{.}\hlstd{}\hlkwd{solve}\hlstd{}\hlopt{(}\hlstd{MatrixXd}\hlopt{::}\hlstd{}\hlkwd{Identity}\hlstd{}\hlopt{(}\hlstd{p}\hlopt{,\ }\hlstd{p}\hlopt{)).}\hlstd{}\hlkwd{rowwise}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{norm}\hlstd{}\hlopt{();}\hspace*{\fill}\\ \hlstd{}\hlopt{\}\ }\hlstd{}\hlkwa{else\ }\hlstd{}\hlopt{\{}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{MatrixXd}\hlstd{\ \ \ \ }\hlstd{}\hlkwd{Rinv}\hlstd{}\hlopt{(}\hlstd{PQR}\hlopt{.}\hlstd{}\hlkwd{matrixQR}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{topLeftCorner}\hlstd{}\hlopt{(}\hlstd{r}\hlopt{,\ }\hlstd{r}\hlopt{)}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ }\hlstd{}\hlopt{.}\hlstd{triangularView}\hlopt{$<$}\hlstd{Upper}\hlopt{$>$().}\hlstd{}\hlkwd{solve}\hlstd{}\hlopt{(}\hlstd{MatrixXd}\hlopt{::}\hlstd{}\hlkwd{Identity}\hlstd{}\hlopt{(}\hlstd{r}\hlopt{,\ }\hlstd{r}\hlopt{)));}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{VectorXd\ }\hlkwd{effects}\hlstd{}\hlopt{(}\hlstd{PQR}\hlopt{.}\hlstd{}\hlkwd{householderQ}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{adjoint}\hlstd{}\hlopt{()\ {*}\ }\hlstd{y}\hlopt{);}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{betahat}\hlopt{.}\hlstd{}\hlkwd{fill}\hlstd{}\hlopt{(::}\hlstd{NA\textunderscore REAL}\hlopt{);}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{betahat}\hlopt{.}\hlstd{}\hlkwd{head}\hlstd{}\hlopt{(}\hlstd{r}\hlopt{)}\hlstd{\ \ }\hlopt{=\ }\hlstd{Rinv\ }\hlopt{{*}\ }\hlstd{effects}\hlopt{.}\hlstd{}\hlkwd{head}\hlstd{}\hlopt{(}\hlstd{r}\hlopt{);}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{betahat}\hlstd{\ \ \ \ \ \ \ \ \ \ }\hlstd{}\hlopt{=\ }\hlstd{Pmat\ }\hlopt{{*}\ }\hlstd{betahat}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{se}\hlopt{.}\hlstd{}\hlkwd{fill}\hlstd{}\hlopt{(::}\hlstd{NA\textunderscore REAL}\hlopt{);}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{se}\hlopt{.}\hlstd{}\hlkwd{head}\hlstd{}\hlopt{(}\hlstd{r}\hlopt{)}\hlstd{\ \ \ \ \ \ \ }\hlopt{=\ }\hlstd{Rinv}\hlopt{.}\hlstd{}\hlkwd{rowwise}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{norm}\hlstd{}\hlopt{();}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{se}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{}\hlopt{=\ }\hlstd{Pmat\ }\hlopt{{*}\ }\hlstd{se}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{}\hlslc{//\ create\ fitted\ values\ from\ effects}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{effects}\hlopt{.}\hlstd{}\hlkwd{tail}\hlstd{}\hlopt{(}\hlstd{X}\hlopt{.}\hlstd{}\hlkwd{rows}\hlstd{}\hlopt{()\ {-}\ }\hlstd{r}\hlopt{).}\hlstd{}\hlkwd{setZero}\hlstd{}\hlopt{();}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ }\hlstd{fitted}\hlstd{\ \ \ \ \ \ \ \ \ \ \ }\hlstd{}\hlopt{=\ }\hlstd{PQR}\hlopt{.}\hlstd{}\hlkwd{householderQ}\hlstd{}\hlopt{()\ {*}\ }\hlstd{effects}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlopt{\}}\hlstd{}\hspace*{\fill}\\ \mbox{} \normalfont \normalsize \hrule \caption{\code{ColPivQRLSCpp}: Least squares using the pivoted QR decomposition. \label{ColPivQRLS}} \end{figure} Just to check that the code in Figure~\ref{ColPivQRLS} does indeed provide the desired answer \begin{CodeInput} R> print(summary(fmPQR <- fastLm(y ~ f1 * f2, dd)), signif.st = FALSE) \end{CodeInput} \begin{CodeOutput} Call: fastLm.formula(formula = y ~ f1 * f2, data = dd) Estimate Std. Error t value Pr(>|t|) (Intercept) 0.977859 0.058165 16.812 3.413e-09 f1B 12.038068 0.082258 146.346 < 2.2e-16 f1C 3.117222 0.082258 37.896 5.221e-13 f1D 4.068523 0.082258 49.461 2.833e-14 f2b 5.060123 0.082258 61.516 2.593e-15 f2c 5.997592 0.082258 72.912 4.015e-16 f1B:f2b -3.014763 0.116330 -25.916 3.266e-11 f1C:f2b 7.702999 0.116330 66.217 1.156e-15 f1D:f2b 8.964251 0.116330 77.059 < 2.2e-16 f1B:f2c NA NA NA NA f1C:f2c 10.961326 0.116330 94.226 < 2.2e-16 f1D:f2c 12.041081 0.116330 103.508 < 2.2e-16 Residual standard error: 0.2868 on 11 degrees of freedom Multiple R-squared: 0.9999, Adjusted R-squared: 0.9999 \end{CodeOutput} \begin{CodeInput} R> all.equal(coef(fm1), coef(fmPQR)) \end{CodeInput} \begin{CodeOutput} [1] TRUE \end{CodeOutput} \begin{CodeInput} R> all.equal(unname(fitted(fm1)), fitted(fmPQR)) \end{CodeInput} \begin{CodeOutput} [1] TRUE \end{CodeOutput} \begin{CodeInput} R> all.equal(unname(residuals(fm1)), residuals(fmPQR)) \end{CodeInput} \begin{CodeOutput} [1] TRUE \end{CodeOutput} The rank-revealing SVD method produces the same fitted values but not the same coefficients. \begin{CodeInput} R> print(summary(fmSVD <- fastLm(y ~ f1 * f2, dd, method = 4)), + signif.st = FALSE) \end{CodeInput} \begin{CodeOutput} Call: fastLm.formula(formula = y ~ f1 * f2, data = dd, method = 4) Estimate Std. Error t value Pr(>|t|) (Intercept) 0.977859 0.058165 16.812 3.413e-09 f1B 7.020458 0.038777 181.049 < 2.2e-16 f1C 3.117222 0.082258 37.896 5.221e-13 f1D 4.068523 0.082258 49.461 2.833e-14 f2b 5.060123 0.082258 61.516 2.593e-15 f2c 5.997592 0.082258 72.912 4.015e-16 f1B:f2b 2.002847 0.061311 32.667 2.638e-12 f1C:f2b 7.702999 0.116330 66.217 1.156e-15 f1D:f2b 8.964251 0.116330 77.059 < 2.2e-16 f1B:f2c 5.017610 0.061311 81.838 < 2.2e-16 f1C:f2c 10.961326 0.116330 94.226 < 2.2e-16 f1D:f2c 12.041081 0.116330 103.508 < 2.2e-16 Residual standard error: 0.2868 on 11 degrees of freedom Multiple R-squared: 0.9999, Adjusted R-squared: 0.9999 \end{CodeOutput} \begin{CodeInput} R> all.equal(coef(fm1), coef(fmSVD)) \end{CodeInput} \begin{CodeOutput} [1] "'is.NA' value mismatch: 0 in current 1 in target" \end{CodeOutput} \begin{CodeInput} R> all.equal(unname(fitted(fm1)), fitted(fmSVD)) \end{CodeInput} \begin{CodeOutput} [1] TRUE \end{CodeOutput} \begin{CodeInput} R> all.equal(unname(residuals(fm1)), residuals(fmSVD)) \end{CodeInput} \begin{CodeOutput} [1] TRUE \end{CodeOutput} The coefficients from the symmetric eigendecomposition method are the same as those from the SVD, hence the fitted values and residuals must be the same for these two methods. \begin{CodeInput} R> summary(fmVLV <- fastLm(y ~ f1 * f2, dd, method = 5)) R> all.equal(coef(fmSVD), coef(fmVLV)) \end{CodeInput} \begin{CodeOutput} [1] TRUE \end{CodeOutput} \subsection{Comparative speed} In the \pkg{RcppEigen} package the \proglang{R} function to fit linear models using the methods described above is called \code{fastLm}. It follows an earlier example in the \pkg{Rcpp} package which was carried over to both \pkg{RcppArmadillo} and \pkg{RcppGSL}. The natural question to ask is, ``Is it indeed fast to use these methods based on \pkg{Eigen}?''. To this end, the package provides benchmarking code for these methods, \proglang{R}'s \code{lm.fit} function and the \code{fastLm} implementations in the \pkg{RcppArmadillo} \citep{CRAN:RcppArmadillo} and \pkg{RcppGSL} \citep{CRAN:RcppGSL} packages, if they are installed. The benchmark code, which uses the \pkg{rbenchmark} \citep{CRAN:rbenchmark} package, is in a file named \code{lmBenchmark.R} in the \code{examples} subdirectory of the installed \pkg{RcppEigen} package. It can be run as \begin{CodeInput} R> source(system.file("examples", "lmBenchmark.R", package = "RcppEigen")) \end{CodeInput} Results will vary according to the speed of the processor and the implementation of the BLAS (Basic Linear Algebra Subroutines) used. (\pkg{Eigen} methods do not use the BLAS but the other methods do.) The \pkg{Eigen}3 template library does not use multi-threaded code for these operations but does use the graphics pipeline instructions (SSE and SSE2, in this case) in some calculations. \begin{table}[t!] \centering \begin{tabular}{r r r r r} \hline Method & Relative& Elapsed & User & Sys \\ \hline LDLt & 1.00 & 1.18 & 1.17 & 0.00 \\ LLt & 1.01 & 1.19 & 1.17 & 0.00 \\ SymmEig & 2.76 & 3.25 & 2.70 & 0.52 \\ QR & 6.35 & 7.47 & 6.93 & 0.53 \\ arma & 6.60 & 7.76 & 25.69 & 4.47 \\ PivQR & 7.15 & 8.41 & 7.78 & 0.62 \\ lm.fit & 11.68 & 13.74 & 21.56 & 16.79 \\ GESDD & 12.58 & 14.79 & 44.01 & 10.96 \\ SVD & 44.48 & 52.30 & 51.38 & 0.80 \\ GSL & 150.46 & 176.95 & 210.52 & 149.86 \\ \hline \end{tabular} \caption{\code{lmBenchmark} results on a desktop computer for the default size, $100,000\times 40$, full-rank model matrix running 20 repetitions for each method. Times (Elapsed, User, and Sys) are in seconds. The BLAS in use is a locally-rebuilt version of the OpenBLAS library included with Ubuntu 11.10. \label{tab:lmRes}} \end{table} Results obtained on a desktop computer, circa 2010, are shown in Table~\ref{tab:lmRes}. The processor used for these timings is a 4-core processor but almost all the methods are single-threaded and not affected by the number of cores. Only the \code{arma}, \code{lm.fit}, \code{GESDD} and \code{GSL} methods benefit from the multi-threaded BLAS implementation provided by OpenBLAS, and the relative speed increase is modest for this problem size and number of cores (at 7.76 seconds relative to 10.29 seconds for \code{arma}, 13.74 seconds relative to 16.91 seconds for \code{lm.fit}, and 176.95 seconds relative to 193.29 seconds for the \code{GSL}). Parallel computing approaches will always have to trade-off increased communication and overhead costs against the potential gains from running multiple execution threads. % Nonetheless, with the ongoing %shift to multi-core architectures and an ever increasing number of cores in %modern processing units, it is conceivable that \pkg{Eigen} may also switch %to a multi-threaded approach to further improve its performance. These results indicate that methods based on forming and decomposing $\bm X^\top\bm X$ (LDLt, LLt and SymmEig) are considerably faster than the others. The SymmEig method, using a rank-revealing decomposition, would be preferred, although the LDLt method could probably be modified to be rank-revealing. However, the dimensions of the problem will influence the comparative results. Because there are 100,000 rows in $\bm X$, methods that decompose the whole $\bm X$ matrix (all the methods except those named above) will be at a disadvantage. The pivoted QR method is 1.6 times faster than \proglang{R}'s \code{lm.fit} on this test and provides nearly the same information as \code{lm.fit}. Methods based on the singular value decomposition (SVD and GSL) are much slower but, as mentioned above, this is caused in part by $\bm X$ having many more rows than columns. The GSL method from the GNU Scientific Library uses an older algorithm for the SVD and is clearly out of contention. The \code{GESDD} method provides an interesting hybrid: It uses the \pkg{Eigen} classes, but then deploys the LAPACK routine \code{dgesdd} for the actual SVD calculation. The resulting time is much faster than using the SVD implementation of \pkg{Eigen} which is not a particularly fast SVD method. \section{Delayed evaluation} \label{sec:delayed} A form of delayed evaluation is used in \pkg{Eigen}. That is, many operators and methods do not evaluate the result but instead return an ``expression object'' that is evaluated when needed. As an example, even though one writes the $\bm X^\top\bm X$ evaluation as \code{.rankUpdate(X.adjoint())} the \code{X.adjoint()} part is not evaluated immediately. The \code{rankUpdate} method detects that it has been passed a matrix that is to be used in its transposed form and evaluates the update by taking inner products of columns of $\bm X$ instead of rows of $\bm X^\top$. As \pkg{Eigen} has developed some of these unevaluated expressions have been modified. In \pkg{Eigen 3.1}, which is incorporated in version 0.3.1 of \pkg{RcppEigen}, the \code{.adjoint()} method applied to a real dense matrix copies the contents of the original matrix, flags it as row-major and interchanges the number of rows and columns. This is indeed the adjoint of the original matrix but, at one time, the \code{wrap} method for the \pkg{Eigen} dense matrix classes would fail on row-major matrices. \begin{figure}[t!] \hrule \smallskip \noindent \ttfamily \hlstd{}\hlkwb{const\ }\hlstd{MapMati}\hlstd{\ \ }\hlstd{}\hlkwd{A}\hlstd{}\hlopt{(}\hlstd{as}\hlopt{$<$}\hlstd{MapMati}\hlopt{$>$(}\hlstd{AA}\hlopt{));}\hspace*{\fill}\\ \hlstd{}\hlkwa{return\ }\hlstd{}\hlkwd{wrap}\hlstd{}\hlopt{(}\hlstd{A}\hlopt{.}\hlstd{}\hlkwd{transpose}\hlstd{}\hlopt{());}\hlstd{}\hspace*{\fill}\\ \mbox{} \normalfont \normalsize \hrule \caption{\code{badtransCpp}: Transpose producing a run-time error in early versions of \pkg{RcppEigen}. \label{badtrans}} \end{figure} In the code for the transpose of an integer matrix shown in Figure~\ref{trans} the transpose is assigned to a \code{MatrixXi} object before applying \code{wrap} to it. The assignment forces the evaluation as a column-major matrix. In early versions of the \pkg{RcppEigen} package if this step is skipped, as in Figure~\ref{badtrans}, the result would have been incorrect. \begin{CodeInput} R> Ai <- matrix(1:6, ncol = 2L) R> ftrans2 <- cxxfunction(signature(AA = "matrix"), badtransCpp, + "RcppEigen", incl) R> (At <- ftrans2(Ai)) \end{CodeInput} \begin{CodeOutput} [,1] [,2] [,3] [1,] 1 2 3 [2,] 4 5 6 \end{CodeOutput} Although the problem no longer persists for this particular example, the recommended practice is to first assign objects before wrapping them for return to \proglang{R}. \section{Sparse matrices} \label{sec:sparse} \begin{figure}[b!] \hrule \smallskip \noindent \ttfamily \hlstd{}\hlkwa{using\ }\hlstd{Eigen}\hlopt{::}\hlstd{MappedSparseMatrix}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwa{using\ }\hlstd{Eigen}\hlopt{::}\hlstd{SparseMatrix}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hspace*{\fill}\\ \hlkwb{const\ }\hlstd{MappedSparseMatrix}\hlopt{$<$}\hlstd{}\hlkwb{double}\hlstd{}\hlopt{$>$\ }\hlstd{}\hlkwd{A}\hlstd{}\hlopt{(}\hlstd{as}\hlopt{$<$}\hlstd{MappedSparseMatrix}\hlopt{$<$}\hlstd{}\hlkwb{double}\hlstd{}\hlopt{$>$\ $>$(}\hlstd{AA}\hlopt{));}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{MapVecd}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{}\hlkwd{y}\hlstd{}\hlopt{(}\hlstd{as}\hlopt{$<$}\hlstd{MapVecd}\hlopt{$>$(}\hlstd{yy}\hlopt{));}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{SparseMatrix}\hlopt{$<$}\hlstd{}\hlkwb{double}\hlstd{}\hlopt{$>$}\hlstd{\ \ \ \ \ \ }\hlopt{}\hlstd{}\hlkwd{At}\hlstd{}\hlopt{(}\hlstd{A}\hlopt{.}\hlstd{}\hlkwd{adjoint}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlkwa{return\ }\hlstd{List}\hlopt{::}\hlstd{}\hlkwd{create}\hlstd{}\hlopt{(}\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"At"}\hlstd{}\hlopt{{)}}\hlstd{\ \ }\hlopt{=\ }\hlstd{At}\hlopt{,}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{Named}\hlopt{{(}}\hlstd{}\hlstr{"Aty"}\hlstd{}\hlopt{{)}\ =\ }\hlstd{At\ }\hlopt{{*}\ }\hlstd{y}\hlopt{);}\hlstd{}\hspace*{\fill}\\ \mbox{} \normalfont \normalsize \hrule \caption{\code{sparseProdCpp}: Transpose and product with sparse matrices. \label{sparseProd}} \end{figure} \pkg{Eigen} provides sparse matrix classes. An \proglang{R} object of class \code{dgCMatrix} (from the \pkg{Matrix} package by \citet{CRAN:Matrix}) can be mapped as in Figure~\ref{sparseProd}. \begin{CodeInput} R> sparse1 <- cxxfunction(signature(AA = "dgCMatrix", yy = "numeric"), + sparseProdCpp, "RcppEigen", incl) R> data("KNex", package = "Matrix") R> rr <- sparse1(KNex$mm, KNex$y) R> stopifnot(all.equal(rr$At, t(KNex$mm)), + all.equal(rr$Aty, as.vector(crossprod(KNex$mm, KNex$y)))) \end{CodeInput} % Sparse Cholesky decompositions are provided by the \code{SimplicialLLT} and \code{SimplicialLDLT} classes in the \pkg{RcppEigen} package for \proglang{R}. These are subclasses of the \code{SimplicialCholesky} templated. A sample usage is shown in Figure~\ref{fig:spLS}. % \begin{figure}[t!] \hrule \smallskip \noindent \ttfamily \hlstd{}\hlkwc{typedef\ }\hlstd{Eigen}\hlopt{::}\hlstd{MappedSparseMatrix}\hlopt{$<$}\hlstd{}\hlkwb{double}\hlstd{}\hlopt{$>$}\hlstd{\ \ }\hlopt{}\hlstd{MSpMat}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwc{typedef\ }\hlstd{Eigen}\hlopt{::}\hlstd{SparseMatrix}\hlopt{$<$}\hlstd{}\hlkwb{double}\hlstd{}\hlopt{$>$}\hlstd{\ \ \ \ \ \ \ \ \ }\hlopt{}\hlstd{SpMat}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwc{typedef\ }\hlstd{Eigen}\hlopt{::}\hlstd{SimplicialLDLT}\hlopt{$<$}\hlstd{SpMat}\hlopt{$>$}\hlstd{\ \ \ \ \ \ \ }\hlopt{}\hlstd{SpChol}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hspace*{\fill}\\ \hlkwb{const\ }\hlstd{SpMat}\hlstd{\ \ \ \ \ \ }\hlstd{}\hlkwd{At}\hlstd{}\hlopt{(}\hlstd{as}\hlopt{$<$}\hlstd{MSpMat}\hlopt{$>$(}\hlstd{AA}\hlopt{).}\hlstd{}\hlkwd{adjoint}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{VectorXd}\hlstd{\ \ }\hlstd{}\hlkwd{Aty}\hlstd{}\hlopt{(}\hlstd{At\ }\hlopt{{*}\ }\hlstd{as}\hlopt{$<$}\hlstd{MapVecd}\hlopt{$>$(}\hlstd{yy}\hlopt{));}\hspace*{\fill}\\ \hlstd{}\hlkwb{const\ }\hlstd{SpChol}\hlstd{\ \ \ \ \ }\hlstd{}\hlkwd{Ch}\hlstd{}\hlopt{(}\hlstd{At\ }\hlopt{{*}\ }\hlstd{At}\hlopt{.}\hlstd{}\hlkwd{adjoint}\hlstd{}\hlopt{());}\hspace*{\fill}\\ \hlstd{}\hlkwa{if\ }\hlstd{}\hlopt{(}\hlstd{Ch}\hlopt{.}\hlstd{}\hlkwd{info}\hlstd{}\hlopt{()\ !=\ }\hlstd{Eigen}\hlopt{::}\hlstd{Success}\hlopt{)\ }\hlstd{}\hlkwa{return\ }\hlstd{R\textunderscore NilValue}\hlopt{;}\hspace*{\fill}\\ \hlstd{}\hlkwa{return\ }\hlstd{List}\hlopt{::}\hlstd{}\hlkwd{create}\hlstd{}\hlopt{(}\hlstd{}\hlkwd{Named}\hlstd{}\hlopt{(}\hlstd{}\hlstr{"betahat"}\hlstd{}\hlopt{)}\hlstd{\ \ }\hlopt{=\ }\hlstd{Ch}\hlopt{.}\hlstd{}\hlkwd{solve}\hlstd{}\hlopt{(}\hlstd{Aty}\hlopt{),}\hspace*{\fill}\\ \hlstd{}\hlstd{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }\hlstd{}\hlkwd{Named}\hlstd{}\hlopt{(}\hlstd{}\hlstr{"perm"}\hlstd{}\hlopt{)}\hlstd{\ \ \ \ \ }\hlopt{=\ }\hlstd{Ch}\hlopt{.}\hlstd{}\hlkwd{permutationP}\hlstd{}\hlopt{().}\hlstd{}\hlkwd{indices}\hlstd{}\hlopt{());}\hlstd{}\hspace*{\fill}\\ \mbox{} \normalfont \normalsize \hrule \caption{\code{sparseLSCpp}: Solving a sparse least squares problem. \label{fig:spLS}} \end{figure} % \begin{CodeInput} R> sparse2 <- cxxfunction(signature(AA = "dgCMatrix", yy = "numeric"), + sparseLSCpp, "RcppEigen", incl) R> str(rr <- sparse2(KNex$mm, KNex$y)) \end{CodeInput} \begin{CodeOutput} List of 2 $ betahat: num [1:712] 823 340 473 349 188 ... $ perm : int [1:712] 572 410 414 580 420 425 417 426 431 445 ... \end{CodeOutput} \begin{CodeInput} R> res <- as.vector(solve(Ch <- Cholesky(crossprod(KNex$mm)), + crossprod(KNex$mm, KNex$y))) R> stopifnot(all.equal(rr$betahat, res)) R> all(rr$perm == Ch@perm) \end{CodeInput} \begin{CodeOutput} [1] FALSE \end{CodeOutput} The fill-reducing permutations are different. \section{Summary} This paper introduced the \pkg{RcppEigen} package which provides high-level linear algebra computations as an extension to the \proglang{R} system. \pkg{RcppEigen} is based on the modern \proglang{C++} library \pkg{Eigen} which combines extended functionality with excellent performance, and utilizes \pkg{Rcpp} to interface \proglang{R} with \proglang{C++}. Several illustrations covered common matrix operations and several approaches to solving a least squares problem---including an extended discussion of rank-revealing approaches. Sparse matrix computations were also discussed, and a short example provided an empirical demonstration of the excellent run-time performance of the \pkg{RcppEigen} package. \pagebreak \bibliography{\Sexpr{Rcpp:::bib()}} \end{document} %%% Local Variables: %%% mode: latex %%% TeX-master: t %%% End: RcppEigen/src/0000755000176200001440000000000014662150312012715 5ustar liggesusersRcppEigen/src/RcppExports.cpp0000644000176200001440000000441614456320632015724 0ustar liggesusers// Generated by using Rcpp::compileAttributes() -> do not edit by hand // Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393 #include "../inst/include/RcppEigen.h" #include using namespace Rcpp; #ifdef RCPP_USE_GLOBAL_ROSTREAM Rcpp::Rostream& Rcpp::Rcout = Rcpp::Rcpp_cout_get(); Rcpp::Rostream& Rcpp::Rcerr = Rcpp::Rcpp_cerr_get(); #endif // eigen_version Rcpp::IntegerVector eigen_version(bool single); RcppExport SEXP _RcppEigen_eigen_version(SEXP singleSEXP) { BEGIN_RCPP Rcpp::RObject rcpp_result_gen; Rcpp::RNGScope rcpp_rngScope_gen; Rcpp::traits::input_parameter< bool >::type single(singleSEXP); rcpp_result_gen = Rcpp::wrap(eigen_version(single)); return rcpp_result_gen; END_RCPP } // Eigen_SSE bool Eigen_SSE(); RcppExport SEXP _RcppEigen_Eigen_SSE() { BEGIN_RCPP Rcpp::RObject rcpp_result_gen; Rcpp::RNGScope rcpp_rngScope_gen; rcpp_result_gen = Rcpp::wrap(Eigen_SSE()); return rcpp_result_gen; END_RCPP } // EigenNbThreads int EigenNbThreads(); RcppExport SEXP _RcppEigen_EigenNbThreads() { BEGIN_RCPP Rcpp::RObject rcpp_result_gen; Rcpp::RNGScope rcpp_rngScope_gen; rcpp_result_gen = Rcpp::wrap(EigenNbThreads()); return rcpp_result_gen; END_RCPP } // fastLm_Impl Rcpp::List fastLm_Impl(Rcpp::NumericMatrix X, Rcpp::NumericVector y, int type); RcppExport SEXP _RcppEigen_fastLm_Impl(SEXP XSEXP, SEXP ySEXP, SEXP typeSEXP) { BEGIN_RCPP Rcpp::RObject rcpp_result_gen; Rcpp::RNGScope rcpp_rngScope_gen; Rcpp::traits::input_parameter< Rcpp::NumericMatrix >::type X(XSEXP); Rcpp::traits::input_parameter< Rcpp::NumericVector >::type y(ySEXP); Rcpp::traits::input_parameter< int >::type type(typeSEXP); rcpp_result_gen = Rcpp::wrap(fastLm_Impl(X, y, type)); return rcpp_result_gen; END_RCPP } static const R_CallMethodDef CallEntries[] = { {"_RcppEigen_eigen_version", (DL_FUNC) &_RcppEigen_eigen_version, 1}, {"_RcppEigen_Eigen_SSE", (DL_FUNC) &_RcppEigen_Eigen_SSE, 0}, {"_RcppEigen_EigenNbThreads", (DL_FUNC) &_RcppEigen_EigenNbThreads, 0}, {"_RcppEigen_fastLm_Impl", (DL_FUNC) &_RcppEigen_fastLm_Impl, 3}, {NULL, NULL, 0} }; RcppExport void R_init_RcppEigen(DllInfo *dll) { R_registerRoutines(dll, NULL, CallEntries, NULL, NULL); R_useDynamicSymbols(dll, FALSE); } RcppEigen/src/Makevars.win0000644000176200001440000000077414456323674015233 0ustar liggesusers## -*- mode: makefile; -*- ## One could add '-fopenmp' here (and below) for multithreaded operations PKG_CXXFLAGS = -I../inst/include ## One could add '-fopenmp' here (and above) for multithreaded operations PKG_LIBS = $(LAPACK_LIBS) $(BLAS_LIBS) $(FLIBS) ## We are not enabling multithreaded operations by default because this would be ## a change in behaviour that would like create trouble for packages using RcppEigen ## as CRAN tests of those packages may seen use of more than two cores and protest RcppEigen/src/RcppEigen.cpp0000644000176200001440000000306414456320130015276 0ustar liggesusers// -*- mode: C++; c-indent-level: 4; c-basic-offset: 4; tab-width: 8 -*- // // RcppEigen.cpp: Rcpp/Eigen glue // // Copyright (C) 2011 - 2023 Douglas Bates, Dirk Eddelbuettel and Romain Francois // // This file is part of RcppEigen. // // RcppEigen is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 2 of the License, or // (at your option) any later version. // // RcppEigen is distributed in the hope that it will be useful, but // WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with RcppEigen. If not, see . #include // [[Rcpp::export]] Rcpp::IntegerVector eigen_version(bool single) { using Rcpp::_; using Rcpp::IntegerVector; if (single) { return Rcpp::wrap( 10000 * EIGEN_WORLD_VERSION + 100 * EIGEN_MAJOR_VERSION + EIGEN_MINOR_VERSION ) ; } return IntegerVector::create(_["major"] = EIGEN_WORLD_VERSION, _["minor"] = EIGEN_MAJOR_VERSION, _["patch"] = EIGEN_MINOR_VERSION); } // [[Rcpp::export]] bool Eigen_SSE() { return Rcpp::wrap(Eigen::SimdInstructionSetsInUse()); } // [[Rcpp::export]] int EigenNbThreads() { return Eigen::nbThreads(); } RcppEigen/src/Makevars0000644000176200001440000000077414456323545014434 0ustar liggesusers## -*- mode: makefile; -*- ## One could add '-fopenmp' here (and below) for multithreaded operations PKG_CXXFLAGS = -I../inst/include ## One could add '-fopenmp' here (and above) for multithreaded operations PKG_LIBS = $(LAPACK_LIBS) $(BLAS_LIBS) $(FLIBS) ## We are not enabling multithreaded operations by default because this would be ## a change in behaviour that would like create trouble for packages using RcppEigen ## as CRAN tests of those packages may seen use of more than two cores and protest RcppEigen/src/fastLm.cpp0000644000176200001440000002171214567757725014703 0ustar liggesusers// -*- mode: C++; c-indent-level: 4; c-basic-offset: 4; tab-width: 8 -*- // // fastLm.cpp: Rcpp/Eigen example of a simple lm() alternative // // Copyright (C) 2011 - 2022 Douglas Bates, Dirk Eddelbuettel and Romain Francois // // This file is part of RcppEigen. // // RcppEigen is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 2 of the License, or // (at your option) any later version. // // RcppEigen is distributed in the hope that it will be useful, but // WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // in file.path(R.home("share"), "licenses"). If not, see // . #include "fastLm.h" #if !defined(EIGEN_USE_MKL) // don't use R Lapack.h if MKL is enabled #include #endif #ifndef FCONE # define FCONE #endif namespace lmsol { using Rcpp::_; using Rcpp::as; using Rcpp::CharacterVector; using Rcpp::clone; using Rcpp::List; using Rcpp::NumericMatrix; using Rcpp::NumericVector; using Rcpp::RObject; using Rcpp::wrap; using std::invalid_argument; using std::numeric_limits; lm::lm(const Map &X, const Map &y) : m_X(X), m_y(y), m_n(X.rows()), m_p(X.cols()), m_coef(VectorXd::Constant(m_p, ::NA_REAL)), // #nocov m_r(::NA_INTEGER), m_fitted(m_n), m_se(VectorXd::Constant(m_p, ::NA_REAL)), // #nocov m_usePrescribedThreshold(false) { } lm& lm::setThreshold(const RealScalar& threshold) { // #nocov start m_usePrescribedThreshold = true; m_prescribedThreshold = threshold; return *this; // #nocov end } inline ArrayXd lm::Dplus(const ArrayXd& d) { ArrayXd di(d.size()); double comp(d.maxCoeff() * threshold()); for (int j = 0; j < d.size(); ++j) di[j] = (d[j] < comp) ? 0. : 1./d[j]; m_r = (di != 0.).count(); return di; } MatrixXd lm::XtX() const { return MatrixXd(m_p, m_p).setZero().selfadjointView(). rankUpdate(m_X.adjoint()); } /** Returns the threshold that will be used by certain methods such as rank(). * * The default value comes from experimenting (see "LU precision * tuning" thread on the Eigen list) and turns out to be * identical to Higham's formula used already in LDLt. * * @return The user-prescribed threshold or the default. */ RealScalar lm::threshold() const { return m_usePrescribedThreshold ? m_prescribedThreshold : numeric_limits::epsilon() * m_p; } ColPivQR::ColPivQR(const Map &X, const Map &y) : lm(X, y) { ColPivHouseholderQR PQR(X); // decompose the model matrix Permutation Pmat(PQR.colsPermutation()); m_r = PQR.rank(); if (m_r == m_p) { // full rank case m_coef = PQR.solve(y); m_fitted = X * m_coef; m_se = Pmat * PQR.matrixQR().topRows(m_p). triangularView().solve(I_p()).rowwise().norm(); return; } MatrixXd Rinv(PQR.matrixQR().topLeftCorner(m_r, m_r). // #nocov start triangularView(). solve(MatrixXd::Identity(m_r, m_r))); VectorXd effects(PQR.householderQ().adjoint() * y); m_coef.head(m_r) = Rinv * effects.head(m_r); m_coef = Pmat * m_coef; // create fitted values from effects // (can't use X*m_coef if X is rank-deficient) effects.tail(m_n - m_r).setZero(); m_fitted = PQR.householderQ() * effects; m_se.head(m_r) = Rinv.rowwise().norm(); m_se = Pmat * m_se; // #nocov end } QR::QR(const Map &X, const Map &y) : lm(X, y) { HouseholderQR QR(X); m_coef = QR.solve(y); m_fitted = X * m_coef; m_se = QR.matrixQR().topRows(m_p). triangularView().solve(I_p()).rowwise().norm(); } Llt::Llt(const Map &X, const Map &y) : lm(X, y) { LLT Ch(XtX().selfadjointView()); m_coef = Ch.solve(X.adjoint() * y); m_fitted = X * m_coef; m_se = Ch.matrixL().solve(I_p()).colwise().norm(); } Ldlt::Ldlt(const Map &X, const Map &y) : lm(X, y) { LDLT Ch(XtX().selfadjointView()); Dplus(Ch.vectorD()); // to set the rank //FIXME: Check on the permutation in the LDLT and incorporate it in //the coefficients and the standard error computation. // m_coef = Ch.matrixL().adjoint(). // solve(Dplus(D) * Ch.matrixL().solve(X.adjoint() * y)); m_coef = Ch.solve(X.adjoint() * y); m_fitted = X * m_coef; m_se = Ch.solve(I_p()).diagonal().array().sqrt(); } int gesdd(MatrixXd& A, ArrayXd& S, MatrixXd& Vt) { int info, mone = -1, m = A.rows(), n = A.cols(); std::vector iwork(8 * n); double wrk; if (m < n || S.size() != n || Vt.rows() != n || Vt.cols() != n) throw std::invalid_argument("dimension mismatch in gesvd"); // #nocov F77_CALL(dgesdd)("O", &m, &n, A.data(), &m, S.data(), A.data(), &m, Vt.data(), &n, &wrk, &mone, &iwork[0], &info FCONE); int lwork(wrk); std::vector work(lwork); F77_CALL(dgesdd)("O", &m, &n, A.data(), &m, S.data(), A.data(), &m, Vt.data(), &n, &work[0], &lwork, &iwork[0], &info FCONE); return info; } GESDD::GESDD(const Map& X, const Map &y) : lm(X, y) { MatrixXd U(X), Vt(m_p, m_p); ArrayXd S(m_p); if (gesdd(U, S, Vt)) throw std::runtime_error("error in gesdd"); MatrixXd VDi(Vt.adjoint() * Dplus(S).matrix().asDiagonal()); m_coef = VDi * U.adjoint() * y; m_fitted = X * m_coef; m_se = VDi.rowwise().norm(); } SVD::SVD(const Map &X, const Map &y) : lm(X, y) { JacobiSVD UDV(X.jacobiSvd(ComputeThinU|ComputeThinV)); MatrixXd VDi(UDV.matrixV() * Dplus(UDV.singularValues().array()).matrix().asDiagonal()); m_coef = VDi * UDV.matrixU().adjoint() * y; m_fitted = X * m_coef; m_se = VDi.rowwise().norm(); } SymmEigen::SymmEigen(const Map &X, const Map &y) : lm(X, y) { SelfAdjointEigenSolver eig(XtX().selfadjointView()); MatrixXd VDi(eig.eigenvectors() * Dplus(eig.eigenvalues().array()).sqrt().matrix().asDiagonal()); m_coef = VDi * VDi.adjoint() * X.adjoint() * y; m_fitted = X * m_coef; m_se = VDi.rowwise().norm(); } enum {ColPivQR_t = 0, QR_t, LLT_t, LDLT_t, SVD_t, SymmEigen_t, GESDD_t}; static inline lm do_lm(const Map &X, const Map &y, int type) { switch(type) { case ColPivQR_t: return ColPivQR(X, y); case QR_t: return QR(X, y); case LLT_t: return Llt(X, y); case LDLT_t: return Ldlt(X, y); case SVD_t: return SVD(X, y); case SymmEigen_t: return SymmEigen(X, y); case GESDD_t: return GESDD(X, y); } throw invalid_argument("invalid type"); return ColPivQR(X, y); // -Wall } List fastLm(Rcpp::NumericMatrix Xs, Rcpp::NumericVector ys, int type) { const Map X(as >(Xs)); const Map y(as >(ys)); Index n = X.rows(); if ((Index)y.size() != n) throw invalid_argument("size mismatch"); // Select and apply the least squares method lm ans(do_lm(X, y, type)); // Copy coefficients and install names, if any NumericVector coef(wrap(ans.coef())); List dimnames(NumericMatrix(Xs).attr("dimnames")); if (dimnames.size() > 1) { RObject colnames = dimnames[1]; if (!(colnames).isNULL()) coef.attr("names") = clone(CharacterVector(colnames)); } VectorXd resid = y - ans.fitted(); int rank = ans.rank(); int df = (rank == ::NA_INTEGER) ? n - X.cols() : n - rank; double s = resid.norm() / std::sqrt(double(df)); // Create the standard errors VectorXd se = s * ans.se(); return List::create(_["coefficients"] = coef, _["se"] = se, _["rank"] = rank, _["df.residual"] = df, _["residuals"] = resid, _["s"] = s, _["fitted.values"] = ans.fitted()); } } // This defines the R-callable function 'fastLm' // [[Rcpp::export]] Rcpp::List fastLm_Impl(Rcpp::NumericMatrix X, Rcpp::NumericVector y, int type) { return lmsol::fastLm(X, y, type); } RcppEigen/src/fastLm.h0000644000176200001440000000727612565743640014344 0ustar liggesusers// -*- mode: C++; c-indent-level: 4; c-basic-offset: 4; tab-width: 8 -*- // // fastLm.h: Rcpp/Eigen example of a simple lm() alternative // // Copyright (C) 2011 - 2015 Douglas Bates, Dirk Eddelbuettel and Romain Francois // // This file is part of RcppEigen. // // RcppEigen is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 2 of the License, or // (at your option) any later version. // // RcppEigen is distributed in the hope that it will be useful, but // WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // in file.path(R.home("share"), "licenses"). If not, see // . #ifndef RCPPEIGEN_FASTLM_H #define RCPPEIGEN_FASTLM_H #include namespace lmsol { using Eigen::ArrayXd; using Eigen::ColPivHouseholderQR; using Eigen::ComputeThinU; using Eigen::ComputeThinV; using Eigen::HouseholderQR; using Eigen::JacobiSVD; using Eigen::LDLT; using Eigen::LLT; using Eigen::Lower; using Eigen::Map; using Eigen::MatrixXd; using Eigen::SelfAdjointEigenSolver; using Eigen::SelfAdjointView; using Eigen::TriangularView; using Eigen::VectorXd; using Eigen::Upper; typedef MatrixXd::Index Index; typedef MatrixXd::Scalar Scalar; typedef MatrixXd::RealScalar RealScalar; typedef ColPivHouseholderQR::PermutationType Permutation; typedef Permutation::IndicesType Indices; class lm { protected: Map m_X; /**< model matrix */ Map m_y; /**< response vector */ Index m_n; /**< number of rows of X */ Index m_p; /**< number of columns of X */ VectorXd m_coef; /**< coefficient vector */ int m_r; /**< computed rank or NA_INTEGER */ VectorXd m_fitted; /**< vector of fitted values */ VectorXd m_se; /**< standard errors */ RealScalar m_prescribedThreshold; /**< user specified tolerance */ bool m_usePrescribedThreshold; public: lm(const Map&, const Map&); ArrayXd Dplus(const ArrayXd& D); MatrixXd I_p() const {return MatrixXd::Identity(m_p, m_p);} MatrixXd XtX() const; // setThreshold and threshold are based on ColPivHouseholderQR methods RealScalar threshold() const; const VectorXd& se() const {return m_se;} const VectorXd& coef() const {return m_coef;} const VectorXd& fitted() const {return m_fitted;} int rank() const {return m_r;} lm& setThreshold(const RealScalar&); }; class ColPivQR : public lm { public: ColPivQR(const Map&, const Map&); }; class Llt : public lm { public: Llt(const Map&, const Map&); }; class Ldlt : public lm { public: Ldlt(const Map&, const Map&); }; class QR : public lm { public: QR(const Map&, const Map&); }; class GESDD : public lm { public: GESDD(const Map&, const Map&); }; class SVD : public lm { public: SVD(const Map&, const Map&); }; class SymmEigen : public lm { public: SymmEigen(const Map&, const Map&); }; } // extern "C" SEXP fastLm(SEXP Xs, SEXP ys, SEXP types); #endif RcppEigen/ChangeLog0000644000176200001440000011763014662147775013733 0ustar liggesusers2024-08-23 Dirk Eddelbuettel * DESCRIPTION (Version, Date): CRAN Release 0.3.4.0.2 * inst/NEWS.Rd: Release 0.3.4.0.2 * DESCRIPTION (Authors@R): Correct two ORCID tag typos 2024-08-14 Dirk Eddelbuettel * DESCRIPTION (Version, Date): CRAN Release 0.3.4.0.1 * inst/NEWS.Rd: Release 0.3.4.0.1 2024-08-13 Dirk Eddelbuettel * inst/include/Eigen/src/misc/blas.h: Conditionally comment out xerbla which is now also provided by r-devel (CRAN request) 2024-08-12 Dirk Eddelbuettel * DESCRIPTION (Authors@R): Added, with ORCID IDs 2024-05-16 Dirk Eddelbuettel * README.md: Use tinyverse.netlify.app for dependency badge 2024-03-01 Dirk Eddelbuettel * .github/workflows/ci.yaml (jobs): Update to actions/checkout@v4, add r-ci-setup actions 2024-02-28 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.3.4.0.0 2024-02-12 Tomas Kalibera * inst/include/unsupported/Eigen/CXX11/src/ThreadPool/ThreadYield.h: Support clang on Windows by including 'sched.h' header 2024-01-17 Yixuan Qiu * DESCRIPTION (Version, Date): Release candidate 0.3.3.99.0 * inst/include/Eigen: Upgraded to Eigen 3.4.0 * inst/include/unsupported/Eigen: Idem * patches/eigen-3.4.0.diff: Carried local CRAN patches forward * patches/howToDiff.md: Idem 2023-11-01 Dirk Eddelbuettel * DESCRIPTION (Version, Date): CRAN Release 0.3.3.9.4 * inst/NEWS.Rd: Release 0.3.3.9.4 2023-10-05 Mikael Jagan * DESCRIPTION: Package 'Matrix' is now only a Suggests: * NAMESPACE: Remove unconditional imports from package Matrix * inst/include/Eigen/CholmodSupport: No longer need to include RcppEigenCholmod.h * inst/include/Eigen/src/CholmodSupport/CholmodSupport.h: Small wrapper adjustments conditional on Matrix use * inst/include/RcppEigenCholmod.h: Updated * inst/include/RcppEigenForward.h: Simplified * inst/include/RcppEigenWrap.h: Ditto * inst/include/RcppEigenStubs.cpp: New shorter helper * inst/include/RcppEigenStubs.h: Removed 2023-07-21 Dirk Eddelbuettel * DESCRIPTION (Date, Version): Roll micro version and date * README.md: Add r-universe badge 2023-07-20 Dirk Eddelbuettel * src/RcppEigen.cpp (EigenNbThreads): Add simple threads reporter * R/fastLm.R (fastLmPure): Simpler call to `fastLm_Impl()` * src/init.c: Replaced by auto-generated section in RcppExports.cpp * src/RcppExports.cpp: Regenerated * R/RcppExports.R: Idem * src/Makevars: Document possible use of '-fopenmp' * src/Makevars.win: Idem 2023-04-18 Dirk Eddelbuettel * README.md: Use app.codecov.io as base for codecov link 2023-03-10 Dirk Eddelbuettel * DESCRIPTION (Date, Version): Roll minor version * R/RcppEigen.package.skeleton.R: No longer set Imports: RcppEigen in DESCRIPTION and NAMESPACE 2023-02-12 Dirk Eddelbuettel * inst/CITATION: Convert to bibentry() style with person() 2022-11-04 Dirk Eddelbuettel * DESCRIPTION (Version, Date): CRAN Release 0.3.3.9.3 * inst/NEWS.Rd: Release 0.3.3.9.3 * src/init.c: Add 'void' for proper prototype pleasing clang-15 * R/fastLm.R (summary.fastLm,print.fastLm): Refer to correct and full variable name df.residual in the returned object * .github/workflows/ci.yaml (jobs): Update to actions/checkout@v3 2022-09-15 Jonah Gabry * inst/skeleton/rcppeigen_hello_world.cpp: Correct typo 2022-04-08 Dirk Eddelbuettel * DESCRIPTION (Depends): Add a versioned dependency on R 3.6.0 or later because of our use of FCONE to support USE_FC_LEN_T 2022-04-05 Dirk Eddelbuettel * DESCRIPTION (Version, Date): CRAN Release 0.3.3.9.2 * inst/NEWS.Rd: Release 0.3.3.9.2 2022-04-04 Dirk Eddelbuettel * src/fastLm.cpp: Add FCONE in two calls for improved Fortran and C character interface per Writing R Extensions, Section 6.6.1 2022-01-16 Dirk Eddelbuettel * inst/tinytest/test_wrap.R: Added (optional) large memory wrap tests * inst/tinytest/cpp/wrap.cpp: Added C++ part of test * .codecov.yml: Added to not trigger PR fail for small additions 2022-01-16 Mikael Jagan * inst/include/RcppEigenWrap.h: Refine use plain dense wrap() change 2022-01-15 Mikael Jagan * inst/include/RcppEigenWrap.h: Use R_xlen_t for vectors rows + cols 2021-12-29 Dirk Eddelbuettel * README.md: Add total downloads badge 2021-12-08 Dirk Eddelbuettel * README.md: Remove unused continuous integration artifact and badge 2021-10-13 Dirk Eddelbuettel * inst/CITATION: Refinment of doi use 2021-10-10 Dirk Eddelbuettel * inst/CITATION: Switch JSS url to doi form per JSS request * README.md: Idem 2021-07-19 Dirk Eddelbuettel * inst/include/RcppEigenWrap.h: Two more #nocov tags * src/fastLm.cpp: One more #nocov tag 2021-07-18 Dirk Eddelbuettel * inst/tinytest/test_fastLm.R: Add tests for summary * src/fastLm.cpp: Add a few #nocov tags 2021-07-17 Dirk Eddelbuettel * DESCRIPTION (Date, Version): Roll minor version * R/RcppEigen.package.skeleton.R (RcppEigen.package.skeleton): Also import RcppEigen in DESCRIPTION 2021-06-07 Dirk Eddelbuettel * DESCRIPTION (Date, Version): Roll minor version * inst/tinytest/test_misc.R: New test file * inst/tinytest/test_fastLm.R: Added tests * R/fastLm.R: Add single one-line nocov tag * .Rbuildignore: Add .covignore and .codecov.yml 2021-06-06 Dirk Eddelbuettel * README.md: Added coverage badge * .github/workflows/ci.yaml (jobs): Turn on coverage * .covrignore: Added * .codecov.yml (ignore): Idem * R/RcppEigen.package.skeleton.R: Set nocov 2021-05-09 Dirk Eddelbuettel * DESCRIPTION (URL): Add GitHub repo to URL field 2021-01-02 Dirk Eddelbuettel * R/RcppEigen.package.skeleton.R: Wrap any() around grepl() 2020-12-25 Dirk Eddelbuettel * .github/workflows/ci.yaml: Small tweaks to CI YAML file 2020-12-17 Dirk Eddelbuettel * DESCRIPTION (Version, Date): CRAN Release 0.3.3.9.1 (following coordinated update of reverse dependency StanHeaders) * inst/NEWS.Rd: Release 0.3.3.9.1 2020-12-14 Dirk Eddelbuettel * .github/workflows/ci.yaml: Add CI runner using r-ci * README.md: Add new badge 2020-12-05 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.3.3.9.0 * inst/include/Eigen: Upgraded to Eigen 3.3.9 * inst/include/unsupported/Eigen: Idem * patches/eigen-3.3.9.diff: Carried local CRAN patches forward * .travis.yml: Switch to r-ci using focal and bspm * README.md: Updated URLs to https and/or redirect location * inst/CITATION: Idem * man/RcppEigen-package.Rd: Idem * man/fastLm.Rd: Idem 2020-08-16 Dirk Eddelbuettel * README.md: Add JSS badge 2020-03-29 Dirk Eddelbuettel * README.md: Added commit badge, edited 2020-01-22 Dirk Eddelbuettel * README.md: README.md: Add a Debian badge 2019-11-16 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.3.3.7.0 * inst/include/Eigen: Upgraded to Eigen 3.3.7 * inst/include/unsupported/Eigen: Idem * patches/eigen-3.3.7.diff: Carried local CRAN patches forward 2019-11-01 Dirk Eddelbuettel * R/unit.test.R (compile_unit_tests): Removed as no longer needed 2019-10-31 Dirk Eddelbuettel * inst/tinytest/test_transform.R: Switch from RUnit to tinytest * inst/tinytest/test_wrap.R: Idem * inst/tinytest/cpp/transform.cpp: Added using Rcpp Attributes * inst/tinytest/cpp/wrap.cpp: Idem * inst/tinytest/cpp/sparse.cpp: More idiomatic Eigen code * inst/tinytest/test_sparse.R: Idem * .editorconfig: Added 2019-10-30 Dirk Eddelbuettel * DESCRIPTION (Suggests): Switch from RUnit to tinytest * .travis.yml (install): Ditto * tests/tinytest.R: Converted to tinytest * test_sparse.R: Converted to tinytest * cpp/sparse.cpp: Added using Rcpp Attributes 2019-10-29 Dirk Eddelbuettel * test_fastLm.R: Converted to tinytest * test_RcppEigen.R: Idem * test_solution.R: Idem * cpp/rcppeigen.cpp: Added using Rcpp Attributes * cpp/solution.cpp: Idem 2019-10-28 Dirk Eddelbuettel * tests/tinytest.R: Renamed from tests/doRUnit.R * inst/tinytest/test_fastLm.R: Renamed from inst/unitTests/runit* * inst/tinytest/test_RcppEigen.R: Idem * inst/tinytest/test_solution.R: Idem * inst/tinytest/test_sparse.R: Idem * inst/tinytest/test_transform.R: Idem * inst/tinytest/test_wrap.R: Idem 2019-10-13 Dirk Eddelbuettel * README.md: Added CRAN + BioConductor badges for reverse depends, add StackOverflow badge to recommend help searches under 'Rcpp' * DESCRIPTION (Version, Date): Roll minor version * R/RcppEigen.package.skeleton.R (RcppEigen.package.skeleton): Test for example_code outside of haveKitten test; remove force argument if unused 2019-05-24 Dirk Eddelbuettel * vignettes/RcppEigen-Introduction.Rnw: Update vignette to use RcppEigen:::eigen_version() instead of .Call() 2019-03-29 Dirk Eddelbuettel * inst/skeleton/rcppeigen_hello_world.cpp: Rework first example to not rely on Eigen RNG which R CMD check would complain about. 2019-02-16 James Joseph Balamuta * R/runit.RcppEigen.R: Removed listing RcppEigen in Imports during skeleton package creation. 2018-11-24 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.3.3.5.0 2018-11-23 Dirk Eddelbuettel * inst/include/Eigen: Updated to release 3.3.5 * inst/include/unsupported: Idem * patches/eigen-3.3.5.diff: Diff of local patches 2018-09-02 Dirk Eddelbuettel * .travis.yml: Switch Travis CI to R 3.5 repo 2018-05-30 Michael Weylandt * inst/examples/lmBenchmark.R: Update benchmark script to use microbenchmark and to use exposed fastLm functions from Rcpp packages rather than invoking .Call directly 2018-05-25 Ralf Stubner * inst/include/RcppEigenWrap.h: Use Rf_xlength and R_xlen_t to support long vectors 2018-02-07 Dirk Eddelbuettel * patches/eigen-3.3.4.diff: Diff to upstream Eigen 3.3.4, cherry-picked in the 3.3.4 branch of Yixuan 2018-02-05 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.3.3.4.0 * inst/include/Eigen/src/Core/arch/CUDA/Half.h (Eigen): Condition use of long long and unsigned long on C++11 * inst/include/Eigen/src/Core/arch/SSE/Complex.h (Eigen): Comment-out use of diagnostic-suppressing pragma for gcc + clang to satisfy CRAN * inst/include/Eigen/src/Core/util/DisableStupidWarnings.h: Idem 2018-02-04 Yixuan Qiu [ In RcppEigen 0.3.3.* branch ] * inst/include/Eigen: Updated to the upstream 3.3 branch of Eigen based on version 3.3.4 * inst/include/unsupported: Idem * DESCRIPTION: Idem * README.md: Idem * inst/include/Eigen/src/Core/util/DisableStupidWarnings.h: Patch from upstream Eigen that has not been ported to the 3.3 branch (cf GitHub issue #48) 2017-11-19 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.3.3.3.1 * R/inline.R: Use '::' not ':::' for Rcpp.plugin.maker 2017-08-26 Dirk Eddelbuettel * .travis.yml (before_install): Use https for curl fetch 2017-06-06 Yu Gong * inst/include/unsupported/Eigen/src/SparseExtra/MatrixMarketIterator.h: Also permit compilation under Haiku-OS 2017-05-28 Dirk Eddelbuettel * inst/examples/lmBenchmark.R (do_bench): Remove spurious argument in call to RcppEigen:::Eigen_SSE() (cf github issue #44) 2017-04-29 Dirk Eddelbuettel * DESCRIPTION: Release 0.3.3.3.0 [ which again owes a very big thank you to Yixuan for doing the work! ] 2017-04-27 Dirk Eddelbuettel * .travis.yml: Switch to using run.sh for Travis CI 2017-04-27 Yixuan Qiu [ In RcppEigen 0.3.3.* branch ] * inst/include/Eigen/src/Core/arch/CUDA/Half.h: Fixed compiler warning on 'long long' type in C++ 98 mode 2017-03-14 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.3.2.9.1 * src/init.c (R_init_RcppEigen): Call R_registerRoutines() and R_useDynamicSymbols() * NAMESPACE: Use .registration=TRUE on useDynLib * R/fastLm.R (fastLmPure): Remove erroneous fourth argument from .Call 2017-03-13 Martin Maechler * inst/include/RcppEigenCholmod.h: Synchronize with Matrix package 2017-02-21 Yixuan Qiu [ In RcppEigen 0.3.3.* branch ] * inst/include/Eigen: Updated to release 3.3.3 of Eigen * inst/include/unsupported: Idem * DESCRIPTION: Idem * README.md: Idem 2017-01-20 Yixuan Qiu [ In RcppEigen 0.3.3.* branch ] * inst/include/Eigen: Updated to release 3.3.2 of Eigen * inst/include/unsupported: Idem * DESCRIPTION: Idem * README.md: Idem * inst/unitTests/runit.RcppEigen.R, inst/unitTests/runit.sparse.R: Explicitly convert matrix size to `double` type such that Rcpp can properly return the value to R, thanks to ChingChuan and Dirk 2017-01-20 ChingChuan Chen [ In RcppEigen 0.3.3.* branch ] * inst/include/Eigen: Updated to release 3.3.1 of Eigen * inst/include/unsupported: Idem * DESCRIPTION: Idem * inst/examples/lmBenchmark.R: Fixed function names 2016-12-22 Dirk Eddelbuettel * DESCRIPTION (URL, BugReports): Added / updated 2016-11-12 Yixuan Qiu [ In RcppEigen 0.3.3.* branch ] * inst/include/Eigen: Updated to release 3.3.0 of Eigen * inst/include/unsupported: Idem * DESCRIPTION: Idem * README.md: Idem * inst/include/RcppEigenForward.h, inst/include/RcppEigenWrap.h: Added exporters for the new Map > type * inst/unitTests/runit.sparse.R: Unit tests for the new type 2016-08-20 Dirk Eddelbuettel * DESCRIPTION: Release 0.3.2.9.0 with big thanks to Yixuan for doing the work! 2016-08-19 Yixuan Qiu * inst/include/Eigen: Updated to release 3.2.9 of Eigen * README.md: Updated version number and fixed the NOTE from CRAN URL check 2016-04-30 Dirk Eddelbuettel * README.md: Expanded 2016-04-28 James Joseph Balamuta * inst/include/RcppEigenWrap.h: Added an exporter class for Map::RowVectorX per http://stackoverflow.com/questions/36920689/ * inst/include/unitTests/runit.RcppEigen.R: Added row exporter unit test. 2016-02-29 Dirk Eddelbuettel * DESCRIPTION (Version): Release 0.3.2.8.1 * inst/NEWS.Rd: Release 0.3.2.8.1 * debian/*: Changes for Debian release of 0.3.2.8.1 2016-02-28 Yixuan Qiu * inst/include/Eigen/src/SparseCore/CompressedStorage.h, inst/include/Eigen/src/SparseCore/SparseBlock.h, inst/include/Eigen/src/SparseCore/SparseMatrix.h, inst/include/Eigen/src/SparseCore/SparseRedux.h, inst/include/Eigen/src/SparseCore/SparseVector.h: Another patch from upstream to fix UBSAN null pointer errors 2016-02-23 Dirk Eddelbuettel * DESCRIPTION (Version): Release 0.3.2.8.0 * DESCRIPTION (Author): Added Yixuan Qiu * inst/NEWS.Rd: Release 0.3.2.8.0 2016-02-22 Yixuan Qiu * inst/include/Eigen: Updated to release 3.2.8 of Eigen * inst/include/unsupported: Idem * DESCRIPTION: Idem * README.md: Idem * inst/include/Eigen/src/SparseCore/CompressedStorage.h, inst/include/Eigen/src/Core/util/Memory.h: Applied patch from upstream to fix UBSAN errors 2016-01-20 Dirk Eddelbuettel * DESCRIPTION: Release 0.3.2.7.0 * debian/*: Changes for Debian release of 0.3.2.7.0 2016-01-19 Dirk Eddelbuettel * inst/unitTests/runit.sparse.R (test.sparseCholesky.R): Accomodate stricter as.vector() requirements in R-devel 2016-01-18 Yixuan Qiu * inst/include/Eigen: Updated to release 3.2.7 of Eigen * inst/include/unsupported: Idem * DESCRIPTION: Idem * README.md: Idem 2015-11-01 Alexey Stukalov * src/fastLm.cpp: Do not include R's LAPACK header if MKL defined 2015-09-23 Dirk Eddelbuettel * DESCRIPTION: Release 0.3.2.5.1 * debian/*: Changes for Debian release of 0.3.2.5.1 2015-09-21 Dirk Eddelbuettel * inst/include/Eigen/src/SparseCore/CompressedStorage.h: Added somewhat adapted code from upstream PR #118 to address UBSAN 2015-08-21 Dirk Eddelbuettel * src/RcppEigen.cpp: Updated code to use Rcpp Attributes * src/fastLm.cpp: Ditto * src/fastLm.h: Ditto * R/fastLm.R: Ditto * inst/unitTests/runit.fastLm.R: Ditto 2015-08-04 Dirk Eddelbuettel * R/RcppEigen.package.skeleton.R: Correct kitten() use based on patch kindly supplied by Grant Brown (cf GitHub issue #21) 2015-07-13 Dirk Eddelbuettel * DESCRIPTION: Version 0.3.2.5.0 2015-07-13 Yixuan Qiu * inst/include/Eigen/src/Core/Assign.h: Applied patch from upstream Eigen that will appear in the next release 2015-07-12 Dirk Eddelbuettel * DESCRIPTION: Added more explicitly quoting per R CMD check * NAMESPACE: Explicitly importFrom() stats and utils packages 2015-07-11 Yixuan Qiu * inst/include/Eigen: Updated to release 3.2.5 of Eigen * inst/include/unsupported: Idem * DESCRIPTION: Idem * README.md: Idem * inst/include/RcppEigenWrap.h: Fixed three incorrect error messages 2015-06-06 Dirk Eddelbuettel * R/RcppEigen.package.skeleton.R: Correct use of requireNamespace() 2015-02-26 Dirk Eddelbuettel * DESCRIPTION: Version 0.3.2.4.0 * debian/*: Updated 2015-02-23 Dirk Eddelbuettel * inst/include/Eigen: Updated to release 3.2.4 of Eigen * DESCRIPTION: Updated Version: and Date: * README.md: Idem * .travis.yml: Switch to ppa:edd/misc for binaries * R/RcppEigen.package.skeleton.R: Use pkgKitten::kitten, if installed * inst/skeleton/rcppeigen_hello_world.Rd: Added new page * inst/skeleton/Makevars: Minor edits * inst/skeleton/Makevars.win: Idem * DESCRIPTION: Added Suggests: for pkgKitten * .travis.yml: Also install pkgKitten 2014-12-22 Yixuan Qiu * inst/include/Eigen: Updated to release 3.2.3 of Eigen * DESCRIPTION: Idem * README.md: Idem 2014-11-23 Yixuan Qiu * inst/unitTests/runit.wrap.R: Added a number of unit tests * inst/unitTests/runit.transform.R: Idem * inst/unitTests/runit.sparse.R: Idem 2014-10-20 Dirk Eddelbuettel * inst/examples/lmBenchmark.R: Updated RcppArmadillo .Call in example 2014-08-21 Dirk Eddelbuettel * DESCRIPTION: New version 0.3.2.2.0 which was prepared expertly by Yixuan Qiu in a nice and clean pull request -- a big thank you for that! * DESCRIPTION: Adjusting Date: as well 2014-08-19 Yixuan Qiu * inst/include/Eigen: Updated to release 3.2.2 of Eigen * inst/unitTests/runit.wrap.R: More unit tests to reflect changes in as() exporters 2014-05-29 Yixuan Qiu * inst/include/RcppEigenWrap.h: Added as() conversion from R vector to Eigen::Array * inst/include/RcppEigenWrap.h: Fixed incorrect as() exporters * inst/include/RcppEigenWrap.h: Check matrix type of "dgCMatrix" instead of "CsparseMatrix" * inst/include/RcppEigenWrap.h, inst/include/RcppEigenForward.h: Added as() conversion from "dgRMatrix" to Eigen::MappedSparseMatrix 2014-05-01 Dirk Eddelbuettel * inst/include/Eigen/src/Core/util/Memory.h: Applied patch by Gael Guennebaud created in response to Doug's bug report (#803) which in turn is based on the UBSAN report at CRAN by Brian Ripley 2014-03-06 Dirk Eddelbuettel * DESCRIPTION: New version 0.3.2.1.1 2014-03-05 Dirk Eddelbuettel * inst/include/unsupported/Eigen/src/SparseExtra/MatrixMarketIterator.h: Better, more general test for DIRENT feature thanks to Brian Ripley 2014-03-04 Dirk Eddelbuettel * inst/include/unsupported/Eigen/src/SparseExtra/MatrixMarketIterator.h: (re-)add another #ifndef excluding a line on Solaris 2014-03-03 Dirk Eddelbuettel * DESCRIPTION: New release version 0.3.2.1.0 2014-03-03 Douglas Bates * inst/include/Eigen: Update to release 3.2.1 of Eigen 2014-03-01 Dirk Eddelbuettel * DESCRIPTION: New minor version 3.2.0.3 * DESCRIPTION: Switched to 'Imports: Rcpp (>= 0.11.0)' * src/Makevars: Updated for Rcpp 0.11.0 * src/Makevars.win: Ditto * R/RcppEigen.package.skeleton.R: Updated for Rcpp 0.11.0, also updated for current R versions by removing deprecated 'namespace' arg * man/RcppEigen.package.skeleton.Rd: Removed 'namespace' for current R * inst/skeleton/rcppeigen_hello_world.cpp: Rewritten / extended similar to the RcppArmadillo variant; now contains four functions exported by Rcpp Attributes * inst/skeleton/Makevars: Updated for Rcpp 0.11.0, also noted C++11 option via setting of 'USE_CXX1X=' with R 3.1.0 or later * inst/skeleton/Makevars.win: Ditto 2014-01-26 Dirk Eddelbuettel * DESCRIPTION: New minor version 3.2.0.2 2014-01-22 Dirk Eddelbuettel * NAMESPACE: Added importFrom(Rcpp, evalCpp) needed for next Rcpp 2014-01-12 Dirk Eddelbuettel * R/SHLIB.maker: Commented-out, file to be removed 2013-12-19 Dirk Eddelbuettel * DESCRIPTION: Updated Description to mention MPL-2 license for Eigen * inst/unitTests/runit.solutions.R: RUnit file converted from testthat * inst/unitTests/runit.transform.R: Idem * inst/unitTests/runit.wrap.R: Idem * DESCRIPTION: Remove Suggests: of testthat 2013-12-18 Dirk Eddelbuettel * DESCRIPTION: New minor version 3.2.0.1 * DESCRIPTION: New maintainer * DESCRIPTION: Added Copyright: reference to new COPYRIGHTS file * DESCRIPTION: Added reference to new LICENSE file * inst/COPYRIGHTS: Added, based on existing debian/copyright file * LICENSE: Added to state MPL-2 for Eigen, GPL (>=2) for RcppEigen * inst/include/Eigen/src/SparseLU/SparseLU.h: Applied Solaris patch * inst/include/unsupported/Eigen/src/SparseExtra/MatrixMarketIterator.h: Idem * inst/include/RcppEigenForward.h: Idem 2013-12-04 Dirk Eddelbuettel * debian/*: Added after emails with Doug * .Rbuildignore: Added debian/ 2013-11-13 Douglas Bates * DESCRIPTION: New version, release date and dependencies * inst/include/Eigen: Update to release 3.2.0 of Eigen 2013-02-03 Dirk Eddelbuettel * DESCRIPTION: New minor version 0.3.1.2.1 for upload to CRAN and JSS * NEWS.org: Updated 2013-01-26 Dirk Eddelbuettel * vignettes/jss835/: Regroup all related files in this directory * .Rbuildignore: Ignore vignettes/jss835 2013-01-14 Dirk Eddelbuettel * inst/CITATION: Added as provided by JSS editor * man/fastLm.Rd: Added reference to JSS paper * man/RcppEigen-package.Rd: Idem 2012-11-29 Douglas Bates * DESCRIPTION: New version * inst/include/Eigen, inst/include/unsupported: Update to version 3.1.2 of Eigen. 2012-11-29 Romain Francois * include/RcppEigenWrap.h: compatibility issue with Rcpp 0.10.1 2012-08-14 Dirk Eddelbuettel * R/flags.R: Add two (unexported) functions CxxFlags() and RcppEigenCxxFlags() for use in Makefiles etc 2012-07-30 Douglas Bates * inst/include/RcppEigenWrap.h, inst/unitTests/runit.RcppEigen.R: Another suggestion from Gael Guennebaud to allow row vectors to be wrapped. 2012-07-28 Douglas Bates * inst/include/RcppEigenWrap.h, inst/unitTests/runit.RcppEigen.R: More fixes to RcppEigenWrap.h and adjustment of tests accordingly. The changes allow RowMajor matrices to be wrapped (thanks to Gael Guennebaud) but cannot handle RowVector types. There will need to be more template metaprogramming done to redirect the case of RowVector, which cannot be changed to a ColMajor form. * src/Makevars: Because of changes in R, -DNDEBUG is automatic. One must override it with -UNDEBUG in the local ~/.R/Makevars to activate the debugging code. * inst/doc/code.R: New version of wrap code provides correct answer to the badtrans example * DESCRIPTION: Bump the date. 2012-07-27 Douglas Bates * inst/include/Eigen/*: Changed to released version of Eigen-3.1.0 using the MPL2 license. 2012-07-19 Dirk Eddelbuettel * R/fastLm.R: correct residual standard error display * R/fastLm.R: improved test for intercept once more with a tip of the hat to Doug 2012-07-17 Dirk Eddelbuettel * R/fastLm.R: Corrections for R^2 in no-intercept case 2012-06-26 Douglas Bates * DESCRIPTION, R/unit.test.R, inst/include/Eigen/*: Massive changes related to upgrade to Eigen-3.1.0 2012-03-13 Douglas Bates * inst/include/RcppEigenWrap.h: Change the wrap methods to avoid creating Rcpp::Dimension objects (which are implicitly created by the Rcpp::Matrix constructor). * inst/tests/test-solutions.R: Clean up. I defined the results then reevaluated them. 2012-03-08 Douglas Bates * inst/include/RcppEigenForward.h: Remove an include of RcppEigenConfig.h which is no longer needed. * ChangeLog, NEWS, NEWS.org: Update Changelog and NEWS which is generated from an org-mode file NEWS.org * inst/include/RcppEigenConfig.h, inst/include/unsupported/Eigen/MoreVectorization, inst/include/unsupported/Eigen/src/MoreVectorization/CMakeLists.txt: Delete the unsupported/Eigen/src/MoreVectorization module which we don't need * inst/include/unsupported/Eigen/src/MoreVectorization/MathFunctions.h: Delete the unsupported/Eigen/src/MoreVectorization module which we don't need * inst/tests/test-solutions.R, inst/tests/test-transform.R, inst/tests/test-wrap.R: Add testthat test files, temporarily disabled for R CMD check because of an unknown conflict * DESCRIPTION, R/unit.test.R, inst/include/Eigen/Cholesky, ..., src/Makevars: Massive changes in upgrade to Eigen-3.1.0-alpha2 2012-03-03 Douglas Bates * inst/include/RcppEigenWrap.h: Correct a typo. 2012-02-28 Douglas Bates * inst/include/RcppEigenForward.h, inst/include/RcppEigenWrap.h: Add as templates for ArrayXd and ArrayXXd 2012-02-03 Douglas Bates * inst/include/RcppEigenWrap.h, inst/unitTests/runit.sparse.R: Allow import of compressed row sparse matrices. Add test of same. 2012-01-18 Douglas Bates * inst/include/RcppEigenWrap.h: Allow for wrapping sparse row-major matrices. 2011-12-31 Douglas Bates * inst/include/RcppEigenCholmod.h: Minor typo in a comment 2011-12-23 Dirk Eddelbuettel * inst/unitTests/runTests.R: unit tests output 'fallback' directory changed to '..' and files are now in top-level of $pkg.Rcheck/ 2011-11-14 Douglas Bates * vignettes/RcppEigen-intro-nojss.Rnw: Many, many changes by Dirk and by Doug over the last two weeks to create a JSS paper and updated package. Too many changes to list. 2011-10-26 Douglas Bates * inst/doc/Rcpp.bib, inst/doc/RcppEigen-Intro.Rnw, inst/doc/RcppEigen-Intro.pdf: Add an introductory vignette. * src/fastLm.cpp, src/fastLm.h: Use a method for XtX but returning a MatrixXd, not a view. * inst/doc/RcppEigen-Intro.Rnw: Describe the SymmEig method correctly. * src/fastLm.cpp, src/fastLm.h: Use a macro for XtX temporarily. Clean up code. * inst/include/RcppEigenCholmod.h, inst/include/RcppEigenStubs.h: Update to Matrix_1.0.2 versions * inst/include/Eigen, inst/include/unsupported: update to eigen 3.0.3 * src/RcppEigen.cpp: Add externally callable functions for Eigen version and SSE instruction sets in use. * inst/examples/lmBenchmark.R: Suppress messages and provide additional information about Eigen version and SSE instructions in use. * R/fastLm.R: Allow optional arguments to be passed to printCoefmat; adapt to new returned structure. 2011-09-16 Douglas Bates * inst/include/unsupported/Eigen/src/SparseExtra/CholmodSupport.h: Avoid compiler warnings about comparing signed and unsigned values. 2011-09-13 Douglas Bates * src/Makevars: Force the -DNDEBUG flag to satisfy R CMD check in R-2.14.0 and higher 2011-09-12 Douglas Bates * inst/include/unsupported/Eigen/src/SparseExtra/CholmodSupport.h: Remove the solvetype specification for the solve method (but type is retained for solveInPlace). 2011-09-11 Douglas Bates * inst/include/RcppEigenForward.h: Remove forward declaration of non-existent templated function. 2011-09-02 Douglas Bates * DESCRIPTION, inst/include/Eigen/src/Core/util/Meta.h: New release. ifdef the use of "long long" using RCPP_HAS_LONG_LONG_TYPES (thanks, Dirk) 2011-08-31 Douglas Bates * inst/include/unsupported/Eigen/src/SparseExtra/CholmodSupport.h: Delete my addition of a CholmodAutoLLt mode, which is not needed (I misunderstood something previously). 2011-08-28 Douglas Bates * many files in inst/include/Eigen and inst/include/unsupported: Upgrade to Eigen release 3.0.2 2011-08-12 Douglas Bates * inst/include/unsupported/Eigen/src/SparseExtra/CholmodSupport.h: By-passing the const reference 2011-08-11 Douglas Bates * inst/include/unsupported/Eigen/src/SparseExtra/CholmodSupport.h: Add a solveInPlace method to try to avoid memory problems. 2011-08-10 Douglas Bates * inst/include/unsupported/Eigen/src/SparseExtra/CholmodSupport.h: Attempted fix of problem of freeing Map'ed memory in the Cholmod solve. 2011-07-31 Douglas Bates * inst/include/unsupported/Eigen/src/SparseExtra/CholmodSupport.h: Trying to fix the memory problems from the CholmodDecomposition solve methods - apparently unsuccessfully. 2011-07-29 Douglas Bates * inst/include/unsupported/Eigen/src/SparseExtra/CholmodSupport.h: Add LLtAuto method (which doesn't seem to work), more extractors and a factorize_p method for a const_CHM_SP argument. * inst/include/RcppEigenCholmod.h: Add a declaration of log determinant squared function. * inst/include/unsupported/Eigen/src/SparseExtra/CholmodSupport.h, inst/unitTests/runit.sparse.R: Allow rectangular matrix in CholmodDecomposition's factorize method. Test rectangular version of KNex example. 2011-07-28 Douglas Bates * inst/include/unsupported/Eigen/src/SparseExtra/CholmodSupport.h: Added a solveType member - there will be better ways of doing this * inst/include/RcppEigenForward.h: Include the stubs unconditionally * inst/include/RcppEigenStubs.cpp: * inst/include/RcppEigenStubs.h: Inlined the function definitions so it is a header file now * inst/include/unsupported/Eigen/src/SparseExtra/CholmodSupport.h: Extended CholmodDecomposition analyzePattern and factorize to support rectangular matrices. Added factorize_p method, primarily for the factorization of A'A + I. * DESCRIPTION, NAMESPACE, R/SHLIB.R, R/inline.R, inst/include/RcppEigen.h, inst/include/RcppEigenConfig.h, inst/include/RcppEigenForward.h, inst/include/RcppEigenWrap.h, inst/include/unsupported/Eigen/SparseExtra, inst/include/unsupported/Eigen/src/SparseExtra/CholmodSupport.h, inst/skeleton/Makevars, inst/unitTests/runit.sparse.R: Add support for Cholmod functions linked through the Matrix package. Tests for same. Wrap of an Eigen::SparseMatrix now returns an S4 dgCMatrix object. * inst/include/RcppEigenCholmod.h, inst/include/RcppEigenStubs.cpp: Create stubs and the Cholmod header. (To Do: make the stubs inline functions in a header file.) 2011-07-27 Douglas Bates * inst/unitTests/runit.sparse.R: Separate the tests for sparse matrices. 2011-07-26 Douglas Bates * inst/include/unsupported/Eigen/src/SparseExtra/SimplicialCholesky.h: Added matrixLDL method 2011-07-18 Douglas Bates * src/fastLm.cpp: clean up code, taking advantage of the more general wrap methods 2011-07-15 Douglas Bates * inst/include/RcppEigenForward.h, inst/include/RcppEigenWrap.h, inst/unitTests/runit.RcppEigen.R: added as methods for Eigen::SparseMatrix and Eigen::MappedSparseMatrix classes and tests of same 2011-07-13 Douglas Bates * DESCRIPTION, inst/include/RcppEigenWrap.h, inst/unitTests/runit.RcppEigen.R: Dispatch on wrap to vector according to T::ColsAtCompileTime; modify tests to avoid the .eval() methods; bump Rcpp version dependence 2011-07-13 Romain Francois * inst/include/RcppEigenWrap.h: dispatch sparse/dense and generalizes dealing with sparse objects * inst/include/RcppEigenWrap.h: comment non generic implementations * inst/include/RcppEigenForward.h, inst/include/RcppEigenWrap.h: comment non generic implementations * inst/include/RcppEigenWrap.h: deal with dimensions in eigen_wrap_is_plain * inst/include/RcppEigenForward.h, inst/include/RcppEigenWrap.h: first steps into dealing with Eigen expressions 2011-07-10 Douglas Bates * inst/include/PlainObjectBaseAddon.h: Added some begin and end methods to PlainObjectBase template through the extension mechanism. Should make these legitimate iterators to simplify some wrap methods (need a value_type member). * inst/include/RcppEigenForward.h, inst/include/RcppEigenWrap.h, inst/unitTests/runit.RcppEigen.R: Added as methods for mapped vectors and mapped matrices (still some code duplication) and tests of same. 2011-07-09 Dirk Eddelbuettel * inst/unitTests/runit.RcppEigen.R: s/Armadillo/Eigen/ in a few places 2011-07-09 Douglas Bates * inst/unitTests/runit.RcppEigen.R: Added tests of wrap and as. Need to create an as method for mapped arrays. 2011-07-08 Douglas Bates * DESCRIPTION: Prepare for a new release. * inst/include/RcppEigen.h, inst/include/RcppEigenForward.h, inst/include/RcppEigenWrap.h: Add wrap methods for mapped Eigen objects; Initial support for as<> with some Eigen classes. * inst/include/unsupported/Eigen/src/SparseExtra/SimplicialCholesky.h: Commit interim version of the SimplicialLLT and SimplicialLDLT classes that will appear in Eigen 3.0.2 2011-06-30 Douglas Bates * src/fastLm.cpp: Code simplification suggested by Gael Guennebaud 2011-06-29 Dirk Eddelbuettel * DESCRIPTION: make Maintainers equal to Authors (but keep our joint email address) 2011-06-29 Douglas Bates * inst/include/RcppEigenWrap.h: Syntax errors corrected. 2011-06-28 Douglas Bates * inst/examples/lmBenchmark.R: Print the results from do_bench() so echo=TRUE is not needed when sourcing; add a suppressSVD argument to do_bench(). 2011-06-27 Douglas Bates * inst/include/RcppEigenForward.h, inst/include/RcppEigenWrap.h: Add a wrap method (compiles but currently untested) for Eigen::SparseMatrix * src/fastLm.cpp: Include sample code for the Moore-Penrose inverse. There are better ways of doing this but I can't work out the syntax. 2011-06-25 Douglas Bates * inst/examples, inst/examples/lmBenchmark.R: Add lm benchmark example * src/fastLm.cpp: tighten code a bit 2011-06-23 Douglas Bates * src/fastLm.cpp: Don't try to extract names that aren't there. * man/fastLm.Rd: Add a simple example. * src/fastLm.cpp, src/fastLm.h: Add the SymmEig lm method. Preliminary support for setting a tolerance for the rank calculation. * src/RcppEigen.cpp: Use the correct macros for eigen_version. * man/fastLm.Rd, tests/simple.R: Add examples and test cases for simple crossprod and tcrossprod functions * inst/include/RcppEigenForward.h, inst/include/RcppEigenWrap.h: bypass the incomplete exporter functions to support as<> 2011-06-21 Douglas Bates * src/fastLm.cpp, src/fastLm.h: Add an SVD method for fastLm * DESCRIPTION: Minor fix in punctuation. * R/fastLm.R, inst/unitTests/runit.fastLm.R, man/fastLm.Rd, src/fastLm.cpp, src/fastLm.h: Refactoring of the fastLm code to allow algorithms to be added easily. Adjust docs and R code accordingly. 2011-06-17 Douglas Bates * inst/include/RcppEigenForward.h, inst/include/RcppEigenWrap.h: Add wrap instantiations for ArrayXd, ArrayXXd classes in Eigen 2011-06-15 Douglas Bates * DESCRIPTION, R/RcppEigen.package.skeleton.R, R/SHLIB.R, R/fastLm.R, R/inline.R, R/unit.test.R, inst/doc/RcppEigen-unitTests.Rnw, inst/include/RcppEigen.h, inst/include/RcppEigenConfig.h, inst/include/RcppEigenForward.h, inst/include/RcppEigenWrap.h, inst/unitTests/runit.fastLm.R, man/fastLm.Rd, src/RcppEigen.cpp, src/fastLm.cpp: Change references to Armadillo into Eigen; author order in copyright statements. 2011-06-15 Dirk Eddelbuettel * tests/doRUnit.R: oops: s/Armadillo/Eigen/ 2011-06-15 Douglas Bates * src/fastLm.cpp: Initial (not very good) implementation of "fastLmSVD", which isn't really that fast. * man/fastLm.Rd: Minor clarification. 2011-06-15 Douglas Bates * cleanup, inst/doc/*: Documentation based on unit tests. * R/SHLIB.R, R/inline.R, R/unit.test.R: Add support for inline package * inst/include/RcppEigen.h: Add support for sparse Cholesky and LU * inst/include/unsupported, inst/include/unsupported/*: Add support for sparse Cholesky and LU * man/RcppEigen-package.Rd: Add reference to web page * NAMESPACE, R/RcppEigen.package.skeleton.R, inst/skeleton/*, man/RcppEigen.package.skeleton.Rd: Add RcppEigen.package.skeleton and support files. * DESCRIPTION: Remove Conrad's text about Armadillo, as suggested by Dirk 2011-06-15 Dirk Eddelbuettel * inst/unitTests/runit.fastLm.R: better way to load trees dataset * man/fastLm.Rd: better way to load trees dataset * inst/unitTests/runit.fastLm.R, man/fastLm.Rd: added unit tests for fastLm{Bench,Chol1,Chol2} * ChangeLog, inst/unitTests, inst/unitTests/runTests.R, inst/unitTests/runit.fastLm.R, tests, tests/doRUnit.R: added initial unit tests 2011-06-14 Dirk Eddelbuettel * inst/unitTests/*: Added initial unit tests * tests/doRUnit.R: Added hook to run RUnit tests 2011-06-14 Douglas Bates * src/fastLm.cpp: Cosmetic fixes. * DESCRIPTION: New version. * man/fastLm.Rd: Change order of fastLmPure arguments in the example. Dirk said I would miss one of these. :-) * R/fastLm.R, man/fastLm.Rd, src/fastLm.cpp: Change order of fastLmPure arguments and the various fastLm* C++ functions. Add fastLmChol1 and fastLmChol2. 2011-06-13 Douglas Bates * ChangeLog: Add ChangeLog * src/fastLm.cpp: Handle the rank-deficient case. * inst/include/Eigen/src/LU/arch/Inverse_SSE.h: Use an _m128d type instead of long long int for the mask. * R/fastLm.R: Associate names with coefficients. Clean up fastLm. Forward the object through summary. * inst/include/Eigen/src/Core/util/Meta.h: Suppress use of long long RcppEigen/NAMESPACE0000644000176200001440000000100114520536112013334 0ustar liggesusersuseDynLib("RcppEigen", .registration=TRUE) importFrom("Rcpp", "evalCpp") importFrom("utils", "packageDescription", "package.skeleton") importFrom("stats", "model.frame", "model.matrix", "model.response", "fitted", "coef", "printCoefmat", "pt") export("fastLm", "fastLmPure", "RcppEigen.package.skeleton" ) S3method("fastLm", "default") S3method("fastLm", "formula") S3method("predict", "fastLm") S3method("print", "fastLm") S3method("summary", "fastLm") S3method("print", "summary.fastLm") RcppEigen/LICENSE0000644000176200001440000000050512254315522013134 0ustar liggesusers Please see the included file COPYRIGHTS for per-file (or per-directory) details on licenses as well as copyrights. In short: - Eigen (as an upstream project) in licensed under the MPL-2 (a version of which is included in the file COPYRIGHT), and - the RcppEigen integration into R is licensed as GPL-2 or later. RcppEigen/inst/0000755000176200001440000000000014662150311013102 5ustar liggesusersRcppEigen/inst/include/0000755000176200001440000000000014520536112014525 5ustar liggesusersRcppEigen/inst/include/Eigen/0000755000176200001440000000000014567757725015606 5ustar liggesusersRcppEigen/inst/include/Eigen/Core0000644000176200001440000003077714567757725016437 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008 Gael Guennebaud // Copyright (C) 2007-2011 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_CORE_H #define EIGEN_CORE_H // first thing Eigen does: stop the compiler from reporting useless warnings. #include "src/Core/util/DisableStupidWarnings.h" // then include this file where all our macros are defined. It's really important to do it first because // it's where we do all the compiler/OS/arch detections and define most defaults. #include "src/Core/util/Macros.h" // This detects SSE/AVX/NEON/etc. and configure alignment settings #include "src/Core/util/ConfigureVectorization.h" // We need cuda_runtime.h/hip_runtime.h to ensure that // the EIGEN_USING_STD macro works properly on the device side #if defined(EIGEN_CUDACC) #include #elif defined(EIGEN_HIPCC) #include #endif #ifdef EIGEN_EXCEPTIONS #include #endif // Disable the ipa-cp-clone optimization flag with MinGW 6.x or newer (enabled by default with -O3) // See http://eigen.tuxfamily.org/bz/show_bug.cgi?id=556 for details. #if EIGEN_COMP_MINGW && EIGEN_GNUC_AT_LEAST(4,6) && EIGEN_GNUC_AT_MOST(5,5) #pragma GCC optimize ("-fno-ipa-cp-clone") #endif // Prevent ICC from specializing std::complex operators that silently fail // on device. This allows us to use our own device-compatible specializations // instead. #if defined(EIGEN_COMP_ICC) && defined(EIGEN_GPU_COMPILE_PHASE) \ && !defined(_OVERRIDE_COMPLEX_SPECIALIZATION_) #define _OVERRIDE_COMPLEX_SPECIALIZATION_ 1 #endif #include // this include file manages BLAS and MKL related macros // and inclusion of their respective header files #include "src/Core/util/MKL_support.h" #if defined(EIGEN_HAS_CUDA_FP16) || defined(EIGEN_HAS_HIP_FP16) #define EIGEN_HAS_GPU_FP16 #endif #if defined(EIGEN_HAS_CUDA_BF16) || defined(EIGEN_HAS_HIP_BF16) #define EIGEN_HAS_GPU_BF16 #endif #if (defined _OPENMP) && (!defined EIGEN_DONT_PARALLELIZE) #define EIGEN_HAS_OPENMP #endif #ifdef EIGEN_HAS_OPENMP #include #endif // MSVC for windows mobile does not have the errno.h file #if !(EIGEN_COMP_MSVC && EIGEN_OS_WINCE) && !EIGEN_COMP_ARM #define EIGEN_HAS_ERRNO #endif #ifdef EIGEN_HAS_ERRNO #include #endif #include #include #include #include #include #include #ifndef EIGEN_NO_IO #include #endif #include #include #include #include // for CHAR_BIT // for min/max: #include #if EIGEN_HAS_CXX11 #include #endif // for std::is_nothrow_move_assignable #ifdef EIGEN_INCLUDE_TYPE_TRAITS #include #endif // for outputting debug info #ifdef EIGEN_DEBUG_ASSIGN #include #endif // required for __cpuid, needs to be included after cmath #if EIGEN_COMP_MSVC && EIGEN_ARCH_i386_OR_x86_64 && !EIGEN_OS_WINCE #include #endif #if defined(EIGEN_USE_SYCL) #undef min #undef max #undef isnan #undef isinf #undef isfinite #include #include #include #include #include #ifndef EIGEN_SYCL_LOCAL_THREAD_DIM0 #define EIGEN_SYCL_LOCAL_THREAD_DIM0 16 #endif #ifndef EIGEN_SYCL_LOCAL_THREAD_DIM1 #define EIGEN_SYCL_LOCAL_THREAD_DIM1 16 #endif #endif #if defined EIGEN2_SUPPORT_STAGE40_FULL_EIGEN3_STRICTNESS || defined EIGEN2_SUPPORT_STAGE30_FULL_EIGEN3_API || defined EIGEN2_SUPPORT_STAGE20_RESOLVE_API_CONFLICTS || defined EIGEN2_SUPPORT_STAGE10_FULL_EIGEN2_API || defined EIGEN2_SUPPORT // This will generate an error message: #error Eigen2-support is only available up to version 3.2. Please go to "http://eigen.tuxfamily.org/index.php?title=Eigen2" for further information #endif namespace Eigen { // we use size_t frequently and we'll never remember to prepend it with std:: every time just to // ensure QNX/QCC support using std::size_t; // gcc 4.6.0 wants std:: for ptrdiff_t using std::ptrdiff_t; } /** \defgroup Core_Module Core module * This is the main module of Eigen providing dense matrix and vector support * (both fixed and dynamic size) with all the features corresponding to a BLAS library * and much more... * * \code * #include * \endcode */ #include "src/Core/util/Constants.h" #include "src/Core/util/Meta.h" #include "src/Core/util/ForwardDeclarations.h" #include "src/Core/util/StaticAssert.h" #include "src/Core/util/XprHelper.h" #include "src/Core/util/Memory.h" #include "src/Core/util/IntegralConstant.h" #include "src/Core/util/SymbolicIndex.h" #include "src/Core/NumTraits.h" #include "src/Core/MathFunctions.h" #include "src/Core/GenericPacketMath.h" #include "src/Core/MathFunctionsImpl.h" #include "src/Core/arch/Default/ConjHelper.h" // Generic half float support #include "src/Core/arch/Default/Half.h" #include "src/Core/arch/Default/BFloat16.h" #include "src/Core/arch/Default/TypeCasting.h" #include "src/Core/arch/Default/GenericPacketMathFunctionsFwd.h" #if defined EIGEN_VECTORIZE_AVX512 #include "src/Core/arch/SSE/PacketMath.h" #include "src/Core/arch/SSE/TypeCasting.h" #include "src/Core/arch/SSE/Complex.h" #include "src/Core/arch/AVX/PacketMath.h" #include "src/Core/arch/AVX/TypeCasting.h" #include "src/Core/arch/AVX/Complex.h" #include "src/Core/arch/AVX512/PacketMath.h" #include "src/Core/arch/AVX512/TypeCasting.h" #include "src/Core/arch/AVX512/Complex.h" #include "src/Core/arch/SSE/MathFunctions.h" #include "src/Core/arch/AVX/MathFunctions.h" #include "src/Core/arch/AVX512/MathFunctions.h" #elif defined EIGEN_VECTORIZE_AVX // Use AVX for floats and doubles, SSE for integers #include "src/Core/arch/SSE/PacketMath.h" #include "src/Core/arch/SSE/TypeCasting.h" #include "src/Core/arch/SSE/Complex.h" #include "src/Core/arch/AVX/PacketMath.h" #include "src/Core/arch/AVX/TypeCasting.h" #include "src/Core/arch/AVX/Complex.h" #include "src/Core/arch/SSE/MathFunctions.h" #include "src/Core/arch/AVX/MathFunctions.h" #elif defined EIGEN_VECTORIZE_SSE #include "src/Core/arch/SSE/PacketMath.h" #include "src/Core/arch/SSE/TypeCasting.h" #include "src/Core/arch/SSE/MathFunctions.h" #include "src/Core/arch/SSE/Complex.h" #elif defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) #include "src/Core/arch/AltiVec/PacketMath.h" #include "src/Core/arch/AltiVec/MathFunctions.h" #include "src/Core/arch/AltiVec/Complex.h" #elif defined EIGEN_VECTORIZE_NEON #include "src/Core/arch/NEON/PacketMath.h" #include "src/Core/arch/NEON/TypeCasting.h" #include "src/Core/arch/NEON/MathFunctions.h" #include "src/Core/arch/NEON/Complex.h" #elif defined EIGEN_VECTORIZE_SVE #include "src/Core/arch/SVE/PacketMath.h" #include "src/Core/arch/SVE/TypeCasting.h" #include "src/Core/arch/SVE/MathFunctions.h" #elif defined EIGEN_VECTORIZE_ZVECTOR #include "src/Core/arch/ZVector/PacketMath.h" #include "src/Core/arch/ZVector/MathFunctions.h" #include "src/Core/arch/ZVector/Complex.h" #elif defined EIGEN_VECTORIZE_MSA #include "src/Core/arch/MSA/PacketMath.h" #include "src/Core/arch/MSA/MathFunctions.h" #include "src/Core/arch/MSA/Complex.h" #endif #if defined EIGEN_VECTORIZE_GPU #include "src/Core/arch/GPU/PacketMath.h" #include "src/Core/arch/GPU/MathFunctions.h" #include "src/Core/arch/GPU/TypeCasting.h" #endif #if defined(EIGEN_USE_SYCL) #include "src/Core/arch/SYCL/SyclMemoryModel.h" #include "src/Core/arch/SYCL/InteropHeaders.h" #if !defined(EIGEN_DONT_VECTORIZE_SYCL) #include "src/Core/arch/SYCL/PacketMath.h" #include "src/Core/arch/SYCL/MathFunctions.h" #include "src/Core/arch/SYCL/TypeCasting.h" #endif #endif #include "src/Core/arch/Default/Settings.h" // This file provides generic implementations valid for scalar as well #include "src/Core/arch/Default/GenericPacketMathFunctions.h" #include "src/Core/functors/TernaryFunctors.h" #include "src/Core/functors/BinaryFunctors.h" #include "src/Core/functors/UnaryFunctors.h" #include "src/Core/functors/NullaryFunctors.h" #include "src/Core/functors/StlFunctors.h" #include "src/Core/functors/AssignmentFunctors.h" // Specialized functors to enable the processing of complex numbers // on CUDA devices #ifdef EIGEN_CUDACC #include "src/Core/arch/CUDA/Complex.h" #endif #include "src/Core/util/IndexedViewHelper.h" #include "src/Core/util/ReshapedHelper.h" #include "src/Core/ArithmeticSequence.h" #ifndef EIGEN_NO_IO #include "src/Core/IO.h" #endif #include "src/Core/DenseCoeffsBase.h" #include "src/Core/DenseBase.h" #include "src/Core/MatrixBase.h" #include "src/Core/EigenBase.h" #include "src/Core/Product.h" #include "src/Core/CoreEvaluators.h" #include "src/Core/AssignEvaluator.h" #ifndef EIGEN_PARSED_BY_DOXYGEN // work around Doxygen bug triggered by Assign.h r814874 // at least confirmed with Doxygen 1.5.5 and 1.5.6 #include "src/Core/Assign.h" #endif #include "src/Core/ArrayBase.h" #include "src/Core/util/BlasUtil.h" #include "src/Core/DenseStorage.h" #include "src/Core/NestByValue.h" // #include "src/Core/ForceAlignedAccess.h" #include "src/Core/ReturnByValue.h" #include "src/Core/NoAlias.h" #include "src/Core/PlainObjectBase.h" #include "src/Core/Matrix.h" #include "src/Core/Array.h" #include "src/Core/CwiseTernaryOp.h" #include "src/Core/CwiseBinaryOp.h" #include "src/Core/CwiseUnaryOp.h" #include "src/Core/CwiseNullaryOp.h" #include "src/Core/CwiseUnaryView.h" #include "src/Core/SelfCwiseBinaryOp.h" #include "src/Core/Dot.h" #include "src/Core/StableNorm.h" #include "src/Core/Stride.h" #include "src/Core/MapBase.h" #include "src/Core/Map.h" #include "src/Core/Ref.h" #include "src/Core/Block.h" #include "src/Core/VectorBlock.h" #include "src/Core/IndexedView.h" #include "src/Core/Reshaped.h" #include "src/Core/Transpose.h" #include "src/Core/DiagonalMatrix.h" #include "src/Core/Diagonal.h" #include "src/Core/DiagonalProduct.h" #include "src/Core/Redux.h" #include "src/Core/Visitor.h" #include "src/Core/Fuzzy.h" #include "src/Core/Swap.h" #include "src/Core/CommaInitializer.h" #include "src/Core/GeneralProduct.h" #include "src/Core/Solve.h" #include "src/Core/Inverse.h" #include "src/Core/SolverBase.h" #include "src/Core/PermutationMatrix.h" #include "src/Core/Transpositions.h" #include "src/Core/TriangularMatrix.h" #include "src/Core/SelfAdjointView.h" #include "src/Core/products/GeneralBlockPanelKernel.h" #include "src/Core/products/Parallelizer.h" #include "src/Core/ProductEvaluators.h" #include "src/Core/products/GeneralMatrixVector.h" #include "src/Core/products/GeneralMatrixMatrix.h" #include "src/Core/SolveTriangular.h" #include "src/Core/products/GeneralMatrixMatrixTriangular.h" #include "src/Core/products/SelfadjointMatrixVector.h" #include "src/Core/products/SelfadjointMatrixMatrix.h" #include "src/Core/products/SelfadjointProduct.h" #include "src/Core/products/SelfadjointRank2Update.h" #include "src/Core/products/TriangularMatrixVector.h" #include "src/Core/products/TriangularMatrixMatrix.h" #include "src/Core/products/TriangularSolverMatrix.h" #include "src/Core/products/TriangularSolverVector.h" #include "src/Core/BandMatrix.h" #include "src/Core/CoreIterators.h" #include "src/Core/ConditionEstimator.h" #if defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) #include "src/Core/arch/AltiVec/MatrixProduct.h" #elif defined EIGEN_VECTORIZE_NEON #include "src/Core/arch/NEON/GeneralBlockPanelKernel.h" #endif #include "src/Core/BooleanRedux.h" #include "src/Core/Select.h" #include "src/Core/VectorwiseOp.h" #include "src/Core/PartialReduxEvaluator.h" #include "src/Core/Random.h" #include "src/Core/Replicate.h" #include "src/Core/Reverse.h" #include "src/Core/ArrayWrapper.h" #include "src/Core/StlIterators.h" #ifdef EIGEN_USE_BLAS #include "src/Core/products/GeneralMatrixMatrix_BLAS.h" #include "src/Core/products/GeneralMatrixVector_BLAS.h" #include "src/Core/products/GeneralMatrixMatrixTriangular_BLAS.h" #include "src/Core/products/SelfadjointMatrixMatrix_BLAS.h" #include "src/Core/products/SelfadjointMatrixVector_BLAS.h" #include "src/Core/products/TriangularMatrixMatrix_BLAS.h" #include "src/Core/products/TriangularMatrixVector_BLAS.h" #include "src/Core/products/TriangularSolverMatrix_BLAS.h" #endif // EIGEN_USE_BLAS #ifdef EIGEN_USE_MKL_VML #include "src/Core/Assign_MKL.h" #endif #include "src/Core/GlobalFunctions.h" #include "src/Core/util/ReenableStupidWarnings.h" #endif // EIGEN_CORE_H RcppEigen/inst/include/Eigen/Eigen0000644000176200001440000000004314107270226016525 0ustar liggesusers#include "Dense" #include "Sparse" RcppEigen/inst/include/Eigen/SuperLUSupport0000644000176200001440000000430314107270226020435 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_SUPERLUSUPPORT_MODULE_H #define EIGEN_SUPERLUSUPPORT_MODULE_H #include "SparseCore" #include "src/Core/util/DisableStupidWarnings.h" #ifdef EMPTY #define EIGEN_EMPTY_WAS_ALREADY_DEFINED #endif typedef int int_t; #include #include #include // slu_util.h defines a preprocessor token named EMPTY which is really polluting, // so we remove it in favor of a SUPERLU_EMPTY token. // If EMPTY was already defined then we don't undef it. #if defined(EIGEN_EMPTY_WAS_ALREADY_DEFINED) # undef EIGEN_EMPTY_WAS_ALREADY_DEFINED #elif defined(EMPTY) # undef EMPTY #endif #define SUPERLU_EMPTY (-1) namespace Eigen { struct SluMatrix; } /** \ingroup Support_modules * \defgroup SuperLUSupport_Module SuperLUSupport module * * This module provides an interface to the SuperLU library. * It provides the following factorization class: * - class SuperLU: a supernodal sequential LU factorization. * - class SuperILU: a supernodal sequential incomplete LU factorization (to be used as a preconditioner for iterative methods). * * \warning This wrapper requires at least versions 4.0 of SuperLU. The 3.x versions are not supported. * * \warning When including this module, you have to use SUPERLU_EMPTY instead of EMPTY which is no longer defined because it is too polluting. * * \code * #include * \endcode * * In order to use this module, the superlu headers must be accessible from the include paths, and your binary must be linked to the superlu library and its dependencies. * The dependencies depend on how superlu has been compiled. * For a cmake based project, you can use our FindSuperLU.cmake module to help you in this task. * */ #include "src/SuperLUSupport/SuperLUSupport.h" #include "src/Core/util/ReenableStupidWarnings.h" #endif // EIGEN_SUPERLUSUPPORT_MODULE_H RcppEigen/inst/include/Eigen/SparseCholesky0000644000176200001440000000232314567757725020470 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2013 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_SPARSECHOLESKY_MODULE_H #define EIGEN_SPARSECHOLESKY_MODULE_H #include "SparseCore" #include "OrderingMethods" #include "src/Core/util/DisableStupidWarnings.h" /** * \defgroup SparseCholesky_Module SparseCholesky module * * This module currently provides two variants of the direct sparse Cholesky decomposition for selfadjoint (hermitian) matrices. * Those decompositions are accessible via the following classes: * - SimplicialLLt, * - SimplicialLDLt * * Such problems can also be solved using the ConjugateGradient solver from the IterativeLinearSolvers module. * * \code * #include * \endcode */ #include "src/SparseCholesky/SimplicialCholesky.h" #include "src/SparseCholesky/SimplicialCholesky_impl.h" #include "src/Core/util/ReenableStupidWarnings.h" #endif // EIGEN_SPARSECHOLESKY_MODULE_H RcppEigen/inst/include/Eigen/Sparse0000644000176200001440000000157014567757725016771 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_SPARSE_MODULE_H #define EIGEN_SPARSE_MODULE_H /** \defgroup Sparse_Module Sparse meta-module * * Meta-module including all related modules: * - \ref SparseCore_Module * - \ref OrderingMethods_Module * - \ref SparseCholesky_Module * - \ref SparseLU_Module * - \ref SparseQR_Module * - \ref IterativeLinearSolvers_Module * \code #include \endcode */ #include "SparseCore" #include "OrderingMethods" #include "SparseCholesky" #include "SparseLU" #include "SparseQR" #include "IterativeLinearSolvers" #endif // EIGEN_SPARSE_MODULE_H RcppEigen/inst/include/Eigen/IterativeLinearSolvers0000644000176200001440000000404314107270226022147 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_ITERATIVELINEARSOLVERS_MODULE_H #define EIGEN_ITERATIVELINEARSOLVERS_MODULE_H #include "SparseCore" #include "OrderingMethods" #include "src/Core/util/DisableStupidWarnings.h" /** * \defgroup IterativeLinearSolvers_Module IterativeLinearSolvers module * * This module currently provides iterative methods to solve problems of the form \c A \c x = \c b, where \c A is a squared matrix, usually very large and sparse. * Those solvers are accessible via the following classes: * - ConjugateGradient for selfadjoint (hermitian) matrices, * - LeastSquaresConjugateGradient for rectangular least-square problems, * - BiCGSTAB for general square matrices. * * These iterative solvers are associated with some preconditioners: * - IdentityPreconditioner - not really useful * - DiagonalPreconditioner - also called Jacobi preconditioner, work very well on diagonal dominant matrices. * - IncompleteLUT - incomplete LU factorization with dual thresholding * * Such problems can also be solved using the direct sparse decomposition modules: SparseCholesky, CholmodSupport, UmfPackSupport, SuperLUSupport. * \code #include \endcode */ #include "src/IterativeLinearSolvers/SolveWithGuess.h" #include "src/IterativeLinearSolvers/IterativeSolverBase.h" #include "src/IterativeLinearSolvers/BasicPreconditioners.h" #include "src/IterativeLinearSolvers/ConjugateGradient.h" #include "src/IterativeLinearSolvers/LeastSquareConjugateGradient.h" #include "src/IterativeLinearSolvers/BiCGSTAB.h" #include "src/IterativeLinearSolvers/IncompleteLUT.h" #include "src/IterativeLinearSolvers/IncompleteCholesky.h" #include "src/Core/util/ReenableStupidWarnings.h" #endif // EIGEN_ITERATIVELINEARSOLVERS_MODULE_H RcppEigen/inst/include/Eigen/SPQRSupport0000644000176200001440000000221214107270226017660 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_SPQRSUPPORT_MODULE_H #define EIGEN_SPQRSUPPORT_MODULE_H #include "SparseCore" #include "src/Core/util/DisableStupidWarnings.h" #include "SuiteSparseQR.hpp" /** \ingroup Support_modules * \defgroup SPQRSupport_Module SuiteSparseQR module * * This module provides an interface to the SPQR library, which is part of the suitesparse package. * * \code * #include * \endcode * * In order to use this module, the SPQR headers must be accessible from the include paths, and your binary must be linked to the SPQR library and its dependencies (Cholmod, AMD, COLAMD,...). * For a cmake based project, you can use our FindSPQR.cmake and FindCholmod.Cmake modules * */ #include "src/CholmodSupport/CholmodSupport.h" #include "src/SPQRSupport/SuiteSparseQRSupport.h" #endif RcppEigen/inst/include/Eigen/PaStiXSupport0000644000176200001440000000332714567757725020303 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_PASTIXSUPPORT_MODULE_H #define EIGEN_PASTIXSUPPORT_MODULE_H #include "SparseCore" #include "src/Core/util/DisableStupidWarnings.h" extern "C" { #include #include } #ifdef complex #undef complex #endif /** \ingroup Support_modules * \defgroup PaStiXSupport_Module PaStiXSupport module * * This module provides an interface to the PaSTiX library. * PaSTiX is a general \b supernodal, \b parallel and \b opensource sparse solver. * It provides the two following main factorization classes: * - class PastixLLT : a supernodal, parallel LLt Cholesky factorization. * - class PastixLDLT: a supernodal, parallel LDLt Cholesky factorization. * - class PastixLU : a supernodal, parallel LU factorization (optimized for a symmetric pattern). * * \code * #include * \endcode * * In order to use this module, the PaSTiX headers must be accessible from the include paths, and your binary must be linked to the PaSTiX library and its dependencies. * This wrapper resuires PaStiX version 5.x compiled without MPI support. * The dependencies depend on how PaSTiX has been compiled. * For a cmake based project, you can use our FindPaSTiX.cmake module to help you in this task. * */ #include "src/PaStiXSupport/PaStiXSupport.h" #include "src/Core/util/ReenableStupidWarnings.h" #endif // EIGEN_PASTIXSUPPORT_MODULE_H RcppEigen/inst/include/Eigen/PardisoSupport0000644000176200001440000000213414567757725020527 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_PARDISOSUPPORT_MODULE_H #define EIGEN_PARDISOSUPPORT_MODULE_H #include "SparseCore" #include "src/Core/util/DisableStupidWarnings.h" #include /** \ingroup Support_modules * \defgroup PardisoSupport_Module PardisoSupport module * * This module brings support for the Intel(R) MKL PARDISO direct sparse solvers. * * \code * #include * \endcode * * In order to use this module, the MKL headers must be accessible from the include paths, and your binary must be linked to the MKL library and its dependencies. * See this \ref TopicUsingIntelMKL "page" for more information on MKL-Eigen integration. * */ #include "src/PardisoSupport/PardisoSupport.h" #include "src/Core/util/ReenableStupidWarnings.h" #endif // EIGEN_PARDISOSUPPORT_MODULE_H RcppEigen/inst/include/Eigen/StdList0000644000176200001440000000132614107270226017071 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2009 Hauke Heibel // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_STDLIST_MODULE_H #define EIGEN_STDLIST_MODULE_H #include "Core" #include #if EIGEN_COMP_MSVC && EIGEN_OS_WIN64 && (EIGEN_MAX_STATIC_ALIGN_BYTES<=16) /* MSVC auto aligns up to 16 bytes in 64 bit builds */ #define EIGEN_DEFINE_STL_LIST_SPECIALIZATION(...) #else #include "src/StlSupport/StdList.h" #endif #endif // EIGEN_STDLIST_MODULE_H RcppEigen/inst/include/Eigen/StdVector0000644000176200001440000000144314107270226017420 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2009 Gael Guennebaud // Copyright (C) 2009 Hauke Heibel // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_STDVECTOR_MODULE_H #define EIGEN_STDVECTOR_MODULE_H #include "Core" #include #if EIGEN_COMP_MSVC && EIGEN_OS_WIN64 && (EIGEN_MAX_STATIC_ALIGN_BYTES<=16) /* MSVC auto aligns up to 16 bytes in 64 bit builds */ #define EIGEN_DEFINE_STL_VECTOR_SPECIALIZATION(...) #else #include "src/StlSupport/StdVector.h" #endif #endif // EIGEN_STDVECTOR_MODULE_H RcppEigen/inst/include/Eigen/Householder0000644000176200001440000000147514567757725020021 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_HOUSEHOLDER_MODULE_H #define EIGEN_HOUSEHOLDER_MODULE_H #include "Core" #include "src/Core/util/DisableStupidWarnings.h" /** \defgroup Householder_Module Householder module * This module provides Householder transformations. * * \code * #include * \endcode */ #include "src/Householder/Householder.h" #include "src/Householder/HouseholderSequence.h" #include "src/Householder/BlockHouseholder.h" #include "src/Core/util/ReenableStupidWarnings.h" #endif // EIGEN_HOUSEHOLDER_MODULE_H RcppEigen/inst/include/Eigen/src/0000755000176200001440000000000014567757725016375 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/0000755000176200001440000000000014567757725017265 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/SolveTriangular.h0000644000176200001440000002223014567757725022556 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2009 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_SOLVETRIANGULAR_H #define EIGEN_SOLVETRIANGULAR_H namespace Eigen { namespace internal { // Forward declarations: // The following two routines are implemented in the products/TriangularSolver*.h files template struct triangular_solve_vector; template struct triangular_solve_matrix; // small helper struct extracting some traits on the underlying solver operation template class trsolve_traits { private: enum { RhsIsVectorAtCompileTime = (Side==OnTheLeft ? Rhs::ColsAtCompileTime : Rhs::RowsAtCompileTime)==1 }; public: enum { Unrolling = (RhsIsVectorAtCompileTime && Rhs::SizeAtCompileTime != Dynamic && Rhs::SizeAtCompileTime <= 8) ? CompleteUnrolling : NoUnrolling, RhsVectors = RhsIsVectorAtCompileTime ? 1 : Dynamic }; }; template::Unrolling, int RhsVectors = trsolve_traits::RhsVectors > struct triangular_solver_selector; template struct triangular_solver_selector { typedef typename Lhs::Scalar LhsScalar; typedef typename Rhs::Scalar RhsScalar; typedef blas_traits LhsProductTraits; typedef typename LhsProductTraits::ExtractType ActualLhsType; typedef Map, Aligned> MappedRhs; static EIGEN_DEVICE_FUNC void run(const Lhs& lhs, Rhs& rhs) { ActualLhsType actualLhs = LhsProductTraits::extract(lhs); // FIXME find a way to allow an inner stride if packet_traits::size==1 bool useRhsDirectly = Rhs::InnerStrideAtCompileTime==1 || rhs.innerStride()==1; ei_declare_aligned_stack_constructed_variable(RhsScalar,actualRhs,rhs.size(), (useRhsDirectly ? rhs.data() : 0)); if(!useRhsDirectly) MappedRhs(actualRhs,rhs.size()) = rhs; triangular_solve_vector ::run(actualLhs.cols(), actualLhs.data(), actualLhs.outerStride(), actualRhs); if(!useRhsDirectly) rhs = MappedRhs(actualRhs, rhs.size()); } }; // the rhs is a matrix template struct triangular_solver_selector { typedef typename Rhs::Scalar Scalar; typedef blas_traits LhsProductTraits; typedef typename LhsProductTraits::DirectLinearAccessType ActualLhsType; static EIGEN_DEVICE_FUNC void run(const Lhs& lhs, Rhs& rhs) { typename internal::add_const_on_value_type::type actualLhs = LhsProductTraits::extract(lhs); const Index size = lhs.rows(); const Index othersize = Side==OnTheLeft? rhs.cols() : rhs.rows(); typedef internal::gemm_blocking_space<(Rhs::Flags&RowMajorBit) ? RowMajor : ColMajor,Scalar,Scalar, Rhs::MaxRowsAtCompileTime, Rhs::MaxColsAtCompileTime, Lhs::MaxRowsAtCompileTime,4> BlockingType; BlockingType blocking(rhs.rows(), rhs.cols(), size, 1, false); triangular_solve_matrix ::run(size, othersize, &actualLhs.coeffRef(0,0), actualLhs.outerStride(), &rhs.coeffRef(0,0), rhs.innerStride(), rhs.outerStride(), blocking); } }; /*************************************************************************** * meta-unrolling implementation ***************************************************************************/ template struct triangular_solver_unroller; template struct triangular_solver_unroller { enum { IsLower = ((Mode&Lower)==Lower), DiagIndex = IsLower ? LoopIndex : Size - LoopIndex - 1, StartIndex = IsLower ? 0 : DiagIndex+1 }; static EIGEN_DEVICE_FUNC void run(const Lhs& lhs, Rhs& rhs) { if (LoopIndex>0) rhs.coeffRef(DiagIndex) -= lhs.row(DiagIndex).template segment(StartIndex).transpose() .cwiseProduct(rhs.template segment(StartIndex)).sum(); if(!(Mode & UnitDiag)) rhs.coeffRef(DiagIndex) /= lhs.coeff(DiagIndex,DiagIndex); triangular_solver_unroller::run(lhs,rhs); } }; template struct triangular_solver_unroller { static EIGEN_DEVICE_FUNC void run(const Lhs&, Rhs&) {} }; template struct triangular_solver_selector { static EIGEN_DEVICE_FUNC void run(const Lhs& lhs, Rhs& rhs) { triangular_solver_unroller::run(lhs,rhs); } }; template struct triangular_solver_selector { static EIGEN_DEVICE_FUNC void run(const Lhs& lhs, Rhs& rhs) { Transpose trLhs(lhs); Transpose trRhs(rhs); triangular_solver_unroller,Transpose, ((Mode&Upper)==Upper ? Lower : Upper) | (Mode&UnitDiag), 0,Rhs::SizeAtCompileTime>::run(trLhs,trRhs); } }; } // end namespace internal /*************************************************************************** * TriangularView methods ***************************************************************************/ #ifndef EIGEN_PARSED_BY_DOXYGEN template template EIGEN_DEVICE_FUNC void TriangularViewImpl::solveInPlace(const MatrixBase& _other) const { OtherDerived& other = _other.const_cast_derived(); eigen_assert( derived().cols() == derived().rows() && ((Side==OnTheLeft && derived().cols() == other.rows()) || (Side==OnTheRight && derived().cols() == other.cols())) ); eigen_assert((!(int(Mode) & int(ZeroDiag))) && bool(int(Mode) & (int(Upper) | int(Lower)))); // If solving for a 0x0 matrix, nothing to do, simply return. if (derived().cols() == 0) return; enum { copy = (internal::traits::Flags & RowMajorBit) && OtherDerived::IsVectorAtCompileTime && OtherDerived::SizeAtCompileTime!=1}; typedef typename internal::conditional::type, OtherDerived&>::type OtherCopy; OtherCopy otherCopy(other); internal::triangular_solver_selector::type, Side, Mode>::run(derived().nestedExpression(), otherCopy); if (copy) other = otherCopy; } template template const internal::triangular_solve_retval,Other> TriangularViewImpl::solve(const MatrixBase& other) const { return internal::triangular_solve_retval(derived(), other.derived()); } #endif namespace internal { template struct traits > { typedef typename internal::plain_matrix_type_column_major::type ReturnType; }; template struct triangular_solve_retval : public ReturnByValue > { typedef typename remove_all::type RhsNestedCleaned; typedef ReturnByValue Base; triangular_solve_retval(const TriangularType& tri, const Rhs& rhs) : m_triangularMatrix(tri), m_rhs(rhs) {} inline EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_rhs.rows(); } inline EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_rhs.cols(); } template inline void evalTo(Dest& dst) const { if(!is_same_dense(dst,m_rhs)) dst = m_rhs; m_triangularMatrix.template solveInPlace(dst); } protected: const TriangularType& m_triangularMatrix; typename Rhs::Nested m_rhs; }; } // namespace internal } // end namespace Eigen #endif // EIGEN_SOLVETRIANGULAR_H RcppEigen/inst/include/Eigen/src/Core/PartialReduxEvaluator.h0000644000176200001440000002176714567757725023742 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2011-2018 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_PARTIALREDUX_H #define EIGEN_PARTIALREDUX_H namespace Eigen { namespace internal { /*************************************************************************** * * This file provides evaluators for partial reductions. * There are two modes: * * - scalar path: simply calls the respective function on the column or row. * -> nothing special here, all the tricky part is handled by the return * types of VectorwiseOp's members. They embed the functor calling the * respective DenseBase's member function. * * - vectorized path: implements a packet-wise reductions followed by * some (optional) processing of the outcome, e.g., division by n for mean. * * For the vectorized path let's observe that the packet-size and outer-unrolling * are both decided by the assignement logic. So all we have to do is to decide * on the inner unrolling. * * For the unrolling, we can reuse "internal::redux_vec_unroller" from Redux.h, * but be need to be careful to specify correct increment. * ***************************************************************************/ /* logic deciding a strategy for unrolling of vectorized paths */ template struct packetwise_redux_traits { enum { OuterSize = int(Evaluator::IsRowMajor) ? Evaluator::RowsAtCompileTime : Evaluator::ColsAtCompileTime, Cost = OuterSize == Dynamic ? HugeCost : OuterSize * Evaluator::CoeffReadCost + (OuterSize-1) * functor_traits::Cost, Unrolling = Cost <= EIGEN_UNROLLING_LIMIT ? CompleteUnrolling : NoUnrolling }; }; /* Value to be returned when size==0 , by default let's return 0 */ template EIGEN_DEVICE_FUNC PacketType packetwise_redux_empty_value(const Func& ) { return pset1(0); } /* For products the default is 1 */ template EIGEN_DEVICE_FUNC PacketType packetwise_redux_empty_value(const scalar_product_op& ) { return pset1(1); } /* Perform the actual reduction */ template::Unrolling > struct packetwise_redux_impl; /* Perform the actual reduction with unrolling */ template struct packetwise_redux_impl { typedef redux_novec_unroller Base; typedef typename Evaluator::Scalar Scalar; template EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE PacketType run(const Evaluator &eval, const Func& func, Index /*size*/) { return redux_vec_unroller::OuterSize>::template run(eval,func); } }; /* Add a specialization of redux_vec_unroller for size==0 at compiletime. * This specialization is not required for general reductions, which is * why it is defined here. */ template struct redux_vec_unroller { template EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE PacketType run(const Evaluator &, const Func& f) { return packetwise_redux_empty_value(f); } }; /* Perform the actual reduction for dynamic sizes */ template struct packetwise_redux_impl { typedef typename Evaluator::Scalar Scalar; typedef typename redux_traits::PacketType PacketScalar; template EIGEN_DEVICE_FUNC static PacketType run(const Evaluator &eval, const Func& func, Index size) { if(size==0) return packetwise_redux_empty_value(func); const Index size4 = (size-1)&(~3); PacketType p = eval.template packetByOuterInner(0,0); Index i = 1; // This loop is optimized for instruction pipelining: // - each iteration generates two independent instructions // - thanks to branch prediction and out-of-order execution we have independent instructions across loops for(; i(i+0,0),eval.template packetByOuterInner(i+1,0)), func.packetOp(eval.template packetByOuterInner(i+2,0),eval.template packetByOuterInner(i+3,0)))); for(; i(i,0)); return p; } }; template< typename ArgType, typename MemberOp, int Direction> struct evaluator > : evaluator_base > { typedef PartialReduxExpr XprType; typedef typename internal::nested_eval::type ArgTypeNested; typedef typename internal::add_const_on_value_type::type ConstArgTypeNested; typedef typename internal::remove_all::type ArgTypeNestedCleaned; typedef typename ArgType::Scalar InputScalar; typedef typename XprType::Scalar Scalar; enum { TraversalSize = Direction==int(Vertical) ? int(ArgType::RowsAtCompileTime) : int(ArgType::ColsAtCompileTime) }; typedef typename MemberOp::template Cost CostOpType; enum { CoeffReadCost = TraversalSize==Dynamic ? HugeCost : TraversalSize==0 ? 1 : int(TraversalSize) * int(evaluator::CoeffReadCost) + int(CostOpType::value), _ArgFlags = evaluator::Flags, _Vectorizable = bool(int(_ArgFlags)&PacketAccessBit) && bool(MemberOp::Vectorizable) && (Direction==int(Vertical) ? bool(_ArgFlags&RowMajorBit) : (_ArgFlags&RowMajorBit)==0) && (TraversalSize!=0), Flags = (traits::Flags&RowMajorBit) | (evaluator::Flags&(HereditaryBits&(~RowMajorBit))) | (_Vectorizable ? PacketAccessBit : 0) | LinearAccessBit, Alignment = 0 // FIXME this will need to be improved once PartialReduxExpr is vectorized }; EIGEN_DEVICE_FUNC explicit evaluator(const XprType xpr) : m_arg(xpr.nestedExpression()), m_functor(xpr.functor()) { EIGEN_INTERNAL_CHECK_COST_VALUE(TraversalSize==Dynamic ? HugeCost : (TraversalSize==0 ? 1 : int(CostOpType::value))); EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } typedef typename XprType::CoeffReturnType CoeffReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar coeff(Index i, Index j) const { return coeff(Direction==Vertical ? j : i); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar coeff(Index index) const { return m_functor(m_arg.template subVector(index)); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index i, Index j) const { return packet(Direction==Vertical ? j : i); } template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC PacketType packet(Index idx) const { enum { PacketSize = internal::unpacket_traits::size }; typedef Block PanelType; PanelType panel(m_arg, Direction==Vertical ? 0 : idx, Direction==Vertical ? idx : 0, Direction==Vertical ? m_arg.rows() : Index(PacketSize), Direction==Vertical ? Index(PacketSize) : m_arg.cols()); // FIXME // See bug 1612, currently if PacketSize==1 (i.e. complex with 128bits registers) then the storage-order of panel get reversed // and methods like packetByOuterInner do not make sense anymore in this context. // So let's just by pass "vectorization" in this case: if(PacketSize==1) return internal::pset1(coeff(idx)); typedef typename internal::redux_evaluator PanelEvaluator; PanelEvaluator panel_eval(panel); typedef typename MemberOp::BinaryOp BinaryOp; PacketType p = internal::packetwise_redux_impl::template run(panel_eval,m_functor.binaryFunc(),m_arg.outerSize()); return p; } protected: ConstArgTypeNested m_arg; const MemberOp m_functor; }; } // end namespace internal } // end namespace Eigen #endif // EIGEN_PARTIALREDUX_H RcppEigen/inst/include/Eigen/src/Core/Reverse.h0000644000176200001440000001654214567757725021061 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2006-2008 Benoit Jacob // Copyright (C) 2009 Ricard Marxer // Copyright (C) 2009-2010 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_REVERSE_H #define EIGEN_REVERSE_H namespace Eigen { namespace internal { template struct traits > : traits { typedef typename MatrixType::Scalar Scalar; typedef typename traits::StorageKind StorageKind; typedef typename traits::XprKind XprKind; typedef typename ref_selector::type MatrixTypeNested; typedef typename remove_reference::type _MatrixTypeNested; enum { RowsAtCompileTime = MatrixType::RowsAtCompileTime, ColsAtCompileTime = MatrixType::ColsAtCompileTime, MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime, Flags = _MatrixTypeNested::Flags & (RowMajorBit | LvalueBit) }; }; template struct reverse_packet_cond { static inline PacketType run(const PacketType& x) { return preverse(x); } }; template struct reverse_packet_cond { static inline PacketType run(const PacketType& x) { return x; } }; } // end namespace internal /** \class Reverse * \ingroup Core_Module * * \brief Expression of the reverse of a vector or matrix * * \tparam MatrixType the type of the object of which we are taking the reverse * \tparam Direction defines the direction of the reverse operation, can be Vertical, Horizontal, or BothDirections * * This class represents an expression of the reverse of a vector. * It is the return type of MatrixBase::reverse() and VectorwiseOp::reverse() * and most of the time this is the only way it is used. * * \sa MatrixBase::reverse(), VectorwiseOp::reverse() */ template class Reverse : public internal::dense_xpr_base< Reverse >::type { public: typedef typename internal::dense_xpr_base::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(Reverse) typedef typename internal::remove_all::type NestedExpression; using Base::IsRowMajor; protected: enum { PacketSize = internal::packet_traits::size, IsColMajor = !IsRowMajor, ReverseRow = (Direction == Vertical) || (Direction == BothDirections), ReverseCol = (Direction == Horizontal) || (Direction == BothDirections), OffsetRow = ReverseRow && IsColMajor ? PacketSize : 1, OffsetCol = ReverseCol && IsRowMajor ? PacketSize : 1, ReversePacket = (Direction == BothDirections) || ((Direction == Vertical) && IsColMajor) || ((Direction == Horizontal) && IsRowMajor) }; typedef internal::reverse_packet_cond reverse_packet; public: EIGEN_DEVICE_FUNC explicit inline Reverse(const MatrixType& matrix) : m_matrix(matrix) { } EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Reverse) EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return m_matrix.rows(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return m_matrix.cols(); } EIGEN_DEVICE_FUNC inline Index innerStride() const { return -m_matrix.innerStride(); } EIGEN_DEVICE_FUNC const typename internal::remove_all::type& nestedExpression() const { return m_matrix; } protected: typename MatrixType::Nested m_matrix; }; /** \returns an expression of the reverse of *this. * * Example: \include MatrixBase_reverse.cpp * Output: \verbinclude MatrixBase_reverse.out * */ template EIGEN_DEVICE_FUNC inline typename DenseBase::ReverseReturnType DenseBase::reverse() { return ReverseReturnType(derived()); } //reverse const overload moved DenseBase.h due to a CUDA compiler bug /** This is the "in place" version of reverse: it reverses \c *this. * * In most cases it is probably better to simply use the reversed expression * of a matrix. However, when reversing the matrix data itself is really needed, * then this "in-place" version is probably the right choice because it provides * the following additional benefits: * - less error prone: doing the same operation with .reverse() requires special care: * \code m = m.reverse().eval(); \endcode * - this API enables reverse operations without the need for a temporary * - it allows future optimizations (cache friendliness, etc.) * * \sa VectorwiseOp::reverseInPlace(), reverse() */ template EIGEN_DEVICE_FUNC inline void DenseBase::reverseInPlace() { if(cols()>rows()) { Index half = cols()/2; leftCols(half).swap(rightCols(half).reverse()); if((cols()%2)==1) { Index half2 = rows()/2; col(half).head(half2).swap(col(half).tail(half2).reverse()); } } else { Index half = rows()/2; topRows(half).swap(bottomRows(half).reverse()); if((rows()%2)==1) { Index half2 = cols()/2; row(half).head(half2).swap(row(half).tail(half2).reverse()); } } } namespace internal { template struct vectorwise_reverse_inplace_impl; template<> struct vectorwise_reverse_inplace_impl { template static void run(ExpressionType &xpr) { const int HalfAtCompileTime = ExpressionType::RowsAtCompileTime==Dynamic?Dynamic:ExpressionType::RowsAtCompileTime/2; Index half = xpr.rows()/2; xpr.topRows(fix(half)) .swap(xpr.bottomRows(fix(half)).colwise().reverse()); } }; template<> struct vectorwise_reverse_inplace_impl { template static void run(ExpressionType &xpr) { const int HalfAtCompileTime = ExpressionType::ColsAtCompileTime==Dynamic?Dynamic:ExpressionType::ColsAtCompileTime/2; Index half = xpr.cols()/2; xpr.leftCols(fix(half)) .swap(xpr.rightCols(fix(half)).rowwise().reverse()); } }; } // end namespace internal /** This is the "in place" version of VectorwiseOp::reverse: it reverses each column or row of \c *this. * * In most cases it is probably better to simply use the reversed expression * of a matrix. However, when reversing the matrix data itself is really needed, * then this "in-place" version is probably the right choice because it provides * the following additional benefits: * - less error prone: doing the same operation with .reverse() requires special care: * \code m = m.reverse().eval(); \endcode * - this API enables reverse operations without the need for a temporary * * \sa DenseBase::reverseInPlace(), reverse() */ template EIGEN_DEVICE_FUNC void VectorwiseOp::reverseInPlace() { internal::vectorwise_reverse_inplace_impl::run(m_matrix); } } // end namespace Eigen #endif // EIGEN_REVERSE_H RcppEigen/inst/include/Eigen/src/Core/ArrayBase.h0000644000176200001440000002003114567757725021303 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2009 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_ARRAYBASE_H #define EIGEN_ARRAYBASE_H namespace Eigen { template class MatrixWrapper; /** \class ArrayBase * \ingroup Core_Module * * \brief Base class for all 1D and 2D array, and related expressions * * An array is similar to a dense vector or matrix. While matrices are mathematical * objects with well defined linear algebra operators, an array is just a collection * of scalar values arranged in a one or two dimensionnal fashion. As the main consequence, * all operations applied to an array are performed coefficient wise. Furthermore, * arrays support scalar math functions of the c++ standard library (e.g., std::sin(x)), and convenient * constructors allowing to easily write generic code working for both scalar values * and arrays. * * This class is the base that is inherited by all array expression types. * * \tparam Derived is the derived type, e.g., an array or an expression type. * * This class can be extended with the help of the plugin mechanism described on the page * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_ARRAYBASE_PLUGIN. * * \sa class MatrixBase, \ref TopicClassHierarchy */ template class ArrayBase : public DenseBase { public: #ifndef EIGEN_PARSED_BY_DOXYGEN /** The base class for a given storage type. */ typedef ArrayBase StorageBaseType; typedef ArrayBase Eigen_BaseClassForSpecializationOfGlobalMathFuncImpl; typedef typename internal::traits::StorageKind StorageKind; typedef typename internal::traits::Scalar Scalar; typedef typename internal::packet_traits::type PacketScalar; typedef typename NumTraits::Real RealScalar; typedef DenseBase Base; using Base::RowsAtCompileTime; using Base::ColsAtCompileTime; using Base::SizeAtCompileTime; using Base::MaxRowsAtCompileTime; using Base::MaxColsAtCompileTime; using Base::MaxSizeAtCompileTime; using Base::IsVectorAtCompileTime; using Base::Flags; using Base::derived; using Base::const_cast_derived; using Base::rows; using Base::cols; using Base::size; using Base::coeff; using Base::coeffRef; using Base::lazyAssign; using Base::operator-; using Base::operator=; using Base::operator+=; using Base::operator-=; using Base::operator*=; using Base::operator/=; typedef typename Base::CoeffReturnType CoeffReturnType; #endif // not EIGEN_PARSED_BY_DOXYGEN #ifndef EIGEN_PARSED_BY_DOXYGEN typedef typename Base::PlainObject PlainObject; /** \internal Represents a matrix with all coefficients equal to one another*/ typedef CwiseNullaryOp,PlainObject> ConstantReturnType; #endif // not EIGEN_PARSED_BY_DOXYGEN #define EIGEN_CURRENT_STORAGE_BASE_CLASS Eigen::ArrayBase #define EIGEN_DOC_UNARY_ADDONS(X,Y) # include "../plugins/MatrixCwiseUnaryOps.h" # include "../plugins/ArrayCwiseUnaryOps.h" # include "../plugins/CommonCwiseBinaryOps.h" # include "../plugins/MatrixCwiseBinaryOps.h" # include "../plugins/ArrayCwiseBinaryOps.h" # ifdef EIGEN_ARRAYBASE_PLUGIN # include EIGEN_ARRAYBASE_PLUGIN # endif #undef EIGEN_CURRENT_STORAGE_BASE_CLASS #undef EIGEN_DOC_UNARY_ADDONS /** Special case of the template operator=, in order to prevent the compiler * from generating a default operator= (issue hit with g++ 4.1) */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const ArrayBase& other) { internal::call_assignment(derived(), other.derived()); return derived(); } /** Set all the entries to \a value. * \sa DenseBase::setConstant(), DenseBase::fill() */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const Scalar &value) { Base::setConstant(value); return derived(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator+=(const Scalar& scalar); EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator-=(const Scalar& scalar); template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator+=(const ArrayBase& other); template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator-=(const ArrayBase& other); template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator*=(const ArrayBase& other); template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator/=(const ArrayBase& other); public: EIGEN_DEVICE_FUNC ArrayBase& array() { return *this; } EIGEN_DEVICE_FUNC const ArrayBase& array() const { return *this; } /** \returns an \link Eigen::MatrixBase Matrix \endlink expression of this array * \sa MatrixBase::array() */ EIGEN_DEVICE_FUNC MatrixWrapper matrix() { return MatrixWrapper(derived()); } EIGEN_DEVICE_FUNC const MatrixWrapper matrix() const { return MatrixWrapper(derived()); } // template // inline void evalTo(Dest& dst) const { dst = matrix(); } protected: EIGEN_DEFAULT_COPY_CONSTRUCTOR(ArrayBase) EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(ArrayBase) private: explicit ArrayBase(Index); ArrayBase(Index,Index); template explicit ArrayBase(const ArrayBase&); protected: // mixing arrays and matrices is not legal template Derived& operator+=(const MatrixBase& ) {EIGEN_STATIC_ASSERT(std::ptrdiff_t(sizeof(typename OtherDerived::Scalar))==-1,YOU_CANNOT_MIX_ARRAYS_AND_MATRICES); return *this;} // mixing arrays and matrices is not legal template Derived& operator-=(const MatrixBase& ) {EIGEN_STATIC_ASSERT(std::ptrdiff_t(sizeof(typename OtherDerived::Scalar))==-1,YOU_CANNOT_MIX_ARRAYS_AND_MATRICES); return *this;} }; /** replaces \c *this by \c *this - \a other. * * \returns a reference to \c *this */ template template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived & ArrayBase::operator-=(const ArrayBase &other) { call_assignment(derived(), other.derived(), internal::sub_assign_op()); return derived(); } /** replaces \c *this by \c *this + \a other. * * \returns a reference to \c *this */ template template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived & ArrayBase::operator+=(const ArrayBase& other) { call_assignment(derived(), other.derived(), internal::add_assign_op()); return derived(); } /** replaces \c *this by \c *this * \a other coefficient wise. * * \returns a reference to \c *this */ template template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived & ArrayBase::operator*=(const ArrayBase& other) { call_assignment(derived(), other.derived(), internal::mul_assign_op()); return derived(); } /** replaces \c *this by \c *this / \a other coefficient wise. * * \returns a reference to \c *this */ template template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived & ArrayBase::operator/=(const ArrayBase& other) { call_assignment(derived(), other.derived(), internal::div_assign_op()); return derived(); } } // end namespace Eigen #endif // EIGEN_ARRAYBASE_H RcppEigen/inst/include/Eigen/src/Core/Transpositions.h0000644000176200001440000003237714567757725022511 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2010-2011 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_TRANSPOSITIONS_H #define EIGEN_TRANSPOSITIONS_H namespace Eigen { template class TranspositionsBase { typedef internal::traits Traits; public: typedef typename Traits::IndicesType IndicesType; typedef typename IndicesType::Scalar StorageIndex; typedef Eigen::Index Index; ///< \deprecated since Eigen 3.3 EIGEN_DEVICE_FUNC Derived& derived() { return *static_cast(this); } EIGEN_DEVICE_FUNC const Derived& derived() const { return *static_cast(this); } /** Copies the \a other transpositions into \c *this */ template Derived& operator=(const TranspositionsBase& other) { indices() = other.indices(); return derived(); } /** \returns the number of transpositions */ EIGEN_DEVICE_FUNC Index size() const { return indices().size(); } /** \returns the number of rows of the equivalent permutation matrix */ EIGEN_DEVICE_FUNC Index rows() const { return indices().size(); } /** \returns the number of columns of the equivalent permutation matrix */ EIGEN_DEVICE_FUNC Index cols() const { return indices().size(); } /** Direct access to the underlying index vector */ EIGEN_DEVICE_FUNC inline const StorageIndex& coeff(Index i) const { return indices().coeff(i); } /** Direct access to the underlying index vector */ inline StorageIndex& coeffRef(Index i) { return indices().coeffRef(i); } /** Direct access to the underlying index vector */ inline const StorageIndex& operator()(Index i) const { return indices()(i); } /** Direct access to the underlying index vector */ inline StorageIndex& operator()(Index i) { return indices()(i); } /** Direct access to the underlying index vector */ inline const StorageIndex& operator[](Index i) const { return indices()(i); } /** Direct access to the underlying index vector */ inline StorageIndex& operator[](Index i) { return indices()(i); } /** const version of indices(). */ EIGEN_DEVICE_FUNC const IndicesType& indices() const { return derived().indices(); } /** \returns a reference to the stored array representing the transpositions. */ EIGEN_DEVICE_FUNC IndicesType& indices() { return derived().indices(); } /** Resizes to given size. */ inline void resize(Index newSize) { indices().resize(newSize); } /** Sets \c *this to represents an identity transformation */ void setIdentity() { for(StorageIndex i = 0; i < indices().size(); ++i) coeffRef(i) = i; } // FIXME: do we want such methods ? // might be useful when the target matrix expression is complex, e.g.: // object.matrix().block(..,..,..,..) = trans * object.matrix().block(..,..,..,..); /* template void applyForwardToRows(MatrixType& mat) const { for(Index k=0 ; k void applyBackwardToRows(MatrixType& mat) const { for(Index k=size()-1 ; k>=0 ; --k) if(m_indices(k)!=k) mat.row(k).swap(mat.row(m_indices(k))); } */ /** \returns the inverse transformation */ inline Transpose inverse() const { return Transpose(derived()); } /** \returns the tranpose transformation */ inline Transpose transpose() const { return Transpose(derived()); } protected: }; namespace internal { template struct traits > : traits > { typedef Matrix<_StorageIndex, SizeAtCompileTime, 1, 0, MaxSizeAtCompileTime, 1> IndicesType; typedef TranspositionsStorage StorageKind; }; } /** \class Transpositions * \ingroup Core_Module * * \brief Represents a sequence of transpositions (row/column interchange) * * \tparam SizeAtCompileTime the number of transpositions, or Dynamic * \tparam MaxSizeAtCompileTime the maximum number of transpositions, or Dynamic. This optional parameter defaults to SizeAtCompileTime. Most of the time, you should not have to specify it. * * This class represents a permutation transformation as a sequence of \em n transpositions * \f$[T_{n-1} \ldots T_{i} \ldots T_{0}]\f$. It is internally stored as a vector of integers \c indices. * Each transposition \f$ T_{i} \f$ applied on the left of a matrix (\f$ T_{i} M\f$) interchanges * the rows \c i and \c indices[i] of the matrix \c M. * A transposition applied on the right (e.g., \f$ M T_{i}\f$) yields a column interchange. * * Compared to the class PermutationMatrix, such a sequence of transpositions is what is * computed during a decomposition with pivoting, and it is faster when applying the permutation in-place. * * To apply a sequence of transpositions to a matrix, simply use the operator * as in the following example: * \code * Transpositions tr; * MatrixXf mat; * mat = tr * mat; * \endcode * In this example, we detect that the matrix appears on both side, and so the transpositions * are applied in-place without any temporary or extra copy. * * \sa class PermutationMatrix */ template class Transpositions : public TranspositionsBase > { typedef internal::traits Traits; public: typedef TranspositionsBase Base; typedef typename Traits::IndicesType IndicesType; typedef typename IndicesType::Scalar StorageIndex; inline Transpositions() {} /** Copy constructor. */ template inline Transpositions(const TranspositionsBase& other) : m_indices(other.indices()) {} /** Generic constructor from expression of the transposition indices. */ template explicit inline Transpositions(const MatrixBase& indices) : m_indices(indices) {} /** Copies the \a other transpositions into \c *this */ template Transpositions& operator=(const TranspositionsBase& other) { return Base::operator=(other); } /** Constructs an uninitialized permutation matrix of given size. */ inline Transpositions(Index size) : m_indices(size) {} /** const version of indices(). */ EIGEN_DEVICE_FUNC const IndicesType& indices() const { return m_indices; } /** \returns a reference to the stored array representing the transpositions. */ EIGEN_DEVICE_FUNC IndicesType& indices() { return m_indices; } protected: IndicesType m_indices; }; namespace internal { template struct traits,_PacketAccess> > : traits > { typedef Map, _PacketAccess> IndicesType; typedef _StorageIndex StorageIndex; typedef TranspositionsStorage StorageKind; }; } template class Map,PacketAccess> : public TranspositionsBase,PacketAccess> > { typedef internal::traits Traits; public: typedef TranspositionsBase Base; typedef typename Traits::IndicesType IndicesType; typedef typename IndicesType::Scalar StorageIndex; explicit inline Map(const StorageIndex* indicesPtr) : m_indices(indicesPtr) {} inline Map(const StorageIndex* indicesPtr, Index size) : m_indices(indicesPtr,size) {} /** Copies the \a other transpositions into \c *this */ template Map& operator=(const TranspositionsBase& other) { return Base::operator=(other); } #ifndef EIGEN_PARSED_BY_DOXYGEN /** This is a special case of the templated operator=. Its purpose is to * prevent a default operator= from hiding the templated operator=. */ Map& operator=(const Map& other) { m_indices = other.m_indices; return *this; } #endif /** const version of indices(). */ EIGEN_DEVICE_FUNC const IndicesType& indices() const { return m_indices; } /** \returns a reference to the stored array representing the transpositions. */ EIGEN_DEVICE_FUNC IndicesType& indices() { return m_indices; } protected: IndicesType m_indices; }; namespace internal { template struct traits > : traits > { typedef TranspositionsStorage StorageKind; }; } template class TranspositionsWrapper : public TranspositionsBase > { typedef internal::traits Traits; public: typedef TranspositionsBase Base; typedef typename Traits::IndicesType IndicesType; typedef typename IndicesType::Scalar StorageIndex; explicit inline TranspositionsWrapper(IndicesType& indices) : m_indices(indices) {} /** Copies the \a other transpositions into \c *this */ template TranspositionsWrapper& operator=(const TranspositionsBase& other) { return Base::operator=(other); } /** const version of indices(). */ EIGEN_DEVICE_FUNC const IndicesType& indices() const { return m_indices; } /** \returns a reference to the stored array representing the transpositions. */ EIGEN_DEVICE_FUNC IndicesType& indices() { return m_indices; } protected: typename IndicesType::Nested m_indices; }; /** \returns the \a matrix with the \a transpositions applied to the columns. */ template EIGEN_DEVICE_FUNC const Product operator*(const MatrixBase &matrix, const TranspositionsBase& transpositions) { return Product (matrix.derived(), transpositions.derived()); } /** \returns the \a matrix with the \a transpositions applied to the rows. */ template EIGEN_DEVICE_FUNC const Product operator*(const TranspositionsBase &transpositions, const MatrixBase& matrix) { return Product (transpositions.derived(), matrix.derived()); } // Template partial specialization for transposed/inverse transpositions namespace internal { template struct traits > > : traits {}; } // end namespace internal template class Transpose > { typedef TranspositionsDerived TranspositionType; typedef typename TranspositionType::IndicesType IndicesType; public: explicit Transpose(const TranspositionType& t) : m_transpositions(t) {} EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index size() const EIGEN_NOEXCEPT { return m_transpositions.size(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_transpositions.size(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_transpositions.size(); } /** \returns the \a matrix with the inverse transpositions applied to the columns. */ template friend const Product operator*(const MatrixBase& matrix, const Transpose& trt) { return Product(matrix.derived(), trt); } /** \returns the \a matrix with the inverse transpositions applied to the rows. */ template const Product operator*(const MatrixBase& matrix) const { return Product(*this, matrix.derived()); } EIGEN_DEVICE_FUNC const TranspositionType& nestedExpression() const { return m_transpositions; } protected: const TranspositionType& m_transpositions; }; } // end namespace Eigen #endif // EIGEN_TRANSPOSITIONS_H RcppEigen/inst/include/Eigen/src/Core/DiagonalMatrix.h0000644000176200001440000003451614567757725022352 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2009 Gael Guennebaud // Copyright (C) 2007-2009 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_DIAGONALMATRIX_H #define EIGEN_DIAGONALMATRIX_H namespace Eigen { #ifndef EIGEN_PARSED_BY_DOXYGEN template class DiagonalBase : public EigenBase { public: typedef typename internal::traits::DiagonalVectorType DiagonalVectorType; typedef typename DiagonalVectorType::Scalar Scalar; typedef typename DiagonalVectorType::RealScalar RealScalar; typedef typename internal::traits::StorageKind StorageKind; typedef typename internal::traits::StorageIndex StorageIndex; enum { RowsAtCompileTime = DiagonalVectorType::SizeAtCompileTime, ColsAtCompileTime = DiagonalVectorType::SizeAtCompileTime, MaxRowsAtCompileTime = DiagonalVectorType::MaxSizeAtCompileTime, MaxColsAtCompileTime = DiagonalVectorType::MaxSizeAtCompileTime, IsVectorAtCompileTime = 0, Flags = NoPreferredStorageOrderBit }; typedef Matrix DenseMatrixType; typedef DenseMatrixType DenseType; typedef DiagonalMatrix PlainObject; EIGEN_DEVICE_FUNC inline const Derived& derived() const { return *static_cast(this); } EIGEN_DEVICE_FUNC inline Derived& derived() { return *static_cast(this); } EIGEN_DEVICE_FUNC DenseMatrixType toDenseMatrix() const { return derived(); } EIGEN_DEVICE_FUNC inline const DiagonalVectorType& diagonal() const { return derived().diagonal(); } EIGEN_DEVICE_FUNC inline DiagonalVectorType& diagonal() { return derived().diagonal(); } EIGEN_DEVICE_FUNC inline Index rows() const { return diagonal().size(); } EIGEN_DEVICE_FUNC inline Index cols() const { return diagonal().size(); } template EIGEN_DEVICE_FUNC const Product operator*(const MatrixBase &matrix) const { return Product(derived(),matrix.derived()); } typedef DiagonalWrapper, const DiagonalVectorType> > InverseReturnType; EIGEN_DEVICE_FUNC inline const InverseReturnType inverse() const { return InverseReturnType(diagonal().cwiseInverse()); } EIGEN_DEVICE_FUNC inline const DiagonalWrapper operator*(const Scalar& scalar) const { return DiagonalWrapper(diagonal() * scalar); } EIGEN_DEVICE_FUNC friend inline const DiagonalWrapper operator*(const Scalar& scalar, const DiagonalBase& other) { return DiagonalWrapper(scalar * other.diagonal()); } template EIGEN_DEVICE_FUNC #ifdef EIGEN_PARSED_BY_DOXYGEN inline unspecified_expression_type #else inline const DiagonalWrapper #endif operator+(const DiagonalBase& other) const { return (diagonal() + other.diagonal()).asDiagonal(); } template EIGEN_DEVICE_FUNC #ifdef EIGEN_PARSED_BY_DOXYGEN inline unspecified_expression_type #else inline const DiagonalWrapper #endif operator-(const DiagonalBase& other) const { return (diagonal() - other.diagonal()).asDiagonal(); } }; #endif /** \class DiagonalMatrix * \ingroup Core_Module * * \brief Represents a diagonal matrix with its storage * * \param _Scalar the type of coefficients * \param SizeAtCompileTime the dimension of the matrix, or Dynamic * \param MaxSizeAtCompileTime the dimension of the matrix, or Dynamic. This parameter is optional and defaults * to SizeAtCompileTime. Most of the time, you do not need to specify it. * * \sa class DiagonalWrapper */ namespace internal { template struct traits > : traits > { typedef Matrix<_Scalar,SizeAtCompileTime,1,0,MaxSizeAtCompileTime,1> DiagonalVectorType; typedef DiagonalShape StorageKind; enum { Flags = LvalueBit | NoPreferredStorageOrderBit }; }; } template class DiagonalMatrix : public DiagonalBase > { public: #ifndef EIGEN_PARSED_BY_DOXYGEN typedef typename internal::traits::DiagonalVectorType DiagonalVectorType; typedef const DiagonalMatrix& Nested; typedef _Scalar Scalar; typedef typename internal::traits::StorageKind StorageKind; typedef typename internal::traits::StorageIndex StorageIndex; #endif protected: DiagonalVectorType m_diagonal; public: /** const version of diagonal(). */ EIGEN_DEVICE_FUNC inline const DiagonalVectorType& diagonal() const { return m_diagonal; } /** \returns a reference to the stored vector of diagonal coefficients. */ EIGEN_DEVICE_FUNC inline DiagonalVectorType& diagonal() { return m_diagonal; } /** Default constructor without initialization */ EIGEN_DEVICE_FUNC inline DiagonalMatrix() {} /** Constructs a diagonal matrix with given dimension */ EIGEN_DEVICE_FUNC explicit inline DiagonalMatrix(Index dim) : m_diagonal(dim) {} /** 2D constructor. */ EIGEN_DEVICE_FUNC inline DiagonalMatrix(const Scalar& x, const Scalar& y) : m_diagonal(x,y) {} /** 3D constructor. */ EIGEN_DEVICE_FUNC inline DiagonalMatrix(const Scalar& x, const Scalar& y, const Scalar& z) : m_diagonal(x,y,z) {} #if EIGEN_HAS_CXX11 /** \brief Construct a diagonal matrix with fixed size from an arbitrary number of coefficients. \cpp11 * * There exists C++98 anologue constructors for fixed-size diagonal matrices having 2 or 3 coefficients. * * \warning To construct a diagonal matrix of fixed size, the number of values passed to this * constructor must match the fixed dimension of \c *this. * * \sa DiagonalMatrix(const Scalar&, const Scalar&) * \sa DiagonalMatrix(const Scalar&, const Scalar&, const Scalar&) */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DiagonalMatrix(const Scalar& a0, const Scalar& a1, const Scalar& a2, const ArgTypes&... args) : m_diagonal(a0, a1, a2, args...) {} /** \brief Constructs a DiagonalMatrix and initializes it by elements given by an initializer list of initializer * lists \cpp11 */ EIGEN_DEVICE_FUNC explicit EIGEN_STRONG_INLINE DiagonalMatrix(const std::initializer_list>& list) : m_diagonal(list) {} #endif // EIGEN_HAS_CXX11 /** Copy constructor. */ template EIGEN_DEVICE_FUNC inline DiagonalMatrix(const DiagonalBase& other) : m_diagonal(other.diagonal()) {} #ifndef EIGEN_PARSED_BY_DOXYGEN /** copy constructor. prevent a default copy constructor from hiding the other templated constructor */ inline DiagonalMatrix(const DiagonalMatrix& other) : m_diagonal(other.diagonal()) {} #endif /** generic constructor from expression of the diagonal coefficients */ template EIGEN_DEVICE_FUNC explicit inline DiagonalMatrix(const MatrixBase& other) : m_diagonal(other) {} /** Copy operator. */ template EIGEN_DEVICE_FUNC DiagonalMatrix& operator=(const DiagonalBase& other) { m_diagonal = other.diagonal(); return *this; } #ifndef EIGEN_PARSED_BY_DOXYGEN /** This is a special case of the templated operator=. Its purpose is to * prevent a default operator= from hiding the templated operator=. */ EIGEN_DEVICE_FUNC DiagonalMatrix& operator=(const DiagonalMatrix& other) { m_diagonal = other.diagonal(); return *this; } #endif /** Resizes to given size. */ EIGEN_DEVICE_FUNC inline void resize(Index size) { m_diagonal.resize(size); } /** Sets all coefficients to zero. */ EIGEN_DEVICE_FUNC inline void setZero() { m_diagonal.setZero(); } /** Resizes and sets all coefficients to zero. */ EIGEN_DEVICE_FUNC inline void setZero(Index size) { m_diagonal.setZero(size); } /** Sets this matrix to be the identity matrix of the current size. */ EIGEN_DEVICE_FUNC inline void setIdentity() { m_diagonal.setOnes(); } /** Sets this matrix to be the identity matrix of the given size. */ EIGEN_DEVICE_FUNC inline void setIdentity(Index size) { m_diagonal.setOnes(size); } }; /** \class DiagonalWrapper * \ingroup Core_Module * * \brief Expression of a diagonal matrix * * \param _DiagonalVectorType the type of the vector of diagonal coefficients * * This class is an expression of a diagonal matrix, but not storing its own vector of diagonal coefficients, * instead wrapping an existing vector expression. It is the return type of MatrixBase::asDiagonal() * and most of the time this is the only way that it is used. * * \sa class DiagonalMatrix, class DiagonalBase, MatrixBase::asDiagonal() */ namespace internal { template struct traits > { typedef _DiagonalVectorType DiagonalVectorType; typedef typename DiagonalVectorType::Scalar Scalar; typedef typename DiagonalVectorType::StorageIndex StorageIndex; typedef DiagonalShape StorageKind; typedef typename traits::XprKind XprKind; enum { RowsAtCompileTime = DiagonalVectorType::SizeAtCompileTime, ColsAtCompileTime = DiagonalVectorType::SizeAtCompileTime, MaxRowsAtCompileTime = DiagonalVectorType::MaxSizeAtCompileTime, MaxColsAtCompileTime = DiagonalVectorType::MaxSizeAtCompileTime, Flags = (traits::Flags & LvalueBit) | NoPreferredStorageOrderBit }; }; } template class DiagonalWrapper : public DiagonalBase >, internal::no_assignment_operator { public: #ifndef EIGEN_PARSED_BY_DOXYGEN typedef _DiagonalVectorType DiagonalVectorType; typedef DiagonalWrapper Nested; #endif /** Constructor from expression of diagonal coefficients to wrap. */ EIGEN_DEVICE_FUNC explicit inline DiagonalWrapper(DiagonalVectorType& a_diagonal) : m_diagonal(a_diagonal) {} /** \returns a const reference to the wrapped expression of diagonal coefficients. */ EIGEN_DEVICE_FUNC const DiagonalVectorType& diagonal() const { return m_diagonal; } protected: typename DiagonalVectorType::Nested m_diagonal; }; /** \returns a pseudo-expression of a diagonal matrix with *this as vector of diagonal coefficients * * \only_for_vectors * * Example: \include MatrixBase_asDiagonal.cpp * Output: \verbinclude MatrixBase_asDiagonal.out * * \sa class DiagonalWrapper, class DiagonalMatrix, diagonal(), isDiagonal() **/ template EIGEN_DEVICE_FUNC inline const DiagonalWrapper MatrixBase::asDiagonal() const { return DiagonalWrapper(derived()); } /** \returns true if *this is approximately equal to a diagonal matrix, * within the precision given by \a prec. * * Example: \include MatrixBase_isDiagonal.cpp * Output: \verbinclude MatrixBase_isDiagonal.out * * \sa asDiagonal() */ template bool MatrixBase::isDiagonal(const RealScalar& prec) const { if(cols() != rows()) return false; RealScalar maxAbsOnDiagonal = static_cast(-1); for(Index j = 0; j < cols(); ++j) { RealScalar absOnDiagonal = numext::abs(coeff(j,j)); if(absOnDiagonal > maxAbsOnDiagonal) maxAbsOnDiagonal = absOnDiagonal; } for(Index j = 0; j < cols(); ++j) for(Index i = 0; i < j; ++i) { if(!internal::isMuchSmallerThan(coeff(i, j), maxAbsOnDiagonal, prec)) return false; if(!internal::isMuchSmallerThan(coeff(j, i), maxAbsOnDiagonal, prec)) return false; } return true; } namespace internal { template<> struct storage_kind_to_shape { typedef DiagonalShape Shape; }; struct Diagonal2Dense {}; template<> struct AssignmentKind { typedef Diagonal2Dense Kind; }; // Diagonal matrix to Dense assignment template< typename DstXprType, typename SrcXprType, typename Functor> struct Assignment { static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op &/*func*/) { Index dstRows = src.rows(); Index dstCols = src.cols(); if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) dst.resize(dstRows, dstCols); dst.setZero(); dst.diagonal() = src.diagonal(); } static void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op &/*func*/) { dst.diagonal() += src.diagonal(); } static void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op &/*func*/) { dst.diagonal() -= src.diagonal(); } }; } // namespace internal } // end namespace Eigen #endif // EIGEN_DIAGONALMATRIX_H RcppEigen/inst/include/Eigen/src/Core/ArrayWrapper.h0000644000176200001440000001555214567757725022065 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2009-2010 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_ARRAYWRAPPER_H #define EIGEN_ARRAYWRAPPER_H namespace Eigen { /** \class ArrayWrapper * \ingroup Core_Module * * \brief Expression of a mathematical vector or matrix as an array object * * This class is the return type of MatrixBase::array(), and most of the time * this is the only way it is use. * * \sa MatrixBase::array(), class MatrixWrapper */ namespace internal { template struct traits > : public traits::type > { typedef ArrayXpr XprKind; // Let's remove NestByRefBit enum { Flags0 = traits::type >::Flags, LvalueBitFlag = is_lvalue::value ? LvalueBit : 0, Flags = (Flags0 & ~(NestByRefBit | LvalueBit)) | LvalueBitFlag }; }; } template class ArrayWrapper : public ArrayBase > { public: typedef ArrayBase Base; EIGEN_DENSE_PUBLIC_INTERFACE(ArrayWrapper) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(ArrayWrapper) typedef typename internal::remove_all::type NestedExpression; typedef typename internal::conditional< internal::is_lvalue::value, Scalar, const Scalar >::type ScalarWithConstIfNotLvalue; typedef typename internal::ref_selector::non_const_type NestedExpressionType; using Base::coeffRef; EIGEN_DEVICE_FUNC explicit EIGEN_STRONG_INLINE ArrayWrapper(ExpressionType& matrix) : m_expression(matrix) {} EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return m_expression.rows(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return m_expression.cols(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT { return m_expression.outerStride(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT { return m_expression.innerStride(); } EIGEN_DEVICE_FUNC inline ScalarWithConstIfNotLvalue* data() { return m_expression.data(); } EIGEN_DEVICE_FUNC inline const Scalar* data() const { return m_expression.data(); } EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index rowId, Index colId) const { return m_expression.coeffRef(rowId, colId); } EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index index) const { return m_expression.coeffRef(index); } template EIGEN_DEVICE_FUNC inline void evalTo(Dest& dst) const { dst = m_expression; } EIGEN_DEVICE_FUNC const typename internal::remove_all::type& nestedExpression() const { return m_expression; } /** Forwards the resizing request to the nested expression * \sa DenseBase::resize(Index) */ EIGEN_DEVICE_FUNC void resize(Index newSize) { m_expression.resize(newSize); } /** Forwards the resizing request to the nested expression * \sa DenseBase::resize(Index,Index)*/ EIGEN_DEVICE_FUNC void resize(Index rows, Index cols) { m_expression.resize(rows,cols); } protected: NestedExpressionType m_expression; }; /** \class MatrixWrapper * \ingroup Core_Module * * \brief Expression of an array as a mathematical vector or matrix * * This class is the return type of ArrayBase::matrix(), and most of the time * this is the only way it is use. * * \sa MatrixBase::matrix(), class ArrayWrapper */ namespace internal { template struct traits > : public traits::type > { typedef MatrixXpr XprKind; // Let's remove NestByRefBit enum { Flags0 = traits::type >::Flags, LvalueBitFlag = is_lvalue::value ? LvalueBit : 0, Flags = (Flags0 & ~(NestByRefBit | LvalueBit)) | LvalueBitFlag }; }; } template class MatrixWrapper : public MatrixBase > { public: typedef MatrixBase > Base; EIGEN_DENSE_PUBLIC_INTERFACE(MatrixWrapper) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(MatrixWrapper) typedef typename internal::remove_all::type NestedExpression; typedef typename internal::conditional< internal::is_lvalue::value, Scalar, const Scalar >::type ScalarWithConstIfNotLvalue; typedef typename internal::ref_selector::non_const_type NestedExpressionType; using Base::coeffRef; EIGEN_DEVICE_FUNC explicit inline MatrixWrapper(ExpressionType& matrix) : m_expression(matrix) {} EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return m_expression.rows(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return m_expression.cols(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT { return m_expression.outerStride(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT { return m_expression.innerStride(); } EIGEN_DEVICE_FUNC inline ScalarWithConstIfNotLvalue* data() { return m_expression.data(); } EIGEN_DEVICE_FUNC inline const Scalar* data() const { return m_expression.data(); } EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index rowId, Index colId) const { return m_expression.derived().coeffRef(rowId, colId); } EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index index) const { return m_expression.coeffRef(index); } EIGEN_DEVICE_FUNC const typename internal::remove_all::type& nestedExpression() const { return m_expression; } /** Forwards the resizing request to the nested expression * \sa DenseBase::resize(Index) */ EIGEN_DEVICE_FUNC void resize(Index newSize) { m_expression.resize(newSize); } /** Forwards the resizing request to the nested expression * \sa DenseBase::resize(Index,Index)*/ EIGEN_DEVICE_FUNC void resize(Index rows, Index cols) { m_expression.resize(rows,cols); } protected: NestedExpressionType m_expression; }; } // end namespace Eigen #endif // EIGEN_ARRAYWRAPPER_H RcppEigen/inst/include/Eigen/src/Core/arch/0000755000176200001440000000000014567757725020202 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/arch/SYCL/0000755000176200001440000000000014567757725020754 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/arch/SYCL/TypeCasting.h0000644000176200001440000000510214567757725023355 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Mehdi Goli Codeplay Software Ltd. // Ralph Potter Codeplay Software Ltd. // Luke Iwanski Codeplay Software Ltd. // Contact: // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. /***************************************************************** * TypeCasting.h * * \brief: * TypeCasting * *****************************************************************/ #ifndef EIGEN_TYPE_CASTING_SYCL_H #define EIGEN_TYPE_CASTING_SYCL_H namespace Eigen { namespace internal { #ifdef SYCL_DEVICE_ONLY template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_int4 pcast(const cl::sycl::cl_float4& a) { return a .template convert(); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_float4 pcast(const cl::sycl::cl_int4& a) { return a.template convert(); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_float4 pcast( const cl::sycl::cl_double2& a, const cl::sycl::cl_double2& b) { auto a1 = a.template convert(); auto b1 = b.template convert(); return cl::sycl::float4(a1.x(), a1.y(), b1.x(), b1.y()); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_double2 pcast(const cl::sycl::cl_float4& a) { // Simply discard the second half of the input return cl::sycl::cl_double2(a.x(), a.y()); } #endif } // end namespace internal } // end namespace Eigen #endif // EIGEN_TYPE_CASTING_SYCL_H RcppEigen/inst/include/Eigen/src/Core/arch/SYCL/SyclMemoryModel.h0000644000176200001440000005254014567757725024217 0ustar liggesusers/*************************************************************************** * Copyright (C) 2017 Codeplay Software Limited * This Source Code Form is subject to the terms of the Mozilla * Public License v. 2.0. If a copy of the MPL was not distributed * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. * * * SyclMemoryModel.h * * Description: * Interface for SYCL buffers to behave as a non-dereferenceable pointer * Interface for Placeholder accessor to behave as a pointer on both host * and device * * Authors: * * Ruyman Reyes Codeplay Software Ltd. * Mehdi Goli Codeplay Software Ltd. * Vanya Yaneva Codeplay Software Ltd. * **************************************************************************/ #if defined(EIGEN_USE_SYCL) && \ !defined(EIGEN_CXX11_TENSOR_TENSOR_SYCL_STORAGE_MEMORY_H) #define EIGEN_CXX11_TENSOR_TENSOR_SYCL_STORAGE_MEMORY_H #include #ifdef EIGEN_EXCEPTIONS #include #endif #include #include #include #include namespace Eigen { namespace TensorSycl { namespace internal { using sycl_acc_target = cl::sycl::access::target; using sycl_acc_mode = cl::sycl::access::mode; /** * Default values for template arguments */ using buffer_data_type_t = uint8_t; const sycl_acc_target default_acc_target = sycl_acc_target::global_buffer; const sycl_acc_mode default_acc_mode = sycl_acc_mode::read_write; /** * PointerMapper * Associates fake pointers with buffers. * */ class PointerMapper { public: using base_ptr_t = std::intptr_t; /* Structure of a virtual pointer * * |================================================| * | POINTER ADDRESS | * |================================================| */ struct virtual_pointer_t { /* Type for the pointers */ base_ptr_t m_contents; /** Conversions from virtual_pointer_t to * void * should just reinterpret_cast the integer number */ operator void *() const { return reinterpret_cast(m_contents); } /** * Convert back to the integer number. */ operator base_ptr_t() const { return m_contents; } /** * Add a certain value to the pointer to create a * new pointer to that offset */ virtual_pointer_t operator+(size_t off) { return m_contents + off; } /* Numerical order for sorting pointers in containers. */ bool operator<(virtual_pointer_t rhs) const { return (static_cast(m_contents) < static_cast(rhs.m_contents)); } bool operator>(virtual_pointer_t rhs) const { return (static_cast(m_contents) > static_cast(rhs.m_contents)); } /** * Numerical order for sorting pointers in containers */ bool operator==(virtual_pointer_t rhs) const { return (static_cast(m_contents) == static_cast(rhs.m_contents)); } /** * Simple forward to the equality overload. */ bool operator!=(virtual_pointer_t rhs) const { return !(this->operator==(rhs)); } /** * Converts a void * into a virtual pointer structure. * Note that this will only work if the void * was * already a virtual_pointer_t, but we have no way of * checking */ virtual_pointer_t(const void *ptr) : m_contents(reinterpret_cast(ptr)){}; /** * Creates a virtual_pointer_t from the given integer * number */ virtual_pointer_t(base_ptr_t u) : m_contents(u){}; }; /* Definition of a null pointer */ const virtual_pointer_t null_virtual_ptr = nullptr; /** * Whether if a pointer is null or not. * A pointer is nullptr if the value is of null_virtual_ptr */ static inline bool is_nullptr(virtual_pointer_t ptr) { return (static_cast(ptr) == nullptr); } /* basic type for all buffers */ using buffer_t = cl::sycl::buffer_mem; /** * Node that stores information about a device allocation. * Nodes are sorted by size to organise a free list of nodes * that can be recovered. */ struct pMapNode_t { buffer_t m_buffer; size_t m_size; bool m_free; pMapNode_t(buffer_t b, size_t size, bool f) : m_buffer{b}, m_size{size}, m_free{f} { m_buffer.set_final_data(nullptr); } bool operator<=(const pMapNode_t &rhs) { return (m_size <= rhs.m_size); } }; /** Storage of the pointer / buffer tree */ using pointerMap_t = std::map; /** * Obtain the insertion point in the pointer map for * a pointer of the given size. * \param requiredSize Size attemted to reclaim */ typename pointerMap_t::iterator get_insertion_point(size_t requiredSize) { typename pointerMap_t::iterator retVal; bool reuse = false; if (!m_freeList.empty()) { // try to re-use an existing block for (auto freeElem : m_freeList) { if (freeElem->second.m_size >= requiredSize) { retVal = freeElem; reuse = true; // Element is not going to be free anymore m_freeList.erase(freeElem); break; } } } if (!reuse) { retVal = std::prev(m_pointerMap.end()); } return retVal; } /** * Returns an iterator to the node that stores the information * of the given virtual pointer from the given pointer map structure. * If pointer is not found, throws std::out_of_range. * If the pointer map structure is empty, throws std::out_of_range * * \param pMap the pointerMap_t structure storing all the pointers * \param virtual_pointer_ptr The virtual pointer to obtain the node of * \throws std::out:of_range if the pointer is not found or pMap is empty */ typename pointerMap_t::iterator get_node(const virtual_pointer_t ptr) { if (this->count() == 0) { m_pointerMap.clear(); EIGEN_THROW_X(std::out_of_range("There are no pointers allocated\n")); } if (is_nullptr(ptr)) { m_pointerMap.clear(); EIGEN_THROW_X(std::out_of_range("Cannot access null pointer\n")); } // The previous element to the lower bound is the node that // holds this memory address auto node = m_pointerMap.lower_bound(ptr); // If the value of the pointer is not the one of the node // then we return the previous one if (node == std::end(m_pointerMap)) { --node; } else if (node->first != ptr) { if (node == std::begin(m_pointerMap)) { m_pointerMap.clear(); EIGEN_THROW_X( std::out_of_range("The pointer is not registered in the map\n")); } --node; } return node; } /* get_buffer. * Returns a buffer from the map using the pointer address */ template cl::sycl::buffer get_buffer( const virtual_pointer_t ptr) { using sycl_buffer_t = cl::sycl::buffer; // get_node() returns a `buffer_mem`, so we need to cast it to a `buffer<>`. // We can do this without the `buffer_mem` being a pointer, as we // only declare member variables in the base class (`buffer_mem`) and not in // the child class (`buffer<>). auto node = get_node(ptr); eigen_assert(node->first == ptr || node->first < ptr); eigen_assert(ptr < static_cast(node->second.m_size + node->first)); return *(static_cast(&node->second.m_buffer)); } /** * @brief Returns an accessor to the buffer of the given virtual pointer * @param accessMode * @param accessTarget * @param ptr The virtual pointer */ template cl::sycl::accessor get_access(const virtual_pointer_t ptr) { auto buf = get_buffer(ptr); return buf.template get_access(); } /** * @brief Returns an accessor to the buffer of the given virtual pointer * in the given command group scope * @param accessMode * @param accessTarget * @param ptr The virtual pointer * @param cgh Reference to the command group scope */ template cl::sycl::accessor get_access(const virtual_pointer_t ptr, cl::sycl::handler &cgh) { auto buf = get_buffer(ptr); return buf.template get_access(cgh); } /* * Returns the offset from the base address of this pointer. */ inline std::ptrdiff_t get_offset(const virtual_pointer_t ptr) { // The previous element to the lower bound is the node that // holds this memory address auto node = get_node(ptr); auto start = node->first; eigen_assert(start == ptr || start < ptr); eigen_assert(ptr < start + node->second.m_size); return (ptr - start); } /* * Returns the number of elements by which the given pointer is offset from * the base address. */ template inline size_t get_element_offset(const virtual_pointer_t ptr) { return get_offset(ptr) / sizeof(buffer_data_type); } /** * Constructs the PointerMapper structure. */ PointerMapper(base_ptr_t baseAddress = 4096) : m_pointerMap{}, m_freeList{}, m_baseAddress{baseAddress} { if (m_baseAddress == 0) { EIGEN_THROW_X(std::invalid_argument("Base address cannot be zero\n")); } }; /** * PointerMapper cannot be copied or moved */ PointerMapper(const PointerMapper &) = delete; /** * Empty the pointer list */ inline void clear() { m_freeList.clear(); m_pointerMap.clear(); } /* add_pointer. * Adds an existing pointer to the map and returns the virtual pointer id. */ inline virtual_pointer_t add_pointer(const buffer_t &b) { return add_pointer_impl(b); } /* add_pointer. * Adds a pointer to the map and returns the virtual pointer id. */ inline virtual_pointer_t add_pointer(buffer_t &&b) { return add_pointer_impl(b); } /** * @brief Fuses the given node with the previous nodes in the * pointer map if they are free * * @param node A reference to the free node to be fused */ void fuse_forward(typename pointerMap_t::iterator &node) { while (node != std::prev(m_pointerMap.end())) { // if following node is free // remove it and extend the current node with its size auto fwd_node = std::next(node); if (!fwd_node->second.m_free) { break; } auto fwd_size = fwd_node->second.m_size; m_freeList.erase(fwd_node); m_pointerMap.erase(fwd_node); node->second.m_size += fwd_size; } } /** * @brief Fuses the given node with the following nodes in the * pointer map if they are free * * @param node A reference to the free node to be fused */ void fuse_backward(typename pointerMap_t::iterator &node) { while (node != m_pointerMap.begin()) { // if previous node is free, extend it // with the size of the current one auto prev_node = std::prev(node); if (!prev_node->second.m_free) { break; } prev_node->second.m_size += node->second.m_size; // remove the current node m_freeList.erase(node); m_pointerMap.erase(node); // point to the previous node node = prev_node; } } /* remove_pointer. * Removes the given pointer from the map. * The pointer is allowed to be reused only if ReUse if true. */ template void remove_pointer(const virtual_pointer_t ptr) { if (is_nullptr(ptr)) { return; } auto node = this->get_node(ptr); node->second.m_free = true; m_freeList.emplace(node); // Fuse the node // with free nodes before and after it fuse_forward(node); fuse_backward(node); // If after fusing the node is the last one // simply remove it (since it is free) if (node == std::prev(m_pointerMap.end())) { m_freeList.erase(node); m_pointerMap.erase(node); } } /* count. * Return the number of active pointers (i.e, pointers that * have been malloc but not freed). */ size_t count() const { return (m_pointerMap.size() - m_freeList.size()); } private: /* add_pointer_impl. * Adds a pointer to the map and returns the virtual pointer id. * BufferT is either a const buffer_t& or a buffer_t&&. */ template virtual_pointer_t add_pointer_impl(BufferT b) { virtual_pointer_t retVal = nullptr; size_t bufSize = b.get_count(); pMapNode_t p{b, bufSize, false}; // If this is the first pointer: if (m_pointerMap.empty()) { virtual_pointer_t initialVal{m_baseAddress}; m_pointerMap.emplace(initialVal, p); return initialVal; } auto lastElemIter = get_insertion_point(bufSize); // We are recovering an existing free node if (lastElemIter->second.m_free) { lastElemIter->second.m_buffer = b; lastElemIter->second.m_free = false; // If the recovered node is bigger than the inserted one // add a new free node with the remaining space if (lastElemIter->second.m_size > bufSize) { // create a new node with the remaining space auto remainingSize = lastElemIter->second.m_size - bufSize; pMapNode_t p2{b, remainingSize, true}; // update size of the current node lastElemIter->second.m_size = bufSize; // add the new free node auto newFreePtr = lastElemIter->first + bufSize; auto freeNode = m_pointerMap.emplace(newFreePtr, p2).first; m_freeList.emplace(freeNode); } retVal = lastElemIter->first; } else { size_t lastSize = lastElemIter->second.m_size; retVal = lastElemIter->first + lastSize; m_pointerMap.emplace(retVal, p); } return retVal; } /** * Compare two iterators to pointer map entries according to * the size of the allocation on the device. */ struct SortBySize { bool operator()(typename pointerMap_t::iterator a, typename pointerMap_t::iterator b) const { return ((a->first < b->first) && (a->second <= b->second)) || ((a->first < b->first) && (b->second <= a->second)); } }; /* Maps the pointer addresses to buffer and size pairs. */ pointerMap_t m_pointerMap; /* List of free nodes available for re-using */ std::set m_freeList; /* Base address used when issuing the first virtual pointer, allows users * to specify alignment. Cannot be zero. */ std::intptr_t m_baseAddress; }; /* remove_pointer. * Removes the given pointer from the map. * The pointer is allowed to be reused only if ReUse if true. */ template <> inline void PointerMapper::remove_pointer(const virtual_pointer_t ptr) { if (is_nullptr(ptr)) { return; } m_pointerMap.erase(this->get_node(ptr)); } /** * Malloc-like interface to the pointer-mapper. * Given a size, creates a byte-typed buffer and returns a * fake pointer to keep track of it. * \param size Size in bytes of the desired allocation * \throw cl::sycl::exception if error while creating the buffer */ inline void *SYCLmalloc(size_t size, PointerMapper &pMap) { if (size == 0) { return nullptr; } // Create a generic buffer of the given size using buffer_t = cl::sycl::buffer; auto thePointer = pMap.add_pointer(buffer_t(cl::sycl::range<1>{size})); // Store the buffer on the global list return static_cast(thePointer); } /** * Free-like interface to the pointer mapper. * Given a fake-pointer created with the virtual-pointer malloc, * destroys the buffer and remove it from the list. * If ReUse is false, the pointer is not added to the freeList, * it should be false only for sub-buffers. */ template inline void SYCLfree(void *ptr, PointerMapper &pMap) { pMap.template remove_pointer(ptr); } /** * Clear all the memory allocated by SYCL. */ template inline void SYCLfreeAll(PointerMapper &pMap) { pMap.clear(); } template struct RangeAccess { static const auto global_access = cl::sycl::access::target::global_buffer; static const auto is_place_holder = cl::sycl::access::placeholder::true_t; typedef T scalar_t; typedef scalar_t &ref_t; typedef typename cl::sycl::global_ptr::pointer_t ptr_t; // the accessor type does not necessarily the same as T typedef cl::sycl::accessor accessor; typedef RangeAccess self_t; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE RangeAccess(accessor access, size_t offset, std::intptr_t virtual_ptr) : access_(access), offset_(offset), virtual_ptr_(virtual_ptr) {} RangeAccess(cl::sycl::buffer buff = cl::sycl::buffer(cl::sycl::range<1>(1))) : access_{accessor{buff}}, offset_(0), virtual_ptr_(-1) {} // This should be only used for null constructor on the host side RangeAccess(std::nullptr_t) : RangeAccess() {} // This template parameter must be removed and scalar_t should be replaced EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ptr_t get_pointer() const { return (access_.get_pointer().get() + offset_); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE self_t &operator+=(Index offset) { offset_ += (offset); return *this; } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE self_t operator+(Index offset) const { return self_t(access_, offset_ + offset, virtual_ptr_); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE self_t operator-(Index offset) const { return self_t(access_, offset_ - offset, virtual_ptr_); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE self_t &operator-=(Index offset) { offset_ -= offset; return *this; } // THIS IS FOR NULL COMPARISON ONLY EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE friend bool operator==( const RangeAccess &lhs, std::nullptr_t) { return ((lhs.virtual_ptr_ == -1)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE friend bool operator!=( const RangeAccess &lhs, std::nullptr_t i) { return !(lhs == i); } // THIS IS FOR NULL COMPARISON ONLY EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE friend bool operator==( std::nullptr_t, const RangeAccess &rhs) { return ((rhs.virtual_ptr_ == -1)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE friend bool operator!=( std::nullptr_t i, const RangeAccess &rhs) { return !(i == rhs); } // Prefix operator (Increment and return value) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE self_t &operator++() { offset_++; return (*this); } // Postfix operator (Return value and increment) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE self_t operator++(int i) { EIGEN_UNUSED_VARIABLE(i); self_t temp_iterator(*this); offset_++; return temp_iterator; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t get_size() const { return (access_.get_count() - offset_); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t get_offset() const { return offset_; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void set_offset(std::ptrdiff_t offset) { offset_ = offset; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ref_t operator*() const { return *get_pointer(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ref_t operator*() { return *get_pointer(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ptr_t operator->() = delete; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ref_t operator[](int x) { return *(get_pointer() + x); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ref_t operator[](int x) const { return *(get_pointer() + x); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE scalar_t *get_virtual_pointer() const { return reinterpret_cast(virtual_ptr_ + (offset_ * sizeof(scalar_t))); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit operator bool() const { return (virtual_ptr_ != -1); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE operator RangeAccess() { return RangeAccess(access_, offset_, virtual_ptr_); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE operator RangeAccess() const { return RangeAccess(access_, offset_, virtual_ptr_); } // binding placeholder accessors to a command group handler for SYCL EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void bind( cl::sycl::handler &cgh) const { cgh.require(access_); } private: accessor access_; size_t offset_; std::intptr_t virtual_ptr_; // the location of the buffer in the map }; template struct RangeAccess : RangeAccess { typedef RangeAccess Base; using Base::Base; }; } // namespace internal } // namespace TensorSycl } // namespace Eigen #endif // EIGEN_CXX11_TENSOR_TENSOR_SYCL_STORAGE_MEMORY_H RcppEigen/inst/include/Eigen/src/Core/arch/SYCL/PacketMath.h0000644000176200001440000006621214567757725023155 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Mehdi Goli Codeplay Software Ltd. // Ralph Potter Codeplay Software Ltd. // Luke Iwanski Codeplay Software Ltd. // Contact: // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. /***************************************************************** * PacketMath.h * * \brief: * PacketMath * *****************************************************************/ #ifndef EIGEN_PACKET_MATH_SYCL_H #define EIGEN_PACKET_MATH_SYCL_H #include namespace Eigen { namespace internal { #ifdef SYCL_DEVICE_ONLY #define SYCL_PLOADT_RO(address_space_target) \ template \ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type ploadt_ro( \ typename cl::sycl::multi_ptr< \ const typename unpacket_traits::type, \ cl::sycl::access::address_space::address_space_target>::pointer_t \ from) { \ typedef typename unpacket_traits::type scalar; \ typedef cl::sycl::multi_ptr< \ scalar, cl::sycl::access::address_space::address_space_target> \ multi_ptr; \ auto res = packet_type( \ static_cast::type>(0)); \ res.load(0, multi_ptr(const_cast(from))); \ return res; \ } SYCL_PLOADT_RO(global_space) SYCL_PLOADT_RO(local_space) #undef SYCL_PLOADT_RO #endif template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type ploadt_ro(const Eigen::TensorSycl::internal::RangeAccess< cl::sycl::access::mode::read_write, T>& from) { return ploadt_ro(from.get_pointer()); } #ifdef SYCL_DEVICE_ONLY #define SYCL_PLOAD(address_space_target, Alignment, AlignedType) \ template \ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type pload##AlignedType( \ typename cl::sycl::multi_ptr< \ const typename unpacket_traits::type, \ cl::sycl::access::address_space::address_space_target>::pointer_t \ from) { \ return ploadt_ro(from); \ } // global space SYCL_PLOAD(global_space, Unaligned, u) SYCL_PLOAD(global_space, Aligned, ) // local space SYCL_PLOAD(local_space, Unaligned, u) SYCL_PLOAD(local_space, Aligned, ) #undef SYCL_PLOAD #endif #define SYCL_PLOAD(Alignment, AlignedType) \ template \ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type pload##AlignedType( \ const Eigen::TensorSycl::internal::RangeAccess< \ cl::sycl::access::mode::read_write, \ typename unpacket_traits::type> \ from) { \ return ploadt_ro(from); \ } SYCL_PLOAD(Unaligned, u) SYCL_PLOAD(Aligned, ) #undef SYCL_PLOAD #ifdef SYCL_DEVICE_ONLY /** \internal \returns a packet version of \a *from. * The pointer \a from must be aligned on a \a Alignment bytes boundary. */ #define SYCL_PLOADT(address_space_target) \ template \ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type ploadt( \ typename cl::sycl::multi_ptr< \ const typename unpacket_traits::type, \ cl::sycl::access::address_space::address_space_target>::pointer_t \ from) { \ if (Alignment >= unpacket_traits::alignment) \ return pload(from); \ else \ return ploadu(from); \ } // global space SYCL_PLOADT(global_space) // local space SYCL_PLOADT(local_space) #undef SYCL_PLOADT #endif template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type ploadt(const Eigen::TensorSycl::internal::RangeAccess< cl::sycl::access::mode::read_write, typename unpacket_traits::type>& from) { return ploadt(from.get_pointer()); } #ifdef SYCL_DEVICE_ONLY // private_space #define SYCL_PLOADT_RO_SPECIAL(packet_type, Alignment) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type \ ploadt_ro( \ const typename unpacket_traits::type* from) { \ typedef typename unpacket_traits::type scalar; \ auto res = packet_type(static_cast(0)); \ res.template load( \ 0, const_cast(from)); \ return res; \ } SYCL_PLOADT_RO_SPECIAL(cl::sycl::cl_float4, Aligned) SYCL_PLOADT_RO_SPECIAL(cl::sycl::cl_double2, Aligned) SYCL_PLOADT_RO_SPECIAL(cl::sycl::cl_float4, Unaligned) SYCL_PLOADT_RO_SPECIAL(cl::sycl::cl_double2, Unaligned) #define SYCL_PLOAD_SPECIAL(packet_type, alignment_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type pload##alignment_type( \ const typename unpacket_traits::type* from) { \ typedef typename unpacket_traits::type scalar; \ auto res = packet_type(static_cast(0)); \ res.template load( \ 0, const_cast(from)); \ return res; \ } SYCL_PLOAD_SPECIAL(cl::sycl::cl_float4, ) SYCL_PLOAD_SPECIAL(cl::sycl::cl_double2, ) SYCL_PLOAD_SPECIAL(cl::sycl::cl_float4, u) SYCL_PLOAD_SPECIAL(cl::sycl::cl_double2, u) #undef SYCL_PLOAD_SPECIAL #define SYCL_PSTORE(scalar, packet_type, address_space_target, alignment) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstore##alignment( \ typename cl::sycl::multi_ptr< \ scalar, \ cl::sycl::access::address_space::address_space_target>::pointer_t \ to, \ const packet_type& from) { \ typedef cl::sycl::multi_ptr< \ scalar, cl::sycl::access::address_space::address_space_target> \ multi_ptr; \ from.store(0, multi_ptr(to)); \ } // global space SYCL_PSTORE(float, cl::sycl::cl_float4, global_space, ) SYCL_PSTORE(float, cl::sycl::cl_float4, global_space, u) SYCL_PSTORE(double, cl::sycl::cl_double2, global_space, ) SYCL_PSTORE(double, cl::sycl::cl_double2, global_space, u) SYCL_PSTORE(float, cl::sycl::cl_float4, local_space, ) SYCL_PSTORE(float, cl::sycl::cl_float4, local_space, u) SYCL_PSTORE(double, cl::sycl::cl_double2, local_space, ) SYCL_PSTORE(double, cl::sycl::cl_double2, local_space, u) SYCL_PSTORE(float, cl::sycl::cl_float4, private_space, ) SYCL_PSTORE(float, cl::sycl::cl_float4, private_space, u) SYCL_PSTORE(double, cl::sycl::cl_double2, private_space, ) SYCL_PSTORE(double, cl::sycl::cl_double2, private_space, u) #undef SYCL_PSTORE #define SYCL_PSTORE_T(address_space_target) \ template \ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret( \ typename cl::sycl::multi_ptr< \ scalar, \ cl::sycl::access::address_space::address_space_target>::pointer_t \ to, \ const packet_type& from) { \ if (Alignment) \ pstore(to, from); \ else \ pstoreu(to, from); \ } SYCL_PSTORE_T(global_space) SYCL_PSTORE_T(local_space) #undef SYCL_PSTORE_T #define SYCL_PSET1(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type pset1( \ const typename unpacket_traits::type& from) { \ return packet_type(from); \ } // global space SYCL_PSET1(cl::sycl::cl_float4) SYCL_PSET1(cl::sycl::cl_double2) #undef SYCL_PSET1 template struct get_base_packet { template static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type get_ploaddup(sycl_multi_pointer) {} template static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type get_pgather(sycl_multi_pointer, Index) {} }; template <> struct get_base_packet { template static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE cl::sycl::cl_float4 get_ploaddup( sycl_multi_pointer from) { return cl::sycl::cl_float4(from[0], from[0], from[1], from[1]); } template static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE cl::sycl::cl_float4 get_pgather( sycl_multi_pointer from, Index stride) { return cl::sycl::cl_float4(from[0 * stride], from[1 * stride], from[2 * stride], from[3 * stride]); } template static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void set_pscatter( sycl_multi_pointer to, const cl::sycl::cl_float4& from, Index stride) { auto tmp = stride; to[0] = from.x(); to[tmp] = from.y(); to[tmp += stride] = from.z(); to[tmp += stride] = from.w(); } static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE cl::sycl::cl_float4 set_plset( const float& a) { return cl::sycl::cl_float4(static_cast(a), static_cast(a + 1), static_cast(a + 2), static_cast(a + 3)); } }; template <> struct get_base_packet { template static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE cl::sycl::cl_double2 get_ploaddup(const sycl_multi_pointer from) { return cl::sycl::cl_double2(from[0], from[0]); } template static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE cl::sycl::cl_double2 get_pgather( const sycl_multi_pointer from, Index stride) { return cl::sycl::cl_double2(from[0 * stride], from[1 * stride]); } template static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void set_pscatter( sycl_multi_pointer to, const cl::sycl::cl_double2& from, Index stride) { to[0] = from.x(); to[stride] = from.y(); } static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE cl::sycl::cl_double2 set_plset( const double& a) { return cl::sycl::cl_double2(static_cast(a), static_cast(a + 1)); } }; #define SYCL_PLOAD_DUP(address_space_target) \ template \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type ploaddup( \ typename cl::sycl::multi_ptr< \ const typename unpacket_traits::type, \ cl::sycl::access::address_space::address_space_target>::pointer_t \ from) { \ return get_base_packet::get_ploaddup(from); \ } // global space SYCL_PLOAD_DUP(global_space) // local_space SYCL_PLOAD_DUP(local_space) #undef SYCL_PLOAD_DUP #define SYCL_PLOAD_DUP_SPECILIZE(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type ploaddup( \ const typename unpacket_traits::type* from) { \ return get_base_packet::get_ploaddup(from); \ } SYCL_PLOAD_DUP_SPECILIZE(cl::sycl::cl_float4) SYCL_PLOAD_DUP_SPECILIZE(cl::sycl::cl_double2) #undef SYCL_PLOAD_DUP_SPECILIZE #define SYCL_PLSET(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type plset( \ const typename unpacket_traits::type& a) { \ return get_base_packet::set_plset(a); \ } SYCL_PLSET(cl::sycl::cl_float4) SYCL_PLSET(cl::sycl::cl_double2) #undef SYCL_PLSET #define SYCL_PGATHER(address_space_target) \ template \ EIGEN_DEVICE_FUNC inline packet_type pgather( \ typename cl::sycl::multi_ptr< \ const typename unpacket_traits::type, \ cl::sycl::access::address_space::address_space_target>::pointer_t \ from, \ Index stride) { \ return get_base_packet::get_pgather(from, stride); \ } // global space SYCL_PGATHER(global_space) // local space SYCL_PGATHER(local_space) #undef SYCL_PGATHER #define SYCL_PGATHER_SPECILIZE(scalar, packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type \ pgather( \ const typename unpacket_traits::type* from, Index stride) { \ return get_base_packet::get_pgather(from, stride); \ } SYCL_PGATHER_SPECILIZE(float, cl::sycl::cl_float4) SYCL_PGATHER_SPECILIZE(double, cl::sycl::cl_double2) #undef SYCL_PGATHER_SPECILIZE #define SYCL_PSCATTER(address_space_target) \ template \ EIGEN_DEVICE_FUNC inline void pscatter( \ typename cl::sycl::multi_ptr< \ typename unpacket_traits::type, \ cl::sycl::access::address_space::address_space_target>::pointer_t \ to, \ const packet_type& from, Index stride) { \ get_base_packet::set_pscatter(to, from, stride); \ } // global space SYCL_PSCATTER(global_space) // local space SYCL_PSCATTER(local_space) #undef SYCL_PSCATTER #define SYCL_PSCATTER_SPECILIZE(scalar, packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter( \ typename unpacket_traits::type * to, \ const packet_type& from, Index stride) { \ get_base_packet::set_pscatter(to, from, stride); \ } SYCL_PSCATTER_SPECILIZE(float, cl::sycl::cl_float4) SYCL_PSCATTER_SPECILIZE(double, cl::sycl::cl_double2) #undef SYCL_PSCATTER_SPECILIZE #define SYCL_PMAD(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE packet_type pmadd( \ const packet_type& a, const packet_type& b, const packet_type& c) { \ return cl::sycl::mad(a, b, c); \ } SYCL_PMAD(cl::sycl::cl_float4) SYCL_PMAD(cl::sycl::cl_double2) #undef SYCL_PMAD template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float pfirst( const cl::sycl::cl_float4& a) { return a.x(); } template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double pfirst( const cl::sycl::cl_double2& a) { return a.x(); } template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float predux( const cl::sycl::cl_float4& a) { return a.x() + a.y() + a.z() + a.w(); } template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double predux( const cl::sycl::cl_double2& a) { return a.x() + a.y(); } template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float predux_max( const cl::sycl::cl_float4& a) { return cl::sycl::fmax(cl::sycl::fmax(a.x(), a.y()), cl::sycl::fmax(a.z(), a.w())); } template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double predux_max( const cl::sycl::cl_double2& a) { return cl::sycl::fmax(a.x(), a.y()); } template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float predux_min( const cl::sycl::cl_float4& a) { return cl::sycl::fmin(cl::sycl::fmin(a.x(), a.y()), cl::sycl::fmin(a.z(), a.w())); } template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double predux_min( const cl::sycl::cl_double2& a) { return cl::sycl::fmin(a.x(), a.y()); } template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float predux_mul( const cl::sycl::cl_float4& a) { return a.x() * a.y() * a.z() * a.w(); } template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double predux_mul( const cl::sycl::cl_double2& a) { return a.x() * a.y(); } template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_float4 pabs(const cl::sycl::cl_float4& a) { return cl::sycl::cl_float4(cl::sycl::fabs(a.x()), cl::sycl::fabs(a.y()), cl::sycl::fabs(a.z()), cl::sycl::fabs(a.w())); } template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_double2 pabs(const cl::sycl::cl_double2& a) { return cl::sycl::cl_double2(cl::sycl::fabs(a.x()), cl::sycl::fabs(a.y())); } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet sycl_pcmp_le(const Packet &a, const Packet &b) { return ((a <= b) .template convert::type, cl::sycl::rounding_mode::automatic>()); } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet sycl_pcmp_lt(const Packet &a, const Packet &b) { return ((a < b) .template convert::type, cl::sycl::rounding_mode::automatic>()); } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet sycl_pcmp_eq(const Packet &a, const Packet &b) { return ((a == b) .template convert::type, cl::sycl::rounding_mode::automatic>()); } #define SYCL_PCMP(OP, TYPE) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE TYPE pcmp_##OP(const TYPE &a, \ const TYPE &b) { \ return sycl_pcmp_##OP(a, b); \ } SYCL_PCMP(le, cl::sycl::cl_float4) SYCL_PCMP(lt, cl::sycl::cl_float4) SYCL_PCMP(eq, cl::sycl::cl_float4) SYCL_PCMP(le, cl::sycl::cl_double2) SYCL_PCMP(lt, cl::sycl::cl_double2) SYCL_PCMP(eq, cl::sycl::cl_double2) #undef SYCL_PCMP template struct convert_to_integer; template <> struct convert_to_integer { using type = std::int32_t; using packet_type = cl::sycl::cl_int4; }; template <> struct convert_to_integer { using type = std::int64_t; using packet_type = cl::sycl::cl_long2; }; template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename convert_to_integer< typename unpacket_traits::type>::packet_type vector_as_int(const PacketIn &p) { return ( p.template convert::type>::type, cl::sycl::rounding_mode::automatic>()); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packetOut convert_vector(const PacketIn &p) { return (p.template convert::type, cl::sycl::rounding_mode::automatic>()); } #define SYCL_PAND(TYPE) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TYPE pand(const TYPE &a, \ const TYPE &b) { \ return convert_vector(vector_as_int(a) & vector_as_int(b)); \ } SYCL_PAND(cl::sycl::cl_float4) SYCL_PAND(cl::sycl::cl_double2) #undef SYCL_PAND #define SYCL_POR(TYPE) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TYPE por(const TYPE &a, \ const TYPE &b) { \ return convert_vector(vector_as_int(a) | vector_as_int(b)); \ } SYCL_POR(cl::sycl::cl_float4) SYCL_POR(cl::sycl::cl_double2) #undef SYCL_POR #define SYCL_PXOR(TYPE) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TYPE pxor(const TYPE &a, \ const TYPE &b) { \ return convert_vector(vector_as_int(a) ^ vector_as_int(b)); \ } SYCL_PXOR(cl::sycl::cl_float4) SYCL_PXOR(cl::sycl::cl_double2) #undef SYCL_PXOR #define SYCL_PANDNOT(TYPE) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TYPE pandnot(const TYPE &a, \ const TYPE &b) { \ return convert_vector(vector_as_int(a) & (~vector_as_int(b))); \ } SYCL_PANDNOT(cl::sycl::cl_float4) SYCL_PANDNOT(cl::sycl::cl_double2) #undef SYCL_PANDNOT EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void ptranspose( PacketBlock& kernel) { float tmp = kernel.packet[0].y(); kernel.packet[0].y() = kernel.packet[1].x(); kernel.packet[1].x() = tmp; tmp = kernel.packet[0].z(); kernel.packet[0].z() = kernel.packet[2].x(); kernel.packet[2].x() = tmp; tmp = kernel.packet[0].w(); kernel.packet[0].w() = kernel.packet[3].x(); kernel.packet[3].x() = tmp; tmp = kernel.packet[1].z(); kernel.packet[1].z() = kernel.packet[2].y(); kernel.packet[2].y() = tmp; tmp = kernel.packet[1].w(); kernel.packet[1].w() = kernel.packet[3].y(); kernel.packet[3].y() = tmp; tmp = kernel.packet[2].w(); kernel.packet[2].w() = kernel.packet[3].z(); kernel.packet[3].z() = tmp; } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void ptranspose( PacketBlock& kernel) { double tmp = kernel.packet[0].y(); kernel.packet[0].y() = kernel.packet[1].x(); kernel.packet[1].x() = tmp; } template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_float4 pblend( const Selector::size>& ifPacket, const cl::sycl::cl_float4& thenPacket, const cl::sycl::cl_float4& elsePacket) { cl::sycl::cl_int4 condition( ifPacket.select[0] ? 0 : -1, ifPacket.select[1] ? 0 : -1, ifPacket.select[2] ? 0 : -1, ifPacket.select[3] ? 0 : -1); return cl::sycl::select(thenPacket, elsePacket, condition); } template <> inline cl::sycl::cl_double2 pblend( const Selector::size>& ifPacket, const cl::sycl::cl_double2& thenPacket, const cl::sycl::cl_double2& elsePacket) { cl::sycl::cl_long2 condition(ifPacket.select[0] ? 0 : -1, ifPacket.select[1] ? 0 : -1); return cl::sycl::select(thenPacket, elsePacket, condition); } #endif // SYCL_DEVICE_ONLY #define SYCL_PSTORE(alignment) \ template \ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstore##alignment( \ const Eigen::TensorSycl::internal::RangeAccess< \ cl::sycl::access::mode::read_write, \ typename unpacket_traits::type>& to, \ const packet_type& from) { \ pstore##alignment(to.get_pointer(), from); \ } // global space SYCL_PSTORE() SYCL_PSTORE(u) #undef SYCL_PSTORE template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret( Eigen::TensorSycl::internal::RangeAccess< cl::sycl::access::mode::read_write, typename unpacket_traits::type> to, const packet_type& from) { pstoret(to.get_pointer(), from); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_PACKET_MATH_SYCL_H RcppEigen/inst/include/Eigen/src/Core/arch/SYCL/MathFunctions.h0000644000176200001440000003037314567757725023715 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Mehdi Goli Codeplay Software Ltd. // Ralph Potter Codeplay Software Ltd. // Luke Iwanski Codeplay Software Ltd. // Contact: // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. /***************************************************************** * MathFunctions.h * * \brief: * MathFunctions * *****************************************************************/ #ifndef EIGEN_MATH_FUNCTIONS_SYCL_H #define EIGEN_MATH_FUNCTIONS_SYCL_H namespace Eigen { namespace internal { // Make sure this is only available when targeting a GPU: we don't want to // introduce conflicts between these packet_traits definitions and the ones // we'll use on the host side (SSE, AVX, ...) #if defined(SYCL_DEVICE_ONLY) #define SYCL_PLOG(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type plog( \ const packet_type& a) { \ return cl::sycl::log(a); \ } SYCL_PLOG(cl::sycl::cl_float4) SYCL_PLOG(cl::sycl::cl_double2) #undef SYCL_PLOG #define SYCL_PLOG1P(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type plog1p( \ const packet_type& a) { \ return cl::sycl::log1p(a); \ } SYCL_PLOG1P(cl::sycl::cl_float4) SYCL_PLOG1P(cl::sycl::cl_double2) #undef SYCL_PLOG1P #define SYCL_PLOG10(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type plog10( \ const packet_type& a) { \ return cl::sycl::log10(a); \ } SYCL_PLOG10(cl::sycl::cl_float4) SYCL_PLOG10(cl::sycl::cl_double2) #undef SYCL_PLOG10 #define SYCL_PEXP(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type pexp( \ const packet_type& a) { \ return cl::sycl::exp(a); \ } SYCL_PEXP(cl::sycl::cl_float4) SYCL_PEXP(cl::sycl::cl_float) SYCL_PEXP(cl::sycl::cl_double2) #undef SYCL_PEXP #define SYCL_PEXPM1(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type pexpm1( \ const packet_type& a) { \ return cl::sycl::expm1(a); \ } SYCL_PEXPM1(cl::sycl::cl_float4) SYCL_PEXPM1(cl::sycl::cl_double2) #undef SYCL_PEXPM1 #define SYCL_PSQRT(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type psqrt( \ const packet_type& a) { \ return cl::sycl::sqrt(a); \ } SYCL_PSQRT(cl::sycl::cl_float4) SYCL_PSQRT(cl::sycl::cl_double2) #undef SYCL_PSQRT #define SYCL_PRSQRT(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type prsqrt( \ const packet_type& a) { \ return cl::sycl::rsqrt(a); \ } SYCL_PRSQRT(cl::sycl::cl_float4) SYCL_PRSQRT(cl::sycl::cl_double2) #undef SYCL_PRSQRT /** \internal \returns the hyperbolic sine of \a a (coeff-wise) */ #define SYCL_PSIN(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type psin( \ const packet_type& a) { \ return cl::sycl::sin(a); \ } SYCL_PSIN(cl::sycl::cl_float4) SYCL_PSIN(cl::sycl::cl_double2) #undef SYCL_PSIN /** \internal \returns the hyperbolic cosine of \a a (coeff-wise) */ #define SYCL_PCOS(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type pcos( \ const packet_type& a) { \ return cl::sycl::cos(a); \ } SYCL_PCOS(cl::sycl::cl_float4) SYCL_PCOS(cl::sycl::cl_double2) #undef SYCL_PCOS /** \internal \returns the hyperbolic tan of \a a (coeff-wise) */ #define SYCL_PTAN(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type ptan( \ const packet_type& a) { \ return cl::sycl::tan(a); \ } SYCL_PTAN(cl::sycl::cl_float4) SYCL_PTAN(cl::sycl::cl_double2) #undef SYCL_PTAN /** \internal \returns the hyperbolic sine of \a a (coeff-wise) */ #define SYCL_PASIN(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type pasin( \ const packet_type& a) { \ return cl::sycl::asin(a); \ } SYCL_PASIN(cl::sycl::cl_float4) SYCL_PASIN(cl::sycl::cl_double2) #undef SYCL_PASIN /** \internal \returns the hyperbolic cosine of \a a (coeff-wise) */ #define SYCL_PACOS(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type pacos( \ const packet_type& a) { \ return cl::sycl::acos(a); \ } SYCL_PACOS(cl::sycl::cl_float4) SYCL_PACOS(cl::sycl::cl_double2) #undef SYCL_PACOS /** \internal \returns the hyperbolic tan of \a a (coeff-wise) */ #define SYCL_PATAN(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type patan( \ const packet_type& a) { \ return cl::sycl::atan(a); \ } SYCL_PATAN(cl::sycl::cl_float4) SYCL_PATAN(cl::sycl::cl_double2) #undef SYCL_PATAN /** \internal \returns the hyperbolic sine of \a a (coeff-wise) */ #define SYCL_PSINH(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type psinh( \ const packet_type& a) { \ return cl::sycl::sinh(a); \ } SYCL_PSINH(cl::sycl::cl_float4) SYCL_PSINH(cl::sycl::cl_double2) #undef SYCL_PSINH /** \internal \returns the hyperbolic cosine of \a a (coeff-wise) */ #define SYCL_PCOSH(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type pcosh( \ const packet_type& a) { \ return cl::sycl::cosh(a); \ } SYCL_PCOSH(cl::sycl::cl_float4) SYCL_PCOSH(cl::sycl::cl_double2) #undef SYCL_PCOSH /** \internal \returns the hyperbolic tan of \a a (coeff-wise) */ #define SYCL_PTANH(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type ptanh( \ const packet_type& a) { \ return cl::sycl::tanh(a); \ } SYCL_PTANH(cl::sycl::cl_float4) SYCL_PTANH(cl::sycl::cl_double2) #undef SYCL_PTANH #define SYCL_PCEIL(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type pceil( \ const packet_type& a) { \ return cl::sycl::ceil(a); \ } SYCL_PCEIL(cl::sycl::cl_float4) SYCL_PCEIL(cl::sycl::cl_double2) #undef SYCL_PCEIL #define SYCL_PROUND(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type pround( \ const packet_type& a) { \ return cl::sycl::round(a); \ } SYCL_PROUND(cl::sycl::cl_float4) SYCL_PROUND(cl::sycl::cl_double2) #undef SYCL_PROUND #define SYCL_PRINT(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type print( \ const packet_type& a) { \ return cl::sycl::rint(a); \ } SYCL_PRINT(cl::sycl::cl_float4) SYCL_PRINT(cl::sycl::cl_double2) #undef SYCL_PRINT #define SYCL_FLOOR(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type pfloor( \ const packet_type& a) { \ return cl::sycl::floor(a); \ } SYCL_FLOOR(cl::sycl::cl_float4) SYCL_FLOOR(cl::sycl::cl_double2) #undef SYCL_FLOOR #define SYCL_PMIN(packet_type, expr) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type pmin( \ const packet_type& a, const packet_type& b) { \ return expr; \ } SYCL_PMIN(cl::sycl::cl_float4, cl::sycl::fmin(a, b)) SYCL_PMIN(cl::sycl::cl_double2, cl::sycl::fmin(a, b)) #undef SYCL_PMIN #define SYCL_PMAX(packet_type, expr) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type pmax( \ const packet_type& a, const packet_type& b) { \ return expr; \ } SYCL_PMAX(cl::sycl::cl_float4, cl::sycl::fmax(a, b)) SYCL_PMAX(cl::sycl::cl_double2, cl::sycl::fmax(a, b)) #undef SYCL_PMAX #define SYCL_PLDEXP(packet_type) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet_type pldexp( \ const packet_type& a, const packet_type& exponent) { \ return cl::sycl::ldexp( \ a, exponent.template convert()); \ } SYCL_PLDEXP(cl::sycl::cl_float4) SYCL_PLDEXP(cl::sycl::cl_double2) #undef SYCL_PLDEXP #endif } // end namespace internal } // end namespace Eigen #endif // EIGEN_MATH_FUNCTIONS_SYCL_H RcppEigen/inst/include/Eigen/src/Core/arch/SYCL/InteropHeaders.h0000644000176200001440000001640414567757725024046 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Mehdi Goli Codeplay Software Ltd. // Ralph Potter Codeplay Software Ltd. // Luke Iwanski Codeplay Software Ltd. // Contact: // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. /***************************************************************** * InteropHeaders.h * * \brief: * InteropHeaders * *****************************************************************/ #ifndef EIGEN_INTEROP_HEADERS_SYCL_H #define EIGEN_INTEROP_HEADERS_SYCL_H namespace Eigen { #if !defined(EIGEN_DONT_VECTORIZE_SYCL) namespace internal { template struct sycl_packet_traits : default_packet_traits { enum { Vectorizable = 1, AlignedOnScalar = 1, size = lengths, HasHalfPacket = 0, HasDiv = 1, HasLog = 1, HasExp = 1, HasSqrt = 1, HasRsqrt = 1, HasSin = 1, HasCos = 1, HasTan = 1, HasASin = 1, HasACos = 1, HasATan = 1, HasSinh = 1, HasCosh = 1, HasTanh = 1, HasLGamma = 0, HasDiGamma = 0, HasZeta = 0, HasPolygamma = 0, HasErf = 0, HasErfc = 0, HasNdtri = 0, HasIGamma = 0, HasIGammac = 0, HasBetaInc = 0, HasBlend = has_blend, // This flag is used to indicate whether packet comparison is supported. // pcmp_eq, pcmp_lt and pcmp_le should be defined for it to be true. HasCmp = 1, HasMax = 1, HasMin = 1, HasMul = 1, HasAdd = 1, HasFloor = 1, HasRound = 1, HasRint = 1, HasLog1p = 1, HasExpm1 = 1, HasCeil = 1, }; }; #ifdef SYCL_DEVICE_ONLY #define SYCL_PACKET_TRAITS(packet_type, has_blend, unpacket_type, lengths) \ template <> \ struct packet_traits \ : sycl_packet_traits { \ typedef packet_type type; \ typedef packet_type half; \ }; SYCL_PACKET_TRAITS(cl::sycl::cl_float4, 1, float, 4) SYCL_PACKET_TRAITS(cl::sycl::cl_float4, 1, const float, 4) SYCL_PACKET_TRAITS(cl::sycl::cl_double2, 0, double, 2) SYCL_PACKET_TRAITS(cl::sycl::cl_double2, 0, const double, 2) #undef SYCL_PACKET_TRAITS // Make sure this is only available when targeting a GPU: we don't want to // introduce conflicts between these packet_traits definitions and the ones // we'll use on the host side (SSE, AVX, ...) #define SYCL_ARITHMETIC(packet_type) \ template <> \ struct is_arithmetic { \ enum { value = true }; \ }; SYCL_ARITHMETIC(cl::sycl::cl_float4) SYCL_ARITHMETIC(cl::sycl::cl_double2) #undef SYCL_ARITHMETIC #define SYCL_UNPACKET_TRAITS(packet_type, unpacket_type, lengths) \ template <> \ struct unpacket_traits { \ typedef unpacket_type type; \ enum { size = lengths, vectorizable = true, alignment = Aligned16 }; \ typedef packet_type half; \ }; SYCL_UNPACKET_TRAITS(cl::sycl::cl_float4, float, 4) SYCL_UNPACKET_TRAITS(cl::sycl::cl_double2, double, 2) #undef SYCL_UNPACKET_TRAITS #endif } // end namespace internal #endif namespace TensorSycl { namespace internal { template struct PacketWrapper; // This function should never get called on the device #ifndef SYCL_DEVICE_ONLY template struct PacketWrapper { typedef typename ::Eigen::internal::unpacket_traits::type Scalar; template EIGEN_DEVICE_FUNC static Scalar scalarize(Index, PacketReturnType &) { eigen_assert(false && "THERE IS NO PACKETIZE VERSION FOR THE CHOSEN TYPE"); abort(); } EIGEN_DEVICE_FUNC static PacketReturnType convert_to_packet_type(Scalar in, Scalar) { return ::Eigen::internal::template plset(in); } EIGEN_DEVICE_FUNC static void set_packet(PacketReturnType, Scalar *) { eigen_assert(false && "THERE IS NO PACKETIZE VERSION FOR THE CHOSEN TYPE"); abort(); } }; #elif defined(SYCL_DEVICE_ONLY) template struct PacketWrapper { typedef typename ::Eigen::internal::unpacket_traits::type Scalar; template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static Scalar scalarize(Index index, PacketReturnType &in) { switch (index) { case 0: return in.x(); case 1: return in.y(); case 2: return in.z(); case 3: return in.w(); default: //INDEX MUST BE BETWEEN 0 and 3.There is no abort function in SYCL kernel. so we cannot use abort here. // The code will never reach here __builtin_unreachable(); } __builtin_unreachable(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static PacketReturnType convert_to_packet_type( Scalar in, Scalar other) { return PacketReturnType(in, other, other, other); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void set_packet(PacketReturnType &lhs, Scalar *rhs) { lhs = PacketReturnType(rhs[0], rhs[1], rhs[2], rhs[3]); } }; template struct PacketWrapper { typedef typename ::Eigen::internal::unpacket_traits::type Scalar; template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static Scalar scalarize(Index, PacketReturnType &in) { return in; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static PacketReturnType convert_to_packet_type(Scalar in, Scalar) { return PacketReturnType(in); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void set_packet(PacketReturnType &lhs, Scalar *rhs) { lhs = rhs[0]; } }; template struct PacketWrapper { typedef typename ::Eigen::internal::unpacket_traits::type Scalar; template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static Scalar scalarize(Index index, PacketReturnType &in) { switch (index) { case 0: return in.x(); case 1: return in.y(); default: //INDEX MUST BE BETWEEN 0 and 1.There is no abort function in SYCL kernel. so we cannot use abort here. // The code will never reach here __builtin_unreachable(); } __builtin_unreachable(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static PacketReturnType convert_to_packet_type( Scalar in, Scalar other) { return PacketReturnType(in, other); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void set_packet(PacketReturnType &lhs, Scalar *rhs) { lhs = PacketReturnType(rhs[0], rhs[1]); } }; #endif } // end namespace internal } // end namespace TensorSycl } // end namespace Eigen #endif // EIGEN_INTEROP_HEADERS_SYCL_H RcppEigen/inst/include/Eigen/src/Core/arch/AltiVec/0000755000176200001440000000000014567757725021531 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/arch/AltiVec/MatrixProduct.h0000644000176200001440000035107314567757725024520 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2020 Everton Constantino (everton.constantino@ibm.com) // Copyright (C) 2021 Chip Kerchner (chip.kerchner@ibm.com) // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_MATRIX_PRODUCT_ALTIVEC_H #define EIGEN_MATRIX_PRODUCT_ALTIVEC_H #ifndef EIGEN_ALTIVEC_USE_CUSTOM_PACK #define EIGEN_ALTIVEC_USE_CUSTOM_PACK 1 #endif #include "MatrixProductCommon.h" // Since LLVM doesn't support dynamic dispatching, force either always MMA or VSX #if EIGEN_COMP_LLVM #if !defined(EIGEN_ALTIVEC_DISABLE_MMA) && !defined(EIGEN_ALTIVEC_MMA_ONLY) #ifdef __MMA__ #define EIGEN_ALTIVEC_MMA_ONLY #else #define EIGEN_ALTIVEC_DISABLE_MMA #endif #endif #endif #ifdef __has_builtin #if __has_builtin(__builtin_mma_assemble_acc) #define ALTIVEC_MMA_SUPPORT #endif #endif #if defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) #include "MatrixProductMMA.h" #endif /************************************************************************************************** * TODO * * - Check StorageOrder on dhs_pack (the innermost second loop seems unvectorized when it could). * * - Check the possibility of transposing as GETREAL and GETIMAG when needed. * **************************************************************************************************/ namespace Eigen { namespace internal { /************************** * Constants and typedefs * **************************/ template struct quad_traits { typedef typename packet_traits::type vectortype; typedef PacketBlock type; typedef vectortype rhstype; enum { vectorsize = packet_traits::size, size = 4, rows = 4 }; }; template<> struct quad_traits { typedef Packet2d vectortype; typedef PacketBlock type; typedef PacketBlock rhstype; enum { vectorsize = packet_traits::size, size = 2, rows = 4 }; }; // MatrixProduct decomposes real/imaginary vectors into a real vector and an imaginary vector, this turned out // to be faster than Eigen's usual approach of having real/imaginary pairs on a single vector. This constants then // are responsible to extract from convert between Eigen's and MatrixProduct approach. const static Packet16uc p16uc_GETREAL32 = { 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27}; const static Packet16uc p16uc_GETIMAG32 = { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31}; const static Packet16uc p16uc_GETREAL64 = { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23}; //[a,ai],[b,bi] = [ai,bi] const static Packet16uc p16uc_GETIMAG64 = { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31}; /********************************************* * Single precision real and complex packing * * *******************************************/ /** * Symm packing is related to packing of symmetric adjoint blocks, as expected the packing leaves * the diagonal real, whatever is below it is copied from the respective upper diagonal element and * conjugated. There's no PanelMode available for symm packing. * * Packing in general is supposed to leave the lhs block and the rhs block easy to be read by gemm using * its respective rank-update instructions. The float32/64 versions are different because at this moment * the size of the accumulator is fixed at 512-bits so you can't have a 4x4 accumulator of 64-bit elements. * * As mentioned earlier MatrixProduct breaks complex numbers into a real vector and a complex vector so packing has * to take that into account, at the moment, we run pack the real part and then the imaginary part, this is the main * reason why packing for complex is broken down into several different parts, also the reason why we endup having a * float32/64 and complex float32/64 version. **/ template EIGEN_ALWAYS_INLINE std::complex getAdjointVal(Index i, Index j, const_blas_data_mapper, Index, StorageOrder>& dt) { std::complex v; if(i < j) { v.real( dt(j,i).real()); v.imag(-dt(j,i).imag()); } else if(i > j) { v.real( dt(i,j).real()); v.imag( dt(i,j).imag()); } else { v.real( dt(i,j).real()); v.imag((Scalar)0.0); } return v; } template EIGEN_STRONG_INLINE void symm_pack_complex_rhs_helper(std::complex* blockB, const std::complex* _rhs, Index rhsStride, Index rows, Index cols, Index k2) { const Index depth = k2 + rows; const_blas_data_mapper, Index, StorageOrder> rhs(_rhs, rhsStride); const Index vectorSize = N*quad_traits::vectorsize; const Index vectorDelta = vectorSize * rows; Scalar* blockBf = reinterpret_cast(blockB); Index rir = 0, rii, j = 0; for(; j + vectorSize <= cols; j+=vectorSize) { rii = rir + vectorDelta; for(Index i = k2; i < depth; i++) { for(Index k = 0; k < vectorSize; k++) { std::complex v = getAdjointVal(i, j + k, rhs); blockBf[rir + k] = v.real(); blockBf[rii + k] = v.imag(); } rir += vectorSize; rii += vectorSize; } rir += vectorDelta; } if (j < cols) { rii = rir + ((cols - j) * rows); for(Index i = k2; i < depth; i++) { Index k = j; for(; k < cols; k++) { std::complex v = getAdjointVal(i, k, rhs); blockBf[rir] = v.real(); blockBf[rii] = v.imag(); rir += 1; rii += 1; } } } } template EIGEN_STRONG_INLINE void symm_pack_complex_lhs_helper(std::complex* blockA, const std::complex* _lhs, Index lhsStride, Index cols, Index rows) { const Index depth = cols; const_blas_data_mapper, Index, StorageOrder> lhs(_lhs, lhsStride); const Index vectorSize = quad_traits::vectorsize; const Index vectorDelta = vectorSize * depth; Scalar* blockAf = (Scalar *)(blockA); Index rir = 0, rii, j = 0; for(; j + vectorSize <= rows; j+=vectorSize) { rii = rir + vectorDelta; for(Index i = 0; i < depth; i++) { for(Index k = 0; k < vectorSize; k++) { std::complex v = getAdjointVal(j+k, i, lhs); blockAf[rir + k] = v.real(); blockAf[rii + k] = v.imag(); } rir += vectorSize; rii += vectorSize; } rir += vectorDelta; } if (j < rows) { rii = rir + ((rows - j) * depth); for(Index i = 0; i < depth; i++) { Index k = j; for(; k < rows; k++) { std::complex v = getAdjointVal(k, i, lhs); blockAf[rir] = v.real(); blockAf[rii] = v.imag(); rir += 1; rii += 1; } } } } template EIGEN_STRONG_INLINE void symm_pack_rhs_helper(Scalar* blockB, const Scalar* _rhs, Index rhsStride, Index rows, Index cols, Index k2) { const Index depth = k2 + rows; const_blas_data_mapper rhs(_rhs, rhsStride); const Index vectorSize = quad_traits::vectorsize; Index ri = 0, j = 0; for(; j + N*vectorSize <= cols; j+=N*vectorSize) { Index i = k2; for(; i < depth; i++) { for(Index k = 0; k < N*vectorSize; k++) { if(i <= j+k) blockB[ri + k] = rhs(j+k, i); else blockB[ri + k] = rhs(i, j+k); } ri += N*vectorSize; } } if (j < cols) { for(Index i = k2; i < depth; i++) { Index k = j; for(; k < cols; k++) { if(k <= i) blockB[ri] = rhs(i, k); else blockB[ri] = rhs(k, i); ri += 1; } } } } template EIGEN_STRONG_INLINE void symm_pack_lhs_helper(Scalar* blockA, const Scalar* _lhs, Index lhsStride, Index cols, Index rows) { const Index depth = cols; const_blas_data_mapper lhs(_lhs, lhsStride); const Index vectorSize = quad_traits::vectorsize; Index ri = 0, j = 0; for(; j + vectorSize <= rows; j+=vectorSize) { Index i = 0; for(; i < depth; i++) { for(Index k = 0; k < vectorSize; k++) { if(i <= j+k) blockA[ri + k] = lhs(j+k, i); else blockA[ri + k] = lhs(i, j+k); } ri += vectorSize; } } if (j < rows) { for(Index i = 0; i < depth; i++) { Index k = j; for(; k < rows; k++) { if(i <= k) blockA[ri] = lhs(k, i); else blockA[ri] = lhs(i, k); ri += 1; } } } } template struct symm_pack_rhs, Index, nr, StorageOrder> { void operator()(std::complex* blockB, const std::complex* _rhs, Index rhsStride, Index rows, Index cols, Index k2) { symm_pack_complex_rhs_helper(blockB, _rhs, rhsStride, rows, cols, k2); } }; template struct symm_pack_lhs, Index, Pack1, Pack2_dummy, StorageOrder> { void operator()(std::complex* blockA, const std::complex* _lhs, Index lhsStride, Index cols, Index rows) { symm_pack_complex_lhs_helper(blockA, _lhs, lhsStride, cols, rows); } }; // *********** symm_pack std::complex *********** template struct symm_pack_rhs, Index, nr, StorageOrder> { void operator()(std::complex* blockB, const std::complex* _rhs, Index rhsStride, Index rows, Index cols, Index k2) { symm_pack_complex_rhs_helper(blockB, _rhs, rhsStride, rows, cols, k2); } }; template struct symm_pack_lhs, Index, Pack1, Pack2_dummy, StorageOrder> { void operator()(std::complex* blockA, const std::complex* _lhs, Index lhsStride, Index cols, Index rows) { symm_pack_complex_lhs_helper(blockA, _lhs, lhsStride, cols, rows); } }; // *********** symm_pack float32 *********** template struct symm_pack_rhs { void operator()(float* blockB, const float* _rhs, Index rhsStride, Index rows, Index cols, Index k2) { symm_pack_rhs_helper(blockB, _rhs, rhsStride, rows, cols, k2); } }; template struct symm_pack_lhs { void operator()(float* blockA, const float* _lhs, Index lhsStride, Index cols, Index rows) { symm_pack_lhs_helper(blockA, _lhs, lhsStride, cols, rows); } }; // *********** symm_pack float64 *********** template struct symm_pack_rhs { void operator()(double* blockB, const double* _rhs, Index rhsStride, Index rows, Index cols, Index k2) { symm_pack_rhs_helper(blockB, _rhs, rhsStride, rows, cols, k2); } }; template struct symm_pack_lhs { void operator()(double* blockA, const double* _lhs, Index lhsStride, Index cols, Index rows) { symm_pack_lhs_helper(blockA, _lhs, lhsStride, cols, rows); } }; /** * PanelMode * Packing might be called several times before being multiplied by gebp_kernel, this happens because * on special occasions it fills part of block with other parts of the matrix. Two variables control * how PanelMode should behave: offset and stride. The idea is that those variables represent whatever * is going to be the real offset and stride in the future and this is what you should obey. The process * is to behave as you would with normal packing but leave the start of each part with the correct offset * and the end as well respecting the real stride the block will have. Gebp is aware of both blocks stride * and offset and behaves accordingly. **/ template EIGEN_ALWAYS_INLINE void storeBlock(Scalar* to, PacketBlock& block) { const Index size = 16 / sizeof(Scalar); pstore(to + (0 * size), block.packet[0]); pstore(to + (1 * size), block.packet[1]); pstore(to + (2 * size), block.packet[2]); pstore(to + (3 * size), block.packet[3]); } template EIGEN_ALWAYS_INLINE void storeBlock(Scalar* to, PacketBlock& block) { const Index size = 16 / sizeof(Scalar); pstore(to + (0 * size), block.packet[0]); pstore(to + (1 * size), block.packet[1]); } // General template for lhs & rhs complex packing. template struct dhs_cpack { EIGEN_STRONG_INLINE void operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { const Index vectorSize = quad_traits::vectorsize; const Index vectorDelta = vectorSize * ((PanelMode) ? stride : depth); Index rir = ((PanelMode) ? (vectorSize*offset) : 0), rii; Scalar* blockAt = reinterpret_cast(blockA); Index j = 0; for(; j + vectorSize <= rows; j+=vectorSize) { Index i = 0; rii = rir + vectorDelta; for(; i + vectorSize <= depth; i+=vectorSize) { PacketBlock blockr, blocki; PacketBlock cblock; if (UseLhs) { bload(cblock, lhs, j, i); } else { bload(cblock, lhs, i, j); } blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[4].v, p16uc_GETREAL32); blockr.packet[1] = vec_perm(cblock.packet[1].v, cblock.packet[5].v, p16uc_GETREAL32); blockr.packet[2] = vec_perm(cblock.packet[2].v, cblock.packet[6].v, p16uc_GETREAL32); blockr.packet[3] = vec_perm(cblock.packet[3].v, cblock.packet[7].v, p16uc_GETREAL32); blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[4].v, p16uc_GETIMAG32); blocki.packet[1] = vec_perm(cblock.packet[1].v, cblock.packet[5].v, p16uc_GETIMAG32); blocki.packet[2] = vec_perm(cblock.packet[2].v, cblock.packet[6].v, p16uc_GETIMAG32); blocki.packet[3] = vec_perm(cblock.packet[3].v, cblock.packet[7].v, p16uc_GETIMAG32); if(Conjugate) { blocki.packet[0] = -blocki.packet[0]; blocki.packet[1] = -blocki.packet[1]; blocki.packet[2] = -blocki.packet[2]; blocki.packet[3] = -blocki.packet[3]; } if(((StorageOrder == RowMajor) && UseLhs) || (((StorageOrder == ColMajor) && !UseLhs))) { ptranspose(blockr); ptranspose(blocki); } storeBlock(blockAt + rir, blockr); storeBlock(blockAt + rii, blocki); rir += 4*vectorSize; rii += 4*vectorSize; } for(; i < depth; i++) { PacketBlock blockr, blocki; PacketBlock cblock; if(((StorageOrder == ColMajor) && UseLhs) || (((StorageOrder == RowMajor) && !UseLhs))) { if (UseLhs) { cblock.packet[0] = lhs.template loadPacket(j + 0, i); cblock.packet[1] = lhs.template loadPacket(j + 2, i); } else { cblock.packet[0] = lhs.template loadPacket(i, j + 0); cblock.packet[1] = lhs.template loadPacket(i, j + 2); } } else { std::complex lhs0, lhs1; if (UseLhs) { lhs0 = lhs(j + 0, i); lhs1 = lhs(j + 1, i); cblock.packet[0] = pload2(&lhs0, &lhs1); lhs0 = lhs(j + 2, i); lhs1 = lhs(j + 3, i); cblock.packet[1] = pload2(&lhs0, &lhs1); } else { lhs0 = lhs(i, j + 0); lhs1 = lhs(i, j + 1); cblock.packet[0] = pload2(&lhs0, &lhs1); lhs0 = lhs(i, j + 2); lhs1 = lhs(i, j + 3); cblock.packet[1] = pload2(&lhs0, &lhs1); } } blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETREAL32); blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETIMAG32); if(Conjugate) { blocki.packet[0] = -blocki.packet[0]; } pstore(blockAt + rir, blockr.packet[0]); pstore(blockAt + rii, blocki.packet[0]); rir += vectorSize; rii += vectorSize; } rir += ((PanelMode) ? (vectorSize*(2*stride - depth)) : vectorDelta); } if (j < rows) { if(PanelMode) rir += (offset*(rows - j - vectorSize)); rii = rir + (((PanelMode) ? stride : depth) * (rows - j)); for(Index i = 0; i < depth; i++) { Index k = j; for(; k < rows; k++) { if (UseLhs) { blockAt[rir] = lhs(k, i).real(); if(Conjugate) blockAt[rii] = -lhs(k, i).imag(); else blockAt[rii] = lhs(k, i).imag(); } else { blockAt[rir] = lhs(i, k).real(); if(Conjugate) blockAt[rii] = -lhs(i, k).imag(); else blockAt[rii] = lhs(i, k).imag(); } rir += 1; rii += 1; } } } } }; // General template for lhs & rhs packing. template struct dhs_pack{ EIGEN_STRONG_INLINE void operator()(Scalar* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { const Index vectorSize = quad_traits::vectorsize; Index ri = 0, j = 0; for(; j + vectorSize <= rows; j+=vectorSize) { Index i = 0; if(PanelMode) ri += vectorSize*offset; for(; i + vectorSize <= depth; i+=vectorSize) { PacketBlock block; if (UseLhs) { bload(block, lhs, j, i); } else { bload(block, lhs, i, j); } if(((StorageOrder == RowMajor) && UseLhs) || ((StorageOrder == ColMajor) && !UseLhs)) { ptranspose(block); } storeBlock(blockA + ri, block); ri += 4*vectorSize; } for(; i < depth; i++) { if(((StorageOrder == RowMajor) && UseLhs) || ((StorageOrder == ColMajor) && !UseLhs)) { if (UseLhs) { blockA[ri+0] = lhs(j+0, i); blockA[ri+1] = lhs(j+1, i); blockA[ri+2] = lhs(j+2, i); blockA[ri+3] = lhs(j+3, i); } else { blockA[ri+0] = lhs(i, j+0); blockA[ri+1] = lhs(i, j+1); blockA[ri+2] = lhs(i, j+2); blockA[ri+3] = lhs(i, j+3); } } else { Packet lhsV; if (UseLhs) { lhsV = lhs.template loadPacket(j, i); } else { lhsV = lhs.template loadPacket(i, j); } pstore(blockA + ri, lhsV); } ri += vectorSize; } if(PanelMode) ri += vectorSize*(stride - offset - depth); } if (j < rows) { if(PanelMode) ri += offset*(rows - j); for(Index i = 0; i < depth; i++) { Index k = j; for(; k < rows; k++) { if (UseLhs) { blockA[ri] = lhs(k, i); } else { blockA[ri] = lhs(i, k); } ri += 1; } } } } }; // General template for lhs packing, float64 specialization. template struct dhs_pack { EIGEN_STRONG_INLINE void operator()(double* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { const Index vectorSize = quad_traits::vectorsize; Index ri = 0, j = 0; for(; j + vectorSize <= rows; j+=vectorSize) { Index i = 0; if(PanelMode) ri += vectorSize*offset; for(; i + vectorSize <= depth; i+=vectorSize) { PacketBlock block; if(StorageOrder == RowMajor) { block.packet[0] = lhs.template loadPacket(j + 0, i); block.packet[1] = lhs.template loadPacket(j + 1, i); ptranspose(block); } else { block.packet[0] = lhs.template loadPacket(j, i + 0); block.packet[1] = lhs.template loadPacket(j, i + 1); } storeBlock(blockA + ri, block); ri += 2*vectorSize; } for(; i < depth; i++) { if(StorageOrder == RowMajor) { blockA[ri+0] = lhs(j+0, i); blockA[ri+1] = lhs(j+1, i); } else { Packet2d lhsV = lhs.template loadPacket(j, i); pstore(blockA + ri, lhsV); } ri += vectorSize; } if(PanelMode) ri += vectorSize*(stride - offset - depth); } if (j < rows) { if(PanelMode) ri += offset*(rows - j); for(Index i = 0; i < depth; i++) { Index k = j; for(; k < rows; k++) { blockA[ri] = lhs(k, i); ri += 1; } } } } }; // General template for rhs packing, float64 specialization. template struct dhs_pack { EIGEN_STRONG_INLINE void operator()(double* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { const Index vectorSize = quad_traits::vectorsize; Index ri = 0, j = 0; for(; j + 2*vectorSize <= cols; j+=2*vectorSize) { Index i = 0; if(PanelMode) ri += offset*(2*vectorSize); for(; i + vectorSize <= depth; i+=vectorSize) { PacketBlock block; if(StorageOrder == ColMajor) { PacketBlock block1, block2; block1.packet[0] = rhs.template loadPacket(i, j + 0); block1.packet[1] = rhs.template loadPacket(i, j + 1); block2.packet[0] = rhs.template loadPacket(i, j + 2); block2.packet[1] = rhs.template loadPacket(i, j + 3); ptranspose(block1); ptranspose(block2); pstore(blockB + ri , block1.packet[0]); pstore(blockB + ri + 2, block2.packet[0]); pstore(blockB + ri + 4, block1.packet[1]); pstore(blockB + ri + 6, block2.packet[1]); } else { block.packet[0] = rhs.template loadPacket(i + 0, j + 0); //[a1 a2] block.packet[1] = rhs.template loadPacket(i + 0, j + 2); //[a3 a4] block.packet[2] = rhs.template loadPacket(i + 1, j + 0); //[b1 b2] block.packet[3] = rhs.template loadPacket(i + 1, j + 2); //[b3 b4] storeBlock(blockB + ri, block); } ri += 4*vectorSize; } for(; i < depth; i++) { if(StorageOrder == ColMajor) { blockB[ri+0] = rhs(i, j+0); blockB[ri+1] = rhs(i, j+1); ri += vectorSize; blockB[ri+0] = rhs(i, j+2); blockB[ri+1] = rhs(i, j+3); } else { Packet2d rhsV = rhs.template loadPacket(i, j); pstore(blockB + ri, rhsV); ri += vectorSize; rhsV = rhs.template loadPacket(i, j + 2); pstore(blockB + ri, rhsV); } ri += vectorSize; } if(PanelMode) ri += (2*vectorSize)*(stride - offset - depth); } if (j < cols) { if(PanelMode) ri += offset*(cols - j); for(Index i = 0; i < depth; i++) { Index k = j; for(; k < cols; k++) { blockB[ri] = rhs(i, k); ri += 1; } } } } }; // General template for lhs complex packing, float64 specialization. template struct dhs_cpack { EIGEN_STRONG_INLINE void operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { const Index vectorSize = quad_traits::vectorsize; const Index vectorDelta = vectorSize * ((PanelMode) ? stride : depth); Index rir = ((PanelMode) ? (vectorSize*offset) : 0), rii; double* blockAt = reinterpret_cast(blockA); Index j = 0; for(; j + vectorSize <= rows; j+=vectorSize) { Index i = 0; rii = rir + vectorDelta; for(; i + vectorSize <= depth; i+=vectorSize) { PacketBlock blockr, blocki; PacketBlock cblock; if(StorageOrder == ColMajor) { cblock.packet[0] = lhs.template loadPacket(j, i + 0); //[a1 a1i] cblock.packet[1] = lhs.template loadPacket(j, i + 1); //[b1 b1i] cblock.packet[2] = lhs.template loadPacket(j + 1, i + 0); //[a2 a2i] cblock.packet[3] = lhs.template loadPacket(j + 1, i + 1); //[b2 b2i] blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[2].v, p16uc_GETREAL64); //[a1 a2] blockr.packet[1] = vec_perm(cblock.packet[1].v, cblock.packet[3].v, p16uc_GETREAL64); //[b1 b2] blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[2].v, p16uc_GETIMAG64); blocki.packet[1] = vec_perm(cblock.packet[1].v, cblock.packet[3].v, p16uc_GETIMAG64); } else { cblock.packet[0] = lhs.template loadPacket(j + 0, i); //[a1 a1i] cblock.packet[1] = lhs.template loadPacket(j + 1, i); //[a2 a2i] cblock.packet[2] = lhs.template loadPacket(j + 0, i + 1); //[b1 b1i] cblock.packet[3] = lhs.template loadPacket(j + 1, i + 1); //[b2 b2i blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETREAL64); //[a1 a2] blockr.packet[1] = vec_perm(cblock.packet[2].v, cblock.packet[3].v, p16uc_GETREAL64); //[b1 b2] blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETIMAG64); blocki.packet[1] = vec_perm(cblock.packet[2].v, cblock.packet[3].v, p16uc_GETIMAG64); } if(Conjugate) { blocki.packet[0] = -blocki.packet[0]; blocki.packet[1] = -blocki.packet[1]; } storeBlock(blockAt + rir, blockr); storeBlock(blockAt + rii, blocki); rir += 2*vectorSize; rii += 2*vectorSize; } for(; i < depth; i++) { PacketBlock blockr, blocki; PacketBlock cblock; cblock.packet[0] = lhs.template loadPacket(j + 0, i); cblock.packet[1] = lhs.template loadPacket(j + 1, i); blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETREAL64); blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETIMAG64); if(Conjugate) { blocki.packet[0] = -blocki.packet[0]; } pstore(blockAt + rir, blockr.packet[0]); pstore(blockAt + rii, blocki.packet[0]); rir += vectorSize; rii += vectorSize; } rir += ((PanelMode) ? (vectorSize*(2*stride - depth)) : vectorDelta); } if (j < rows) { if(PanelMode) rir += (offset*(rows - j - vectorSize)); rii = rir + (((PanelMode) ? stride : depth) * (rows - j)); for(Index i = 0; i < depth; i++) { Index k = j; for(; k < rows; k++) { blockAt[rir] = lhs(k, i).real(); if(Conjugate) blockAt[rii] = -lhs(k, i).imag(); else blockAt[rii] = lhs(k, i).imag(); rir += 1; rii += 1; } } } } }; // General template for rhs complex packing, float64 specialization. template struct dhs_cpack { EIGEN_STRONG_INLINE void operator()(std::complex* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { const Index vectorSize = quad_traits::vectorsize; const Index vectorDelta = 2*vectorSize * ((PanelMode) ? stride : depth); Index rir = ((PanelMode) ? (2*vectorSize*offset) : 0), rii; double* blockBt = reinterpret_cast(blockB); Index j = 0; for(; j + 2*vectorSize <= cols; j+=2*vectorSize) { Index i = 0; rii = rir + vectorDelta; for(; i < depth; i++) { PacketBlock cblock; PacketBlock blockr, blocki; bload(cblock, rhs, i, j); blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETREAL64); blockr.packet[1] = vec_perm(cblock.packet[2].v, cblock.packet[3].v, p16uc_GETREAL64); blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETIMAG64); blocki.packet[1] = vec_perm(cblock.packet[2].v, cblock.packet[3].v, p16uc_GETIMAG64); if(Conjugate) { blocki.packet[0] = -blocki.packet[0]; blocki.packet[1] = -blocki.packet[1]; } storeBlock(blockBt + rir, blockr); storeBlock(blockBt + rii, blocki); rir += 2*vectorSize; rii += 2*vectorSize; } rir += ((PanelMode) ? (2*vectorSize*(2*stride - depth)) : vectorDelta); } if (j < cols) { if(PanelMode) rir += (offset*(cols - j - 2*vectorSize)); rii = rir + (((PanelMode) ? stride : depth) * (cols - j)); for(Index i = 0; i < depth; i++) { Index k = j; for(; k < cols; k++) { blockBt[rir] = rhs(i, k).real(); if(Conjugate) blockBt[rii] = -rhs(i, k).imag(); else blockBt[rii] = rhs(i, k).imag(); rir += 1; rii += 1; } } } } }; /************** * GEMM utils * **************/ // 512-bits rank1-update of acc. It can either positive or negative accumulate (useful for complex gemm). template EIGEN_ALWAYS_INLINE void pger_common(PacketBlock* acc, const Packet& lhsV, const Packet* rhsV) { if(NegativeAccumulate) { acc->packet[0] = vec_nmsub(lhsV, rhsV[0], acc->packet[0]); acc->packet[1] = vec_nmsub(lhsV, rhsV[1], acc->packet[1]); acc->packet[2] = vec_nmsub(lhsV, rhsV[2], acc->packet[2]); acc->packet[3] = vec_nmsub(lhsV, rhsV[3], acc->packet[3]); } else { acc->packet[0] = vec_madd(lhsV, rhsV[0], acc->packet[0]); acc->packet[1] = vec_madd(lhsV, rhsV[1], acc->packet[1]); acc->packet[2] = vec_madd(lhsV, rhsV[2], acc->packet[2]); acc->packet[3] = vec_madd(lhsV, rhsV[3], acc->packet[3]); } } template EIGEN_ALWAYS_INLINE void pger_common(PacketBlock* acc, const Packet& lhsV, const Packet* rhsV) { if(NegativeAccumulate) { acc->packet[0] = vec_nmsub(lhsV, rhsV[0], acc->packet[0]); } else { acc->packet[0] = vec_madd(lhsV, rhsV[0], acc->packet[0]); } } template EIGEN_ALWAYS_INLINE void pger(PacketBlock* acc, const Scalar* lhs, const Packet* rhsV) { Packet lhsV = pload(lhs); pger_common(acc, lhsV, rhsV); } template EIGEN_ALWAYS_INLINE void loadPacketRemaining(const Scalar* lhs, Packet &lhsV, Index remaining_rows) { #ifdef _ARCH_PWR9 lhsV = vec_xl_len((Scalar *)lhs, remaining_rows * sizeof(Scalar)); #else Index i = 0; do { lhsV[i] = lhs[i]; } while (++i < remaining_rows); #endif } template EIGEN_ALWAYS_INLINE void pger(PacketBlock* acc, const Scalar* lhs, const Packet* rhsV, Index remaining_rows) { Packet lhsV; loadPacketRemaining(lhs, lhsV, remaining_rows); pger_common(acc, lhsV, rhsV); } // 512-bits rank1-update of complex acc. It takes decoupled accumulators as entries. It also takes cares of mixed types real * complex and complex * real. template EIGEN_ALWAYS_INLINE void pgerc_common(PacketBlock* accReal, PacketBlock* accImag, const Packet &lhsV, const Packet &lhsVi, const Packet* rhsV, const Packet* rhsVi) { pger_common(accReal, lhsV, rhsV); if(LhsIsReal) { pger_common(accImag, lhsV, rhsVi); EIGEN_UNUSED_VARIABLE(lhsVi); } else { if (!RhsIsReal) { pger_common(accReal, lhsVi, rhsVi); pger_common(accImag, lhsV, rhsVi); } else { EIGEN_UNUSED_VARIABLE(rhsVi); } pger_common(accImag, lhsVi, rhsV); } } template EIGEN_ALWAYS_INLINE void pgerc(PacketBlock* accReal, PacketBlock* accImag, const Scalar* lhs_ptr, const Scalar* lhs_ptr_imag, const Packet* rhsV, const Packet* rhsVi) { Packet lhsV = ploadLhs(lhs_ptr); Packet lhsVi; if(!LhsIsReal) lhsVi = ploadLhs(lhs_ptr_imag); else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); pgerc_common(accReal, accImag, lhsV, lhsVi, rhsV, rhsVi); } template EIGEN_ALWAYS_INLINE void loadPacketRemaining(const Scalar* lhs_ptr, const Scalar* lhs_ptr_imag, Packet &lhsV, Packet &lhsVi, Index remaining_rows) { #ifdef _ARCH_PWR9 lhsV = vec_xl_len((Scalar *)lhs_ptr, remaining_rows * sizeof(Scalar)); if(!LhsIsReal) lhsVi = vec_xl_len((Scalar *)lhs_ptr_imag, remaining_rows * sizeof(Scalar)); else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); #else Index i = 0; do { lhsV[i] = lhs_ptr[i]; if(!LhsIsReal) lhsVi[i] = lhs_ptr_imag[i]; } while (++i < remaining_rows); if(LhsIsReal) EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); #endif } template EIGEN_ALWAYS_INLINE void pgerc(PacketBlock* accReal, PacketBlock* accImag, const Scalar* lhs_ptr, const Scalar* lhs_ptr_imag, const Packet* rhsV, const Packet* rhsVi, Index remaining_rows) { Packet lhsV, lhsVi; loadPacketRemaining(lhs_ptr, lhs_ptr_imag, lhsV, lhsVi, remaining_rows); pgerc_common(accReal, accImag, lhsV, lhsVi, rhsV, rhsVi); } template EIGEN_ALWAYS_INLINE Packet ploadLhs(const Scalar* lhs) { return ploadu(lhs); } // Zero the accumulator on PacketBlock. template EIGEN_ALWAYS_INLINE void bsetzero(PacketBlock& acc) { acc.packet[0] = pset1((Scalar)0); acc.packet[1] = pset1((Scalar)0); acc.packet[2] = pset1((Scalar)0); acc.packet[3] = pset1((Scalar)0); } template EIGEN_ALWAYS_INLINE void bsetzero(PacketBlock& acc) { acc.packet[0] = pset1((Scalar)0); } // Scale the PacketBlock vectors by alpha. template EIGEN_ALWAYS_INLINE void bscale(PacketBlock& acc, PacketBlock& accZ, const Packet& pAlpha) { acc.packet[0] = pmadd(pAlpha, accZ.packet[0], acc.packet[0]); acc.packet[1] = pmadd(pAlpha, accZ.packet[1], acc.packet[1]); acc.packet[2] = pmadd(pAlpha, accZ.packet[2], acc.packet[2]); acc.packet[3] = pmadd(pAlpha, accZ.packet[3], acc.packet[3]); } template EIGEN_ALWAYS_INLINE void bscale(PacketBlock& acc, PacketBlock& accZ, const Packet& pAlpha) { acc.packet[0] = pmadd(pAlpha, accZ.packet[0], acc.packet[0]); } template EIGEN_ALWAYS_INLINE void bscalec_common(PacketBlock& acc, PacketBlock& accZ, const Packet& pAlpha) { acc.packet[0] = pmul(accZ.packet[0], pAlpha); acc.packet[1] = pmul(accZ.packet[1], pAlpha); acc.packet[2] = pmul(accZ.packet[2], pAlpha); acc.packet[3] = pmul(accZ.packet[3], pAlpha); } template EIGEN_ALWAYS_INLINE void bscalec_common(PacketBlock& acc, PacketBlock& accZ, const Packet& pAlpha) { acc.packet[0] = pmul(accZ.packet[0], pAlpha); } // Complex version of PacketBlock scaling. template EIGEN_ALWAYS_INLINE void bscalec(PacketBlock& aReal, PacketBlock& aImag, const Packet& bReal, const Packet& bImag, PacketBlock& cReal, PacketBlock& cImag) { bscalec_common(cReal, aReal, bReal); bscalec_common(cImag, aImag, bReal); pger_common(&cReal, bImag, aImag.packet); pger_common(&cImag, bImag, aReal.packet); } template EIGEN_ALWAYS_INLINE void band(PacketBlock& acc, const Packet& pMask) { acc.packet[0] = pand(acc.packet[0], pMask); acc.packet[1] = pand(acc.packet[1], pMask); acc.packet[2] = pand(acc.packet[2], pMask); acc.packet[3] = pand(acc.packet[3], pMask); } template EIGEN_ALWAYS_INLINE void bscalec(PacketBlock& aReal, PacketBlock& aImag, const Packet& bReal, const Packet& bImag, PacketBlock& cReal, PacketBlock& cImag, const Packet& pMask) { band(aReal, pMask); band(aImag, pMask); bscalec(aReal, aImag, bReal, bImag, cReal, cImag); } // Load a PacketBlock, the N parameters make tunning gemm easier so we can add more accumulators as needed. template EIGEN_ALWAYS_INLINE void bload(PacketBlock& acc, const DataMapper& res, Index row, Index col) { if (StorageOrder == RowMajor) { acc.packet[0] = res.template loadPacket(row + 0, col + N*accCols); acc.packet[1] = res.template loadPacket(row + 1, col + N*accCols); acc.packet[2] = res.template loadPacket(row + 2, col + N*accCols); acc.packet[3] = res.template loadPacket(row + 3, col + N*accCols); } else { acc.packet[0] = res.template loadPacket(row + N*accCols, col + 0); acc.packet[1] = res.template loadPacket(row + N*accCols, col + 1); acc.packet[2] = res.template loadPacket(row + N*accCols, col + 2); acc.packet[3] = res.template loadPacket(row + N*accCols, col + 3); } } // An overload of bload when you have a PacketBLock with 8 vectors. template EIGEN_ALWAYS_INLINE void bload(PacketBlock& acc, const DataMapper& res, Index row, Index col) { if (StorageOrder == RowMajor) { acc.packet[0] = res.template loadPacket(row + 0, col + N*accCols); acc.packet[1] = res.template loadPacket(row + 1, col + N*accCols); acc.packet[2] = res.template loadPacket(row + 2, col + N*accCols); acc.packet[3] = res.template loadPacket(row + 3, col + N*accCols); acc.packet[4] = res.template loadPacket(row + 0, col + (N+1)*accCols); acc.packet[5] = res.template loadPacket(row + 1, col + (N+1)*accCols); acc.packet[6] = res.template loadPacket(row + 2, col + (N+1)*accCols); acc.packet[7] = res.template loadPacket(row + 3, col + (N+1)*accCols); } else { acc.packet[0] = res.template loadPacket(row + N*accCols, col + 0); acc.packet[1] = res.template loadPacket(row + N*accCols, col + 1); acc.packet[2] = res.template loadPacket(row + N*accCols, col + 2); acc.packet[3] = res.template loadPacket(row + N*accCols, col + 3); acc.packet[4] = res.template loadPacket(row + (N+1)*accCols, col + 0); acc.packet[5] = res.template loadPacket(row + (N+1)*accCols, col + 1); acc.packet[6] = res.template loadPacket(row + (N+1)*accCols, col + 2); acc.packet[7] = res.template loadPacket(row + (N+1)*accCols, col + 3); } } template EIGEN_ALWAYS_INLINE void bload(PacketBlock& acc, const DataMapper& res, Index row, Index col) { acc.packet[0] = res.template loadPacket(row + N*accCols, col + 0); acc.packet[1] = res.template loadPacket(row + (N+1)*accCols, col + 0); } const static Packet4i mask41 = { -1, 0, 0, 0 }; const static Packet4i mask42 = { -1, -1, 0, 0 }; const static Packet4i mask43 = { -1, -1, -1, 0 }; const static Packet2l mask21 = { -1, 0 }; template EIGEN_ALWAYS_INLINE Packet bmask(const int remaining_rows) { if (remaining_rows == 0) { return pset1(float(0.0)); // Not used } else { switch (remaining_rows) { case 1: return Packet(mask41); case 2: return Packet(mask42); default: return Packet(mask43); } } } template<> EIGEN_ALWAYS_INLINE Packet2d bmask(const int remaining_rows) { if (remaining_rows == 0) { return pset1(double(0.0)); // Not used } else { return Packet2d(mask21); } } template EIGEN_ALWAYS_INLINE void bscale(PacketBlock& acc, PacketBlock& accZ, const Packet& pAlpha, const Packet& pMask) { band(accZ, pMask); bscale(acc, accZ, pAlpha); } template EIGEN_ALWAYS_INLINE void pbroadcast4_old(const __UNPACK_TYPE__(Packet)* a, Packet& a0, Packet& a1, Packet& a2, Packet& a3) { pbroadcast4(a, a0, a1, a2, a3); } template<> EIGEN_ALWAYS_INLINE void pbroadcast4_old(const double* a, Packet2d& a0, Packet2d& a1, Packet2d& a2, Packet2d& a3) { a1 = pload(a); a3 = pload(a + 2); a0 = vec_splat(a1, 0); a1 = vec_splat(a1, 1); a2 = vec_splat(a3, 0); a3 = vec_splat(a3, 1); } // PEEL loop factor. #define PEEL 7 template EIGEN_ALWAYS_INLINE void MICRO_EXTRA_COL( const Scalar* &lhs_ptr, const Scalar* &rhs_ptr, PacketBlock &accZero, Index remaining_rows, Index remaining_cols) { Packet rhsV[1]; rhsV[0] = pset1(rhs_ptr[0]); pger<1,Scalar, Packet, false>(&accZero, lhs_ptr, rhsV); lhs_ptr += remaining_rows; rhs_ptr += remaining_cols; } template EIGEN_STRONG_INLINE void gemm_extra_col( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index row, Index col, Index remaining_rows, Index remaining_cols, const Packet& pAlpha) { const Scalar* rhs_ptr = rhs_base; const Scalar* lhs_ptr = lhs_base + row*strideA + remaining_rows*offsetA; PacketBlock accZero; bsetzero(accZero); Index remaining_depth = (depth & -accRows); Index k = 0; for(; k + PEEL <= remaining_depth; k+= PEEL) { EIGEN_POWER_PREFETCH(rhs_ptr); EIGEN_POWER_PREFETCH(lhs_ptr); for (int l = 0; l < PEEL; l++) { MICRO_EXTRA_COL(lhs_ptr, rhs_ptr, accZero, remaining_rows, remaining_cols); } } for(; k < remaining_depth; k++) { MICRO_EXTRA_COL(lhs_ptr, rhs_ptr, accZero, remaining_rows, remaining_cols); } for(; k < depth; k++) { Packet rhsV[1]; rhsV[0] = pset1(rhs_ptr[0]); pger<1, Scalar, Packet, Index, false>(&accZero, lhs_ptr, rhsV, remaining_rows); lhs_ptr += remaining_rows; rhs_ptr += remaining_cols; } accZero.packet[0] = vec_mul(pAlpha, accZero.packet[0]); for(Index i = 0; i < remaining_rows; i++) { res(row + i, col) += accZero.packet[0][i]; } } template EIGEN_ALWAYS_INLINE void MICRO_EXTRA_ROW( const Scalar* &lhs_ptr, const Scalar* &rhs_ptr, PacketBlock &accZero, Index remaining_rows) { Packet rhsV[4]; pbroadcast4(rhs_ptr, rhsV[0], rhsV[1], rhsV[2], rhsV[3]); pger<4, Scalar, Packet, false>(&accZero, lhs_ptr, rhsV); lhs_ptr += remaining_rows; rhs_ptr += accRows; } template EIGEN_STRONG_INLINE void gemm_extra_row( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index row, Index col, Index rows, Index cols, Index remaining_rows, const Packet& pAlpha, const Packet& pMask) { const Scalar* rhs_ptr = rhs_base; const Scalar* lhs_ptr = lhs_base + row*strideA + remaining_rows*offsetA; PacketBlock accZero, acc; bsetzero(accZero); Index remaining_depth = (col + accRows < cols) ? depth : (depth & -accRows); Index k = 0; for(; k + PEEL <= remaining_depth; k+= PEEL) { EIGEN_POWER_PREFETCH(rhs_ptr); EIGEN_POWER_PREFETCH(lhs_ptr); for (int l = 0; l < PEEL; l++) { MICRO_EXTRA_ROW(lhs_ptr, rhs_ptr, accZero, remaining_rows); } } for(; k < remaining_depth; k++) { MICRO_EXTRA_ROW(lhs_ptr, rhs_ptr, accZero, remaining_rows); } if ((remaining_depth == depth) && (rows >= accCols)) { for(Index j = 0; j < 4; j++) { acc.packet[j] = res.template loadPacket(row, col + j); } bscale(acc, accZero, pAlpha, pMask); res.template storePacketBlock(row, col, acc); } else { for(; k < depth; k++) { Packet rhsV[4]; pbroadcast4(rhs_ptr, rhsV[0], rhsV[1], rhsV[2], rhsV[3]); pger<4, Scalar, Packet, Index, false>(&accZero, lhs_ptr, rhsV, remaining_rows); lhs_ptr += remaining_rows; rhs_ptr += accRows; } for(Index j = 0; j < 4; j++) { accZero.packet[j] = vec_mul(pAlpha, accZero.packet[j]); } for(Index j = 0; j < 4; j++) { for(Index i = 0; i < remaining_rows; i++) { res(row + i, col + j) += accZero.packet[j][i]; } } } } #define MICRO_UNROLL(func) \ func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7) #define MICRO_UNROLL_WORK(func, func2, peel) \ MICRO_UNROLL(func2); \ func(0,peel) func(1,peel) func(2,peel) func(3,peel) \ func(4,peel) func(5,peel) func(6,peel) func(7,peel) #define MICRO_LOAD_ONE(iter) \ if (unroll_factor > iter) { \ lhsV##iter = ploadLhs(lhs_ptr##iter); \ lhs_ptr##iter += accCols; \ } else { \ EIGEN_UNUSED_VARIABLE(lhsV##iter); \ } #define MICRO_WORK_ONE(iter, peel) \ if (unroll_factor > iter) { \ pger_common(&accZero##iter, lhsV##iter, rhsV##peel); \ } #define MICRO_TYPE_PEEL4(func, func2, peel) \ if (PEEL > peel) { \ Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \ pbroadcast4(rhs_ptr + (accRows * peel), rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \ MICRO_UNROLL_WORK(func, func2, peel) \ } else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ } #define MICRO_TYPE_PEEL1(func, func2, peel) \ if (PEEL > peel) { \ Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \ rhsV##peel[0] = pset1(rhs_ptr[remaining_cols * peel]); \ MICRO_UNROLL_WORK(func, func2, peel) \ } else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ } #define MICRO_UNROLL_TYPE_PEEL(M, func, func1, func2) \ Packet rhsV0[M], rhsV1[M], rhsV2[M], rhsV3[M], rhsV4[M], rhsV5[M], rhsV6[M], rhsV7[M], rhsV8[M], rhsV9[M]; \ func(func1,func2,0); func(func1,func2,1); \ func(func1,func2,2); func(func1,func2,3); \ func(func1,func2,4); func(func1,func2,5); \ func(func1,func2,6); func(func1,func2,7); \ func(func1,func2,8); func(func1,func2,9); #define MICRO_UNROLL_TYPE_ONE(M, func, func1, func2) \ Packet rhsV0[M]; \ func(func1,func2,0); #define MICRO_ONE_PEEL4 \ MICRO_UNROLL_TYPE_PEEL(4, MICRO_TYPE_PEEL4, MICRO_WORK_ONE, MICRO_LOAD_ONE); \ rhs_ptr += (accRows * PEEL); #define MICRO_ONE4 \ MICRO_UNROLL_TYPE_ONE(4, MICRO_TYPE_PEEL4, MICRO_WORK_ONE, MICRO_LOAD_ONE); \ rhs_ptr += accRows; #define MICRO_ONE_PEEL1 \ MICRO_UNROLL_TYPE_PEEL(1, MICRO_TYPE_PEEL1, MICRO_WORK_ONE, MICRO_LOAD_ONE); \ rhs_ptr += (remaining_cols * PEEL); #define MICRO_ONE1 \ MICRO_UNROLL_TYPE_ONE(1, MICRO_TYPE_PEEL1, MICRO_WORK_ONE, MICRO_LOAD_ONE); \ rhs_ptr += remaining_cols; #define MICRO_DST_PTR_ONE(iter) \ if (unroll_factor > iter) { \ bsetzero(accZero##iter); \ } else { \ EIGEN_UNUSED_VARIABLE(accZero##iter); \ } #define MICRO_DST_PTR MICRO_UNROLL(MICRO_DST_PTR_ONE) #define MICRO_SRC_PTR_ONE(iter) \ if (unroll_factor > iter) { \ lhs_ptr##iter = lhs_base + ( (row/accCols) + iter )*strideA*accCols + accCols*offsetA; \ } else { \ EIGEN_UNUSED_VARIABLE(lhs_ptr##iter); \ } #define MICRO_SRC_PTR MICRO_UNROLL(MICRO_SRC_PTR_ONE) #define MICRO_PREFETCH_ONE(iter) \ if (unroll_factor > iter) { \ EIGEN_POWER_PREFETCH(lhs_ptr##iter); \ } #define MICRO_PREFETCH MICRO_UNROLL(MICRO_PREFETCH_ONE) #define MICRO_STORE_ONE(iter) \ if (unroll_factor > iter) { \ acc.packet[0] = res.template loadPacket(row + iter*accCols, col + 0); \ acc.packet[1] = res.template loadPacket(row + iter*accCols, col + 1); \ acc.packet[2] = res.template loadPacket(row + iter*accCols, col + 2); \ acc.packet[3] = res.template loadPacket(row + iter*accCols, col + 3); \ bscale(acc, accZero##iter, pAlpha); \ res.template storePacketBlock(row + iter*accCols, col, acc); \ } #define MICRO_STORE MICRO_UNROLL(MICRO_STORE_ONE) #define MICRO_COL_STORE_ONE(iter) \ if (unroll_factor > iter) { \ acc.packet[0] = res.template loadPacket(row + iter*accCols, col + 0); \ bscale(acc, accZero##iter, pAlpha); \ res.template storePacketBlock(row + iter*accCols, col, acc); \ } #define MICRO_COL_STORE MICRO_UNROLL(MICRO_COL_STORE_ONE) template EIGEN_STRONG_INLINE void gemm_unrolled_iteration( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index& row, Index col, const Packet& pAlpha) { const Scalar* rhs_ptr = rhs_base; const Scalar* lhs_ptr0 = NULL, * lhs_ptr1 = NULL, * lhs_ptr2 = NULL, * lhs_ptr3 = NULL, * lhs_ptr4 = NULL, * lhs_ptr5 = NULL, * lhs_ptr6 = NULL, * lhs_ptr7 = NULL; PacketBlock accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7; PacketBlock acc; MICRO_SRC_PTR MICRO_DST_PTR Index k = 0; for(; k + PEEL <= depth; k+= PEEL) { EIGEN_POWER_PREFETCH(rhs_ptr); MICRO_PREFETCH MICRO_ONE_PEEL4 } for(; k < depth; k++) { MICRO_ONE4 } MICRO_STORE row += unroll_factor*accCols; } template EIGEN_STRONG_INLINE void gemm_unrolled_col_iteration( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index& row, Index col, Index remaining_cols, const Packet& pAlpha) { const Scalar* rhs_ptr = rhs_base; const Scalar* lhs_ptr0 = NULL, * lhs_ptr1 = NULL, * lhs_ptr2 = NULL, * lhs_ptr3 = NULL, * lhs_ptr4 = NULL, * lhs_ptr5 = NULL, * lhs_ptr6 = NULL, *lhs_ptr7 = NULL; PacketBlock accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7; PacketBlock acc; MICRO_SRC_PTR MICRO_DST_PTR Index k = 0; for(; k + PEEL <= depth; k+= PEEL) { EIGEN_POWER_PREFETCH(rhs_ptr); MICRO_PREFETCH MICRO_ONE_PEEL1 } for(; k < depth; k++) { MICRO_ONE1 } MICRO_COL_STORE row += unroll_factor*accCols; } template EIGEN_STRONG_INLINE void gemm_unrolled_col( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index& row, Index rows, Index col, Index remaining_cols, const Packet& pAlpha) { #define MAX_UNROLL 6 while(row + MAX_UNROLL*accCols <= rows) { gemm_unrolled_col_iteration(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_cols, pAlpha); } switch( (rows-row)/accCols ) { #if MAX_UNROLL > 7 case 7: gemm_unrolled_col_iteration<7, Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_cols, pAlpha); break; #endif #if MAX_UNROLL > 6 case 6: gemm_unrolled_col_iteration<6, Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_cols, pAlpha); break; #endif #if MAX_UNROLL > 5 case 5: gemm_unrolled_col_iteration<5, Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_cols, pAlpha); break; #endif #if MAX_UNROLL > 4 case 4: gemm_unrolled_col_iteration<4, Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_cols, pAlpha); break; #endif #if MAX_UNROLL > 3 case 3: gemm_unrolled_col_iteration<3, Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_cols, pAlpha); break; #endif #if MAX_UNROLL > 2 case 2: gemm_unrolled_col_iteration<2, Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_cols, pAlpha); break; #endif #if MAX_UNROLL > 1 case 1: gemm_unrolled_col_iteration<1, Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_cols, pAlpha); break; #endif default: break; } #undef MAX_UNROLL } /**************** * GEMM kernels * * **************/ template EIGEN_STRONG_INLINE void gemm(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { const Index remaining_rows = rows % accCols; const Index remaining_cols = cols % accRows; if( strideA == -1 ) strideA = depth; if( strideB == -1 ) strideB = depth; const Packet pAlpha = pset1(alpha); const Packet pMask = bmask((const int)(remaining_rows)); Index col = 0; for(; col + accRows <= cols; col += accRows) { const Scalar* rhs_base = blockB + col*strideB + accRows*offsetB; const Scalar* lhs_base = blockA; Index row = 0; #define MAX_UNROLL 6 while(row + MAX_UNROLL*accCols <= rows) { gemm_unrolled_iteration(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha); } switch( (rows-row)/accCols ) { #if MAX_UNROLL > 7 case 7: gemm_unrolled_iteration<7, Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha); break; #endif #if MAX_UNROLL > 6 case 6: gemm_unrolled_iteration<6, Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha); break; #endif #if MAX_UNROLL > 5 case 5: gemm_unrolled_iteration<5, Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha); break; #endif #if MAX_UNROLL > 4 case 4: gemm_unrolled_iteration<4, Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha); break; #endif #if MAX_UNROLL > 3 case 3: gemm_unrolled_iteration<3, Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha); break; #endif #if MAX_UNROLL > 2 case 2: gemm_unrolled_iteration<2, Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha); break; #endif #if MAX_UNROLL > 1 case 1: gemm_unrolled_iteration<1, Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha); break; #endif default: break; } #undef MAX_UNROLL if(remaining_rows > 0) { gemm_extra_row(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, rows, cols, remaining_rows, pAlpha, pMask); } } if(remaining_cols > 0) { const Scalar* rhs_base = blockB + col*strideB + remaining_cols*offsetB; const Scalar* lhs_base = blockA; for(; col < cols; col++) { Index row = 0; gemm_unrolled_col(res, lhs_base, rhs_base, depth, strideA, offsetA, row, rows, col, remaining_cols, pAlpha); if (remaining_rows > 0) { gemm_extra_col(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_rows, remaining_cols, pAlpha); } rhs_base++; } } } #define accColsC (accCols / 2) #define advanceRows ((LhsIsReal) ? 1 : 2) #define advanceCols ((RhsIsReal) ? 1 : 2) // PEEL_COMPLEX loop factor. #define PEEL_COMPLEX 3 template EIGEN_ALWAYS_INLINE void MICRO_COMPLEX_EXTRA_COL( const Scalar* &lhs_ptr_real, const Scalar* &lhs_ptr_imag, const Scalar* &rhs_ptr_real, const Scalar* &rhs_ptr_imag, PacketBlock &accReal, PacketBlock &accImag, Index remaining_rows, Index remaining_cols) { Packet rhsV[1], rhsVi[1]; rhsV[0] = pset1(rhs_ptr_real[0]); if(!RhsIsReal) rhsVi[0] = pset1(rhs_ptr_imag[0]); pgerc<1, Scalar, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal, &accImag, lhs_ptr_real, lhs_ptr_imag, rhsV, rhsVi); lhs_ptr_real += remaining_rows; if(!LhsIsReal) lhs_ptr_imag += remaining_rows; else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); rhs_ptr_real += remaining_cols; if(!RhsIsReal) rhs_ptr_imag += remaining_cols; else EIGEN_UNUSED_VARIABLE(rhs_ptr_imag); } template EIGEN_STRONG_INLINE void gemm_complex_extra_col( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index col, Index remaining_rows, Index remaining_cols, const Packet& pAlphaReal, const Packet& pAlphaImag) { const Scalar* rhs_ptr_real = rhs_base; const Scalar* rhs_ptr_imag; if(!RhsIsReal) rhs_ptr_imag = rhs_base + remaining_cols*strideB; else EIGEN_UNUSED_VARIABLE(rhs_ptr_imag); const Scalar* lhs_ptr_real = lhs_base + advanceRows*row*strideA + remaining_rows*offsetA; const Scalar* lhs_ptr_imag; if(!LhsIsReal) lhs_ptr_imag = lhs_ptr_real + remaining_rows*strideA; else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); PacketBlock accReal, accImag; PacketBlock taccReal, taccImag; PacketBlock acc0, acc1; bsetzero(accReal); bsetzero(accImag); Index remaining_depth = (depth & -accRows); Index k = 0; for(; k + PEEL_COMPLEX <= remaining_depth; k+= PEEL_COMPLEX) { EIGEN_POWER_PREFETCH(rhs_ptr_real); if(!RhsIsReal) { EIGEN_POWER_PREFETCH(rhs_ptr_imag); } EIGEN_POWER_PREFETCH(lhs_ptr_real); if(!LhsIsReal) { EIGEN_POWER_PREFETCH(lhs_ptr_imag); } for (int l = 0; l < PEEL_COMPLEX; l++) { MICRO_COMPLEX_EXTRA_COL(lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real, rhs_ptr_imag, accReal, accImag, remaining_rows, remaining_cols); } } for(; k < remaining_depth; k++) { MICRO_COMPLEX_EXTRA_COL(lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real, rhs_ptr_imag, accReal, accImag, remaining_rows, remaining_cols); } for(; k < depth; k++) { Packet rhsV[1], rhsVi[1]; rhsV[0] = pset1(rhs_ptr_real[0]); if(!RhsIsReal) rhsVi[0] = pset1(rhs_ptr_imag[0]); pgerc<1, Scalar, Packet, Index, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal, &accImag, lhs_ptr_real, lhs_ptr_imag, rhsV, rhsVi, remaining_rows); lhs_ptr_real += remaining_rows; if(!LhsIsReal) lhs_ptr_imag += remaining_rows; rhs_ptr_real += remaining_cols; if(!RhsIsReal) rhs_ptr_imag += remaining_cols; } bscalec(accReal, accImag, pAlphaReal, pAlphaImag, taccReal, taccImag); bcouple_common(taccReal, taccImag, acc0, acc1); if ((sizeof(Scalar) == sizeof(float)) && (remaining_rows == 1)) { res(row + 0, col + 0) += pfirst(acc0.packet[0]); } else { acc0.packet[0] += res.template loadPacket(row + 0, col + 0); res.template storePacketBlock(row + 0, col + 0, acc0); if(remaining_rows > accColsC) { res(row + accColsC, col + 0) += pfirst(acc1.packet[0]); } } } template EIGEN_ALWAYS_INLINE void MICRO_COMPLEX_EXTRA_ROW( const Scalar* &lhs_ptr_real, const Scalar* &lhs_ptr_imag, const Scalar* &rhs_ptr_real, const Scalar* &rhs_ptr_imag, PacketBlock &accReal, PacketBlock &accImag, Index remaining_rows) { Packet rhsV[4], rhsVi[4]; pbroadcast4_old(rhs_ptr_real, rhsV[0], rhsV[1], rhsV[2], rhsV[3]); if(!RhsIsReal) pbroadcast4_old(rhs_ptr_imag, rhsVi[0], rhsVi[1], rhsVi[2], rhsVi[3]); pgerc<4, Scalar, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal, &accImag, lhs_ptr_real, lhs_ptr_imag, rhsV, rhsVi); lhs_ptr_real += remaining_rows; if(!LhsIsReal) lhs_ptr_imag += remaining_rows; else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); rhs_ptr_real += accRows; if(!RhsIsReal) rhs_ptr_imag += accRows; else EIGEN_UNUSED_VARIABLE(rhs_ptr_imag); } template EIGEN_STRONG_INLINE void gemm_complex_extra_row( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index col, Index rows, Index cols, Index remaining_rows, const Packet& pAlphaReal, const Packet& pAlphaImag, const Packet& pMask) { const Scalar* rhs_ptr_real = rhs_base; const Scalar* rhs_ptr_imag; if(!RhsIsReal) rhs_ptr_imag = rhs_base + accRows*strideB; else EIGEN_UNUSED_VARIABLE(rhs_ptr_imag); const Scalar* lhs_ptr_real = lhs_base + advanceRows*row*strideA + remaining_rows*offsetA; const Scalar* lhs_ptr_imag; if(!LhsIsReal) lhs_ptr_imag = lhs_ptr_real + remaining_rows*strideA; else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); PacketBlock accReal, accImag; PacketBlock taccReal, taccImag; PacketBlock acc0, acc1; PacketBlock tRes; bsetzero(accReal); bsetzero(accImag); Index remaining_depth = (col + accRows < cols) ? depth : (depth & -accRows); Index k = 0; for(; k + PEEL_COMPLEX <= remaining_depth; k+= PEEL_COMPLEX) { EIGEN_POWER_PREFETCH(rhs_ptr_real); if(!RhsIsReal) { EIGEN_POWER_PREFETCH(rhs_ptr_imag); } EIGEN_POWER_PREFETCH(lhs_ptr_real); if(!LhsIsReal) { EIGEN_POWER_PREFETCH(lhs_ptr_imag); } for (int l = 0; l < PEEL_COMPLEX; l++) { MICRO_COMPLEX_EXTRA_ROW(lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real, rhs_ptr_imag, accReal, accImag, remaining_rows); } } for(; k < remaining_depth; k++) { MICRO_COMPLEX_EXTRA_ROW(lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real, rhs_ptr_imag, accReal, accImag, remaining_rows); } if ((remaining_depth == depth) && (rows >= accCols)) { bload(tRes, res, row, col); bscalec(accReal, accImag, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask); bcouple(taccReal, taccImag, tRes, acc0, acc1); res.template storePacketBlock(row + 0, col, acc0); res.template storePacketBlock(row + accColsC, col, acc1); } else { for(; k < depth; k++) { Packet rhsV[4], rhsVi[4]; pbroadcast4_old(rhs_ptr_real, rhsV[0], rhsV[1], rhsV[2], rhsV[3]); if(!RhsIsReal) pbroadcast4_old(rhs_ptr_imag, rhsVi[0], rhsVi[1], rhsVi[2], rhsVi[3]); pgerc<4, Scalar, Packet, Index, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal, &accImag, lhs_ptr_real, lhs_ptr_imag, rhsV, rhsVi, remaining_rows); lhs_ptr_real += remaining_rows; if(!LhsIsReal) lhs_ptr_imag += remaining_rows; rhs_ptr_real += accRows; if(!RhsIsReal) rhs_ptr_imag += accRows; } bscalec(accReal, accImag, pAlphaReal, pAlphaImag, taccReal, taccImag); bcouple_common(taccReal, taccImag, acc0, acc1); if ((sizeof(Scalar) == sizeof(float)) && (remaining_rows == 1)) { for(Index j = 0; j < 4; j++) { res(row + 0, col + j) += pfirst(acc0.packet[j]); } } else { for(Index j = 0; j < 4; j++) { PacketBlock acc2; acc2.packet[0] = res.template loadPacket(row + 0, col + j) + acc0.packet[j]; res.template storePacketBlock(row + 0, col + j, acc2); if(remaining_rows > accColsC) { res(row + accColsC, col + j) += pfirst(acc1.packet[j]); } } } } } #define MICRO_COMPLEX_UNROLL(func) \ func(0) func(1) func(2) func(3) func(4) #define MICRO_COMPLEX_UNROLL_WORK(func, func2, peel) \ MICRO_COMPLEX_UNROLL(func2); \ func(0,peel) func(1,peel) func(2,peel) func(3,peel) func(4,peel) #define MICRO_COMPLEX_LOAD_ONE(iter) \ if (unroll_factor > iter) { \ lhsV##iter = ploadLhs(lhs_ptr_real##iter); \ lhs_ptr_real##iter += accCols; \ if(!LhsIsReal) { \ lhsVi##iter = ploadLhs(lhs_ptr_imag##iter); \ lhs_ptr_imag##iter += accCols; \ } else { \ EIGEN_UNUSED_VARIABLE(lhsVi##iter); \ } \ } else { \ EIGEN_UNUSED_VARIABLE(lhsV##iter); \ EIGEN_UNUSED_VARIABLE(lhsVi##iter); \ } #define MICRO_COMPLEX_WORK_ONE4(iter, peel) \ if (unroll_factor > iter) { \ pgerc_common<4, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal##iter, &accImag##iter, lhsV##iter, lhsVi##iter, rhsV##peel, rhsVi##peel); \ } #define MICRO_COMPLEX_WORK_ONE1(iter, peel) \ if (unroll_factor > iter) { \ pgerc_common<1, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal##iter, &accImag##iter, lhsV##iter, lhsVi##iter, rhsV##peel, rhsVi##peel); \ } #define MICRO_COMPLEX_TYPE_PEEL4(func, func2, peel) \ if (PEEL_COMPLEX > peel) { \ Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4; \ Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3, lhsVi4; \ pbroadcast4_old(rhs_ptr_real + (accRows * peel), rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \ if(!RhsIsReal) { \ pbroadcast4_old(rhs_ptr_imag + (accRows * peel), rhsVi##peel[0], rhsVi##peel[1], rhsVi##peel[2], rhsVi##peel[3]); \ } else { \ EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ } \ MICRO_COMPLEX_UNROLL_WORK(func, func2, peel) \ } else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ } #define MICRO_COMPLEX_TYPE_PEEL1(func, func2, peel) \ if (PEEL_COMPLEX > peel) { \ Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4; \ Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3, lhsVi4; \ rhsV##peel[0] = pset1(rhs_ptr_real[remaining_cols * peel]); \ if(!RhsIsReal) { \ rhsVi##peel[0] = pset1(rhs_ptr_imag[remaining_cols * peel]); \ } else { \ EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ } \ MICRO_COMPLEX_UNROLL_WORK(func, func2, peel) \ } else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ } #define MICRO_COMPLEX_UNROLL_TYPE_PEEL(M, func, func1, func2) \ Packet rhsV0[M], rhsV1[M], rhsV2[M], rhsV3[M], rhsV4[M], rhsV5[M], rhsV6[M], rhsV7[M], rhsV8[M], rhsV9[M]; \ Packet rhsVi0[M], rhsVi1[M], rhsVi2[M], rhsVi3[M], rhsVi4[M], rhsVi5[M], rhsVi6[M], rhsVi7[M], rhsVi8[M], rhsVi9[M]; \ func(func1,func2,0); func(func1,func2,1); \ func(func1,func2,2); func(func1,func2,3); \ func(func1,func2,4); func(func1,func2,5); \ func(func1,func2,6); func(func1,func2,7); \ func(func1,func2,8); func(func1,func2,9); #define MICRO_COMPLEX_UNROLL_TYPE_ONE(M, func, func1, func2) \ Packet rhsV0[M], rhsVi0[M];\ func(func1,func2,0); #define MICRO_COMPLEX_ONE_PEEL4 \ MICRO_COMPLEX_UNROLL_TYPE_PEEL(4, MICRO_COMPLEX_TYPE_PEEL4, MICRO_COMPLEX_WORK_ONE4, MICRO_COMPLEX_LOAD_ONE); \ rhs_ptr_real += (accRows * PEEL_COMPLEX); \ if(!RhsIsReal) rhs_ptr_imag += (accRows * PEEL_COMPLEX); #define MICRO_COMPLEX_ONE4 \ MICRO_COMPLEX_UNROLL_TYPE_ONE(4, MICRO_COMPLEX_TYPE_PEEL4, MICRO_COMPLEX_WORK_ONE4, MICRO_COMPLEX_LOAD_ONE); \ rhs_ptr_real += accRows; \ if(!RhsIsReal) rhs_ptr_imag += accRows; #define MICRO_COMPLEX_ONE_PEEL1 \ MICRO_COMPLEX_UNROLL_TYPE_PEEL(1, MICRO_COMPLEX_TYPE_PEEL1, MICRO_COMPLEX_WORK_ONE1, MICRO_COMPLEX_LOAD_ONE); \ rhs_ptr_real += (remaining_cols * PEEL_COMPLEX); \ if(!RhsIsReal) rhs_ptr_imag += (remaining_cols * PEEL_COMPLEX); #define MICRO_COMPLEX_ONE1 \ MICRO_COMPLEX_UNROLL_TYPE_ONE(1, MICRO_COMPLEX_TYPE_PEEL1, MICRO_COMPLEX_WORK_ONE1, MICRO_COMPLEX_LOAD_ONE); \ rhs_ptr_real += remaining_cols; \ if(!RhsIsReal) rhs_ptr_imag += remaining_cols; #define MICRO_COMPLEX_DST_PTR_ONE(iter) \ if (unroll_factor > iter) { \ bsetzero(accReal##iter); \ bsetzero(accImag##iter); \ } else { \ EIGEN_UNUSED_VARIABLE(accReal##iter); \ EIGEN_UNUSED_VARIABLE(accImag##iter); \ } #define MICRO_COMPLEX_DST_PTR MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_DST_PTR_ONE) #define MICRO_COMPLEX_SRC_PTR_ONE(iter) \ if (unroll_factor > iter) { \ lhs_ptr_real##iter = lhs_base + ( ((advanceRows*row)/accCols) + iter*advanceRows )*strideA*accCols + accCols*offsetA; \ if(!LhsIsReal) { \ lhs_ptr_imag##iter = lhs_ptr_real##iter + accCols*strideA; \ } else { \ EIGEN_UNUSED_VARIABLE(lhs_ptr_imag##iter); \ } \ } else { \ EIGEN_UNUSED_VARIABLE(lhs_ptr_real##iter); \ EIGEN_UNUSED_VARIABLE(lhs_ptr_imag##iter); \ } #define MICRO_COMPLEX_SRC_PTR MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_SRC_PTR_ONE) #define MICRO_COMPLEX_PREFETCH_ONE(iter) \ if (unroll_factor > iter) { \ EIGEN_POWER_PREFETCH(lhs_ptr_real##iter); \ if(!LhsIsReal) { \ EIGEN_POWER_PREFETCH(lhs_ptr_imag##iter); \ } \ } #define MICRO_COMPLEX_PREFETCH MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_PREFETCH_ONE) #define MICRO_COMPLEX_STORE_ONE(iter) \ if (unroll_factor > iter) { \ bload(tRes, res, row + iter*accCols, col); \ bscalec(accReal##iter, accImag##iter, pAlphaReal, pAlphaImag, taccReal, taccImag); \ bcouple(taccReal, taccImag, tRes, acc0, acc1); \ res.template storePacketBlock(row + iter*accCols + 0, col, acc0); \ res.template storePacketBlock(row + iter*accCols + accColsC, col, acc1); \ } #define MICRO_COMPLEX_STORE MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_STORE_ONE) #define MICRO_COMPLEX_COL_STORE_ONE(iter) \ if (unroll_factor > iter) { \ bload(tRes, res, row + iter*accCols, col); \ bscalec(accReal##iter, accImag##iter, pAlphaReal, pAlphaImag, taccReal, taccImag); \ bcouple(taccReal, taccImag, tRes, acc0, acc1); \ res.template storePacketBlock(row + iter*accCols + 0, col, acc0); \ res.template storePacketBlock(row + iter*accCols + accColsC, col, acc1); \ } #define MICRO_COMPLEX_COL_STORE MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_COL_STORE_ONE) template EIGEN_STRONG_INLINE void gemm_complex_unrolled_iteration( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index& row, Index col, const Packet& pAlphaReal, const Packet& pAlphaImag) { const Scalar* rhs_ptr_real = rhs_base; const Scalar* rhs_ptr_imag; if(!RhsIsReal) { rhs_ptr_imag = rhs_base + accRows*strideB; } else { EIGEN_UNUSED_VARIABLE(rhs_ptr_imag); } const Scalar* lhs_ptr_real0 = NULL, * lhs_ptr_imag0 = NULL, * lhs_ptr_real1 = NULL, * lhs_ptr_imag1 = NULL; const Scalar* lhs_ptr_real2 = NULL, * lhs_ptr_imag2 = NULL, * lhs_ptr_real3 = NULL, * lhs_ptr_imag3 = NULL; const Scalar* lhs_ptr_real4 = NULL, * lhs_ptr_imag4 = NULL; PacketBlock accReal0, accImag0, accReal1, accImag1; PacketBlock accReal2, accImag2, accReal3, accImag3; PacketBlock accReal4, accImag4; PacketBlock taccReal, taccImag; PacketBlock acc0, acc1; PacketBlock tRes; MICRO_COMPLEX_SRC_PTR MICRO_COMPLEX_DST_PTR Index k = 0; for(; k + PEEL_COMPLEX <= depth; k+= PEEL_COMPLEX) { EIGEN_POWER_PREFETCH(rhs_ptr_real); if(!RhsIsReal) { EIGEN_POWER_PREFETCH(rhs_ptr_imag); } MICRO_COMPLEX_PREFETCH MICRO_COMPLEX_ONE_PEEL4 } for(; k < depth; k++) { MICRO_COMPLEX_ONE4 } MICRO_COMPLEX_STORE row += unroll_factor*accCols; } template EIGEN_STRONG_INLINE void gemm_complex_unrolled_col_iteration( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index& row, Index col, Index remaining_cols, const Packet& pAlphaReal, const Packet& pAlphaImag) { const Scalar* rhs_ptr_real = rhs_base; const Scalar* rhs_ptr_imag; if(!RhsIsReal) { rhs_ptr_imag = rhs_base + remaining_cols*strideB; } else { EIGEN_UNUSED_VARIABLE(rhs_ptr_imag); } const Scalar* lhs_ptr_real0 = NULL, * lhs_ptr_imag0 = NULL, * lhs_ptr_real1 = NULL, * lhs_ptr_imag1 = NULL; const Scalar* lhs_ptr_real2 = NULL, * lhs_ptr_imag2 = NULL, * lhs_ptr_real3 = NULL, * lhs_ptr_imag3 = NULL; const Scalar* lhs_ptr_real4 = NULL, * lhs_ptr_imag4 = NULL; PacketBlock accReal0, accImag0, accReal1, accImag1; PacketBlock accReal2, accImag2, accReal3, accImag3; PacketBlock accReal4, accImag4; PacketBlock taccReal, taccImag; PacketBlock acc0, acc1; PacketBlock tRes; MICRO_COMPLEX_SRC_PTR MICRO_COMPLEX_DST_PTR Index k = 0; for(; k + PEEL_COMPLEX <= depth; k+= PEEL_COMPLEX) { EIGEN_POWER_PREFETCH(rhs_ptr_real); if(!RhsIsReal) { EIGEN_POWER_PREFETCH(rhs_ptr_imag); } MICRO_COMPLEX_PREFETCH MICRO_COMPLEX_ONE_PEEL1 } for(; k < depth; k++) { MICRO_COMPLEX_ONE1 } MICRO_COMPLEX_COL_STORE row += unroll_factor*accCols; } template EIGEN_STRONG_INLINE void gemm_complex_unrolled_col( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index& row, Index rows, Index col, Index remaining_cols, const Packet& pAlphaReal, const Packet& pAlphaImag) { #define MAX_COMPLEX_UNROLL 3 while(row + MAX_COMPLEX_UNROLL*accCols <= rows) { gemm_complex_unrolled_col_iteration(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, remaining_cols, pAlphaReal, pAlphaImag); } switch( (rows-row)/accCols ) { #if MAX_COMPLEX_UNROLL > 4 case 4: gemm_complex_unrolled_col_iteration<4, Scalar, Packet, Packetc, DataMapper, Index, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, remaining_cols, pAlphaReal, pAlphaImag); break; #endif #if MAX_COMPLEX_UNROLL > 3 case 3: gemm_complex_unrolled_col_iteration<3, Scalar, Packet, Packetc, DataMapper, Index, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, remaining_cols, pAlphaReal, pAlphaImag); break; #endif #if MAX_COMPLEX_UNROLL > 2 case 2: gemm_complex_unrolled_col_iteration<2, Scalar, Packet, Packetc, DataMapper, Index, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, remaining_cols, pAlphaReal, pAlphaImag); break; #endif #if MAX_COMPLEX_UNROLL > 1 case 1: gemm_complex_unrolled_col_iteration<1, Scalar, Packet, Packetc, DataMapper, Index, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, remaining_cols, pAlphaReal, pAlphaImag); break; #endif default: break; } #undef MAX_COMPLEX_UNROLL } template EIGEN_STRONG_INLINE void gemm_complex(const DataMapper& res, const LhsScalar* blockAc, const RhsScalar* blockBc, Index rows, Index depth, Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { const Index remaining_rows = rows % accCols; const Index remaining_cols = cols % accRows; if( strideA == -1 ) strideA = depth; if( strideB == -1 ) strideB = depth; const Packet pAlphaReal = pset1(alpha.real()); const Packet pAlphaImag = pset1(alpha.imag()); const Packet pMask = bmask((const int)(remaining_rows)); const Scalar* blockA = (Scalar *) blockAc; const Scalar* blockB = (Scalar *) blockBc; Index col = 0; for(; col + accRows <= cols; col += accRows) { const Scalar* rhs_base = blockB + advanceCols*col*strideB + accRows*offsetB; const Scalar* lhs_base = blockA; Index row = 0; #define MAX_COMPLEX_UNROLL 3 while(row + MAX_COMPLEX_UNROLL*accCols <= rows) { gemm_complex_unrolled_iteration(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag); } switch( (rows-row)/accCols ) { #if MAX_COMPLEX_UNROLL > 4 case 4: gemm_complex_unrolled_iteration<4, Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag); break; #endif #if MAX_COMPLEX_UNROLL > 3 case 3: gemm_complex_unrolled_iteration<3, Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag); break; #endif #if MAX_COMPLEX_UNROLL > 2 case 2: gemm_complex_unrolled_iteration<2, Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag); break; #endif #if MAX_COMPLEX_UNROLL > 1 case 1: gemm_complex_unrolled_iteration<1, Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag); break; #endif default: break; } #undef MAX_COMPLEX_UNROLL if(remaining_rows > 0) { gemm_complex_extra_row(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, rows, cols, remaining_rows, pAlphaReal, pAlphaImag, pMask); } } if(remaining_cols > 0) { const Scalar* rhs_base = blockB + advanceCols*col*strideB + remaining_cols*offsetB; const Scalar* lhs_base = blockA; for(; col < cols; col++) { Index row = 0; gemm_complex_unrolled_col(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, rows, col, remaining_cols, pAlphaReal, pAlphaImag); if (remaining_rows > 0) { gemm_complex_extra_col(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, remaining_rows, remaining_cols, pAlphaReal, pAlphaImag); } rhs_base++; } } } #undef accColsC #undef advanceCols #undef advanceRows /************************************ * ppc64le template specializations * * **********************************/ template struct gemm_pack_lhs { void operator()(double* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0); }; template void gemm_pack_lhs ::operator()(double* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { dhs_pack pack; pack(blockA, lhs, depth, rows, stride, offset); } template struct gemm_pack_lhs { void operator()(double* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0); }; template void gemm_pack_lhs ::operator()(double* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { dhs_pack pack; pack(blockA, lhs, depth, rows, stride, offset); } #if EIGEN_ALTIVEC_USE_CUSTOM_PACK template struct gemm_pack_rhs { void operator()(double* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0); }; template void gemm_pack_rhs ::operator()(double* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { dhs_pack pack; pack(blockB, rhs, depth, cols, stride, offset); } template struct gemm_pack_rhs { void operator()(double* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0); }; template void gemm_pack_rhs ::operator()(double* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { dhs_pack pack; pack(blockB, rhs, depth, cols, stride, offset); } #endif template struct gemm_pack_lhs { void operator()(float* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0); }; template void gemm_pack_lhs ::operator()(float* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { dhs_pack pack; pack(blockA, lhs, depth, rows, stride, offset); } template struct gemm_pack_lhs { void operator()(float* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0); }; template void gemm_pack_lhs ::operator()(float* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { dhs_pack pack; pack(blockA, lhs, depth, rows, stride, offset); } template struct gemm_pack_lhs, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode> { void operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0); }; template void gemm_pack_lhs, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode> ::operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { dhs_cpack pack; pack(blockA, lhs, depth, rows, stride, offset); } template struct gemm_pack_lhs, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode> { void operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0); }; template void gemm_pack_lhs, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode> ::operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { dhs_cpack pack; pack(blockA, lhs, depth, rows, stride, offset); } #if EIGEN_ALTIVEC_USE_CUSTOM_PACK template struct gemm_pack_rhs { void operator()(float* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0); }; template void gemm_pack_rhs ::operator()(float* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { dhs_pack pack; pack(blockB, rhs, depth, cols, stride, offset); } template struct gemm_pack_rhs { void operator()(float* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0); }; template void gemm_pack_rhs ::operator()(float* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { dhs_pack pack; pack(blockB, rhs, depth, cols, stride, offset); } #endif template struct gemm_pack_rhs, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode> { void operator()(std::complex* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0); }; template void gemm_pack_rhs, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode> ::operator()(std::complex* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { dhs_cpack pack; pack(blockB, rhs, depth, cols, stride, offset); } template struct gemm_pack_rhs, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode> { void operator()(std::complex* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0); }; template void gemm_pack_rhs, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode> ::operator()(std::complex* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { dhs_cpack pack; pack(blockB, rhs, depth, cols, stride, offset); } template struct gemm_pack_lhs, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode> { void operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0); }; template void gemm_pack_lhs, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode> ::operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { dhs_cpack pack; pack(blockA, lhs, depth, rows, stride, offset); } template struct gemm_pack_lhs, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode> { void operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0); }; template void gemm_pack_lhs, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode> ::operator()(std::complex* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) { dhs_cpack pack; pack(blockA, lhs, depth, rows, stride, offset); } template struct gemm_pack_rhs, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode> { void operator()(std::complex* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0); }; template void gemm_pack_rhs, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode> ::operator()(std::complex* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { dhs_cpack pack; pack(blockB, rhs, depth, cols, stride, offset); } template struct gemm_pack_rhs, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode> { void operator()(std::complex* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0); }; template void gemm_pack_rhs, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode> ::operator()(std::complex* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { dhs_cpack pack; pack(blockB, rhs, depth, cols, stride, offset); } // ********* gebp specializations ********* template struct gebp_kernel { typedef typename quad_traits::vectortype Packet; typedef typename quad_traits::rhstype RhsPacket; void operator()(const DataMapper& res, const float* blockA, const float* blockB, Index rows, Index depth, Index cols, float alpha, Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0); }; template void gebp_kernel ::operator()(const DataMapper& res, const float* blockA, const float* blockB, Index rows, Index depth, Index cols, float alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { const Index accRows = quad_traits::rows; const Index accCols = quad_traits::size; void (*gemm_function)(const DataMapper&, const float*, const float*, Index, Index, Index, float, Index, Index, Index, Index); #ifdef EIGEN_ALTIVEC_MMA_ONLY //generate with MMA only gemm_function = &Eigen::internal::gemmMMA; #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma")){ gemm_function = &Eigen::internal::gemmMMA; } else{ gemm_function = &Eigen::internal::gemm; } #else gemm_function = &Eigen::internal::gemm; #endif gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); } template struct gebp_kernel, std::complex, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> { typedef Packet4f Packet; typedef Packet2cf Packetc; typedef Packet4f RhsPacket; void operator()(const DataMapper& res, const std::complex* blockA, const std::complex* blockB, Index rows, Index depth, Index cols, std::complex alpha, Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0); }; template void gebp_kernel, std::complex, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> ::operator()(const DataMapper& res, const std::complex* blockA, const std::complex* blockB, Index rows, Index depth, Index cols, std::complex alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { const Index accRows = quad_traits::rows; const Index accCols = quad_traits::size; void (*gemm_function)(const DataMapper&, const std::complex*, const std::complex*, Index, Index, Index, std::complex, Index, Index, Index, Index); #ifdef EIGEN_ALTIVEC_MMA_ONLY //generate with MMA only gemm_function = &Eigen::internal::gemm_complexMMA, std::complex, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma")){ gemm_function = &Eigen::internal::gemm_complexMMA, std::complex, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; } else{ gemm_function = &Eigen::internal::gemm_complex, std::complex, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; } #else gemm_function = &Eigen::internal::gemm_complex, std::complex, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; #endif gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); } template struct gebp_kernel, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> { typedef Packet4f Packet; typedef Packet2cf Packetc; typedef Packet4f RhsPacket; void operator()(const DataMapper& res, const float* blockA, const std::complex* blockB, Index rows, Index depth, Index cols, std::complex alpha, Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0); }; template void gebp_kernel, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> ::operator()(const DataMapper& res, const float* blockA, const std::complex* blockB, Index rows, Index depth, Index cols, std::complex alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { const Index accRows = quad_traits::rows; const Index accCols = quad_traits::size; void (*gemm_function)(const DataMapper&, const float*, const std::complex*, Index, Index, Index, std::complex, Index, Index, Index, Index); #ifdef EIGEN_ALTIVEC_MMA_ONLY //generate with MMA only gemm_function = &Eigen::internal::gemm_complexMMA, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma")){ gemm_function = &Eigen::internal::gemm_complexMMA, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; } else{ gemm_function = &Eigen::internal::gemm_complex, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; } #else gemm_function = &Eigen::internal::gemm_complex, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; #endif gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); } template struct gebp_kernel, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> { typedef Packet4f Packet; typedef Packet2cf Packetc; typedef Packet4f RhsPacket; void operator()(const DataMapper& res, const std::complex* blockA, const float* blockB, Index rows, Index depth, Index cols, std::complex alpha, Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0); }; template void gebp_kernel, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> ::operator()(const DataMapper& res, const std::complex* blockA, const float* blockB, Index rows, Index depth, Index cols, std::complex alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { const Index accRows = quad_traits::rows; const Index accCols = quad_traits::size; void (*gemm_function)(const DataMapper&, const std::complex*, const float*, Index, Index, Index, std::complex, Index, Index, Index, Index); #ifdef EIGEN_ALTIVEC_MMA_ONLY //generate with MMA only gemm_function = &Eigen::internal::gemm_complexMMA, float, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma")){ gemm_function = &Eigen::internal::gemm_complexMMA, float, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; } else{ gemm_function = &Eigen::internal::gemm_complex, float, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; } #else gemm_function = &Eigen::internal::gemm_complex, float, std::complex, float, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; #endif gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); } template struct gebp_kernel { typedef typename quad_traits::vectortype Packet; typedef typename quad_traits::rhstype RhsPacket; void operator()(const DataMapper& res, const double* blockA, const double* blockB, Index rows, Index depth, Index cols, double alpha, Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0); }; template void gebp_kernel ::operator()(const DataMapper& res, const double* blockA, const double* blockB, Index rows, Index depth, Index cols, double alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { const Index accRows = quad_traits::rows; const Index accCols = quad_traits::size; void (*gemm_function)(const DataMapper&, const double*, const double*, Index, Index, Index, double, Index, Index, Index, Index); #ifdef EIGEN_ALTIVEC_MMA_ONLY //generate with MMA only gemm_function = &Eigen::internal::gemmMMA; #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma")){ gemm_function = &Eigen::internal::gemmMMA; } else{ gemm_function = &Eigen::internal::gemm; } #else gemm_function = &Eigen::internal::gemm; #endif gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); } template struct gebp_kernel, std::complex, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> { typedef quad_traits::vectortype Packet; typedef Packet1cd Packetc; typedef quad_traits::rhstype RhsPacket; void operator()(const DataMapper& res, const std::complex* blockA, const std::complex* blockB, Index rows, Index depth, Index cols, std::complex alpha, Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0); }; template void gebp_kernel, std::complex, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> ::operator()(const DataMapper& res, const std::complex* blockA, const std::complex* blockB, Index rows, Index depth, Index cols, std::complex alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { const Index accRows = quad_traits::rows; const Index accCols = quad_traits::size; void (*gemm_function)(const DataMapper&, const std::complex*, const std::complex*, Index, Index, Index, std::complex, Index, Index, Index, Index); #ifdef EIGEN_ALTIVEC_MMA_ONLY //generate with MMA only gemm_function = &Eigen::internal::gemm_complexMMA, std::complex, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma")){ gemm_function = &Eigen::internal::gemm_complexMMA, std::complex, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; } else{ gemm_function = &Eigen::internal::gemm_complex, std::complex, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; } #else gemm_function = &Eigen::internal::gemm_complex, std::complex, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, false>; #endif gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); } template struct gebp_kernel, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> { typedef quad_traits::vectortype Packet; typedef Packet1cd Packetc; typedef quad_traits::rhstype RhsPacket; void operator()(const DataMapper& res, const std::complex* blockA, const double* blockB, Index rows, Index depth, Index cols, std::complex alpha, Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0); }; template void gebp_kernel, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> ::operator()(const DataMapper& res, const std::complex* blockA, const double* blockB, Index rows, Index depth, Index cols, std::complex alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { const Index accRows = quad_traits::rows; const Index accCols = quad_traits::size; void (*gemm_function)(const DataMapper&, const std::complex*, const double*, Index, Index, Index, std::complex, Index, Index, Index, Index); #ifdef EIGEN_ALTIVEC_MMA_ONLY //generate with MMA only gemm_function = &Eigen::internal::gemm_complexMMA, double, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma")){ gemm_function = &Eigen::internal::gemm_complexMMA, double, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; } else{ gemm_function = &Eigen::internal::gemm_complex, double, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; } #else gemm_function = &Eigen::internal::gemm_complex, double, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, false, true>; #endif gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); } template struct gebp_kernel, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> { typedef quad_traits::vectortype Packet; typedef Packet1cd Packetc; typedef quad_traits::rhstype RhsPacket; void operator()(const DataMapper& res, const double* blockA, const std::complex* blockB, Index rows, Index depth, Index cols, std::complex alpha, Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0); }; template void gebp_kernel, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> ::operator()(const DataMapper& res, const double* blockA, const std::complex* blockB, Index rows, Index depth, Index cols, std::complex alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { const Index accRows = quad_traits::rows; const Index accCols = quad_traits::size; void (*gemm_function)(const DataMapper&, const double*, const std::complex*, Index, Index, Index, std::complex, Index, Index, Index, Index); #ifdef EIGEN_ALTIVEC_MMA_ONLY //generate with MMA only gemm_function = &Eigen::internal::gemm_complexMMA, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) if (__builtin_cpu_supports ("arch_3_1") && __builtin_cpu_supports ("mma")){ gemm_function = &Eigen::internal::gemm_complexMMA, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; } else{ gemm_function = &Eigen::internal::gemm_complex, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; } #else gemm_function = &Eigen::internal::gemm_complex, std::complex, double, Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, true, false>; #endif gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_MATRIX_PRODUCT_ALTIVEC_H RcppEigen/inst/include/Eigen/src/Core/arch/AltiVec/MatrixProductCommon.h0000644000176200001440000002242214567757725025662 0ustar liggesusers//#define EIGEN_POWER_USE_PREFETCH // Use prefetching in gemm routines #ifdef EIGEN_POWER_USE_PREFETCH #define EIGEN_POWER_PREFETCH(p) prefetch(p) #else #define EIGEN_POWER_PREFETCH(p) #endif namespace Eigen { namespace internal { template EIGEN_STRONG_INLINE void gemm_extra_col( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index row, Index col, Index remaining_rows, Index remaining_cols, const Packet& pAlpha); template EIGEN_STRONG_INLINE void gemm_extra_row( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index row, Index col, Index rows, Index cols, Index remaining_rows, const Packet& pAlpha, const Packet& pMask); template EIGEN_STRONG_INLINE void gemm_unrolled_col( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index& row, Index rows, Index col, Index remaining_cols, const Packet& pAlpha); template EIGEN_ALWAYS_INLINE Packet bmask(const int remaining_rows); template EIGEN_STRONG_INLINE void gemm_complex_extra_col( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index col, Index remaining_rows, Index remaining_cols, const Packet& pAlphaReal, const Packet& pAlphaImag); template EIGEN_STRONG_INLINE void gemm_complex_extra_row( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index col, Index rows, Index cols, Index remaining_rows, const Packet& pAlphaReal, const Packet& pAlphaImag, const Packet& pMask); template EIGEN_STRONG_INLINE void gemm_complex_unrolled_col( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index& row, Index rows, Index col, Index remaining_cols, const Packet& pAlphaReal, const Packet& pAlphaImag); template EIGEN_ALWAYS_INLINE Packet ploadLhs(const Scalar* lhs); template EIGEN_ALWAYS_INLINE void bload(PacketBlock& acc, const DataMapper& res, Index row, Index col); template EIGEN_ALWAYS_INLINE void bload(PacketBlock& acc, const DataMapper& res, Index row, Index col); template EIGEN_ALWAYS_INLINE void bscale(PacketBlock& acc, PacketBlock& accZ, const Packet& pAlpha); template EIGEN_ALWAYS_INLINE void bscalec(PacketBlock& aReal, PacketBlock& aImag, const Packet& bReal, const Packet& bImag, PacketBlock& cReal, PacketBlock& cImag); const static Packet16uc p16uc_SETCOMPLEX32_FIRST = { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23}; const static Packet16uc p16uc_SETCOMPLEX32_SECOND = { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31}; //[a,b],[ai,bi] = [a,ai] - This is equivalent to p16uc_GETREAL64 const static Packet16uc p16uc_SETCOMPLEX64_FIRST = { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23}; //[a,b],[ai,bi] = [b,bi] - This is equivalent to p16uc_GETIMAG64 const static Packet16uc p16uc_SETCOMPLEX64_SECOND = { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31}; // Grab two decouples real/imaginary PacketBlocks and return two coupled (real/imaginary pairs) PacketBlocks. template EIGEN_ALWAYS_INLINE void bcouple_common(PacketBlock& taccReal, PacketBlock& taccImag, PacketBlock& acc1, PacketBlock& acc2) { acc1.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX32_FIRST); acc1.packet[1].v = vec_perm(taccReal.packet[1], taccImag.packet[1], p16uc_SETCOMPLEX32_FIRST); acc1.packet[2].v = vec_perm(taccReal.packet[2], taccImag.packet[2], p16uc_SETCOMPLEX32_FIRST); acc1.packet[3].v = vec_perm(taccReal.packet[3], taccImag.packet[3], p16uc_SETCOMPLEX32_FIRST); acc2.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX32_SECOND); acc2.packet[1].v = vec_perm(taccReal.packet[1], taccImag.packet[1], p16uc_SETCOMPLEX32_SECOND); acc2.packet[2].v = vec_perm(taccReal.packet[2], taccImag.packet[2], p16uc_SETCOMPLEX32_SECOND); acc2.packet[3].v = vec_perm(taccReal.packet[3], taccImag.packet[3], p16uc_SETCOMPLEX32_SECOND); } template EIGEN_ALWAYS_INLINE void bcouple(PacketBlock& taccReal, PacketBlock& taccImag, PacketBlock& tRes, PacketBlock& acc1, PacketBlock& acc2) { bcouple_common(taccReal, taccImag, acc1, acc2); acc1.packet[0] = padd(tRes.packet[0], acc1.packet[0]); acc1.packet[1] = padd(tRes.packet[1], acc1.packet[1]); acc1.packet[2] = padd(tRes.packet[2], acc1.packet[2]); acc1.packet[3] = padd(tRes.packet[3], acc1.packet[3]); acc2.packet[0] = padd(tRes.packet[4], acc2.packet[0]); acc2.packet[1] = padd(tRes.packet[5], acc2.packet[1]); acc2.packet[2] = padd(tRes.packet[6], acc2.packet[2]); acc2.packet[3] = padd(tRes.packet[7], acc2.packet[3]); } template EIGEN_ALWAYS_INLINE void bcouple_common(PacketBlock& taccReal, PacketBlock& taccImag, PacketBlock& acc1, PacketBlock& acc2) { acc1.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX32_FIRST); acc2.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX32_SECOND); } template EIGEN_ALWAYS_INLINE void bcouple(PacketBlock& taccReal, PacketBlock& taccImag, PacketBlock& tRes, PacketBlock& acc1, PacketBlock& acc2) { bcouple_common(taccReal, taccImag, acc1, acc2); acc1.packet[0] = padd(tRes.packet[0], acc1.packet[0]); acc2.packet[0] = padd(tRes.packet[1], acc2.packet[0]); } template<> EIGEN_ALWAYS_INLINE void bcouple_common(PacketBlock& taccReal, PacketBlock& taccImag, PacketBlock& acc1, PacketBlock& acc2) { acc1.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX64_FIRST); acc1.packet[1].v = vec_perm(taccReal.packet[1], taccImag.packet[1], p16uc_SETCOMPLEX64_FIRST); acc1.packet[2].v = vec_perm(taccReal.packet[2], taccImag.packet[2], p16uc_SETCOMPLEX64_FIRST); acc1.packet[3].v = vec_perm(taccReal.packet[3], taccImag.packet[3], p16uc_SETCOMPLEX64_FIRST); acc2.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX64_SECOND); acc2.packet[1].v = vec_perm(taccReal.packet[1], taccImag.packet[1], p16uc_SETCOMPLEX64_SECOND); acc2.packet[2].v = vec_perm(taccReal.packet[2], taccImag.packet[2], p16uc_SETCOMPLEX64_SECOND); acc2.packet[3].v = vec_perm(taccReal.packet[3], taccImag.packet[3], p16uc_SETCOMPLEX64_SECOND); } template<> EIGEN_ALWAYS_INLINE void bcouple_common(PacketBlock& taccReal, PacketBlock& taccImag, PacketBlock& acc1, PacketBlock& acc2) { acc1.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX64_FIRST); acc2.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX64_SECOND); } // This is necessary because ploadRhs for double returns a pair of vectors when MMA is enabled. template EIGEN_ALWAYS_INLINE Packet ploadRhs(const Scalar* rhs) { return ploadu(rhs); } } // end namespace internal } // end namespace Eigen RcppEigen/inst/include/Eigen/src/Core/arch/AltiVec/Complex.h0000644000176200001440000004023414567757725023314 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2010 Gael Guennebaud // Copyright (C) 2010-2016 Konstantinos Margaritis // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_COMPLEX32_ALTIVEC_H #define EIGEN_COMPLEX32_ALTIVEC_H namespace Eigen { namespace internal { static Packet4ui p4ui_CONJ_XOR = vec_mergeh((Packet4ui)p4i_ZERO, (Packet4ui)p4f_MZERO);//{ 0x00000000, 0x80000000, 0x00000000, 0x80000000 }; #ifdef __VSX__ #if defined(_BIG_ENDIAN) static Packet2ul p2ul_CONJ_XOR1 = (Packet2ul) vec_sld((Packet4ui) p2d_MZERO, (Packet4ui) p2l_ZERO, 8);//{ 0x8000000000000000, 0x0000000000000000 }; static Packet2ul p2ul_CONJ_XOR2 = (Packet2ul) vec_sld((Packet4ui) p2l_ZERO, (Packet4ui) p2d_MZERO, 8);//{ 0x8000000000000000, 0x0000000000000000 }; #else static Packet2ul p2ul_CONJ_XOR1 = (Packet2ul) vec_sld((Packet4ui) p2l_ZERO, (Packet4ui) p2d_MZERO, 8);//{ 0x8000000000000000, 0x0000000000000000 }; static Packet2ul p2ul_CONJ_XOR2 = (Packet2ul) vec_sld((Packet4ui) p2d_MZERO, (Packet4ui) p2l_ZERO, 8);//{ 0x8000000000000000, 0x0000000000000000 }; #endif #endif //---------- float ---------- struct Packet2cf { EIGEN_STRONG_INLINE explicit Packet2cf() {} EIGEN_STRONG_INLINE explicit Packet2cf(const Packet4f& a) : v(a) {} EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) { Packet4f v1, v2; // Permute and multiply the real parts of a and b v1 = vec_perm(a.v, a.v, p16uc_PSET32_WODD); // Get the imaginary parts of a v2 = vec_perm(a.v, a.v, p16uc_PSET32_WEVEN); // multiply a_re * b v1 = vec_madd(v1, b.v, p4f_ZERO); // multiply a_im * b and get the conjugate result v2 = vec_madd(v2, b.v, p4f_ZERO); v2 = reinterpret_cast(pxor(v2, reinterpret_cast(p4ui_CONJ_XOR))); // permute back to a proper order v2 = vec_perm(v2, v2, p16uc_COMPLEX32_REV); return Packet2cf(padd(v1, v2)); } EIGEN_STRONG_INLINE Packet2cf& operator*=(const Packet2cf& b) { v = pmul(Packet2cf(*this), b).v; return *this; } EIGEN_STRONG_INLINE Packet2cf operator*(const Packet2cf& b) const { return Packet2cf(*this) *= b; } EIGEN_STRONG_INLINE Packet2cf& operator+=(const Packet2cf& b) { v = padd(v, b.v); return *this; } EIGEN_STRONG_INLINE Packet2cf operator+(const Packet2cf& b) const { return Packet2cf(*this) += b; } EIGEN_STRONG_INLINE Packet2cf& operator-=(const Packet2cf& b) { v = psub(v, b.v); return *this; } EIGEN_STRONG_INLINE Packet2cf operator-(const Packet2cf& b) const { return Packet2cf(*this) -= b; } EIGEN_STRONG_INLINE Packet2cf operator-(void) const { return Packet2cf(-v); } Packet4f v; }; template<> struct packet_traits > : default_packet_traits { typedef Packet2cf type; typedef Packet2cf half; typedef Packet4f as_real; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 2, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, HasMax = 0, #ifdef __VSX__ HasBlend = 1, #endif HasSetLinear = 0 }; }; template<> struct unpacket_traits { typedef std::complex type; enum {size=2, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet2cf half; typedef Packet4f as_real; }; template<> EIGEN_STRONG_INLINE Packet2cf pset1(const std::complex& from) { Packet2cf res; if((std::ptrdiff_t(&from) % 16) == 0) res.v = pload((const float *)&from); else res.v = ploadu((const float *)&from); res.v = vec_perm(res.v, res.v, p16uc_PSET64_HI); return res; } template<> EIGEN_STRONG_INLINE Packet2cf pload(const std::complex* from) { return Packet2cf(pload((const float *) from)); } template<> EIGEN_STRONG_INLINE Packet2cf ploadu(const std::complex* from) { return Packet2cf(ploadu((const float*) from)); } template<> EIGEN_STRONG_INLINE Packet2cf ploaddup(const std::complex* from) { return pset1(*from); } template<> EIGEN_STRONG_INLINE void pstore >(std::complex * to, const Packet2cf& from) { pstore((float*)to, from.v); } template<> EIGEN_STRONG_INLINE void pstoreu >(std::complex * to, const Packet2cf& from) { pstoreu((float*)to, from.v); } EIGEN_STRONG_INLINE Packet2cf pload2(const std::complex* from0, const std::complex* from1) { Packet4f res0, res1; #ifdef __VSX__ __asm__ ("lxsdx %x0,%y1" : "=wa" (res0) : "Z" (*from0)); __asm__ ("lxsdx %x0,%y1" : "=wa" (res1) : "Z" (*from1)); #ifdef _BIG_ENDIAN __asm__ ("xxpermdi %x0, %x1, %x2, 0" : "=wa" (res0) : "wa" (res0), "wa" (res1)); #else __asm__ ("xxpermdi %x0, %x2, %x1, 0" : "=wa" (res0) : "wa" (res0), "wa" (res1)); #endif #else *reinterpret_cast *>(&res0) = *from0; *reinterpret_cast *>(&res1) = *from1; res0 = vec_perm(res0, res1, p16uc_TRANSPOSE64_HI); #endif return Packet2cf(res0); } template<> EIGEN_DEVICE_FUNC inline Packet2cf pgather, Packet2cf>(const std::complex* from, Index stride) { EIGEN_ALIGN16 std::complex af[2]; af[0] = from[0*stride]; af[1] = from[1*stride]; return pload(af); } template<> EIGEN_DEVICE_FUNC inline void pscatter, Packet2cf>(std::complex* to, const Packet2cf& from, Index stride) { EIGEN_ALIGN16 std::complex af[2]; pstore >((std::complex *) af, from); to[0*stride] = af[0]; to[1*stride] = af[1]; } template<> EIGEN_STRONG_INLINE Packet2cf padd(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(a.v + b.v); } template<> EIGEN_STRONG_INLINE Packet2cf psub(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(a.v - b.v); } template<> EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf& a) { return Packet2cf(pnegate(a.v)); } template<> EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) { return Packet2cf(pxor(a.v, reinterpret_cast(p4ui_CONJ_XOR))); } template<> EIGEN_STRONG_INLINE Packet2cf pand (const Packet2cf& a, const Packet2cf& b) { return Packet2cf(pand(a.v, b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf por (const Packet2cf& a, const Packet2cf& b) { return Packet2cf(por(a.v, b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf pxor (const Packet2cf& a, const Packet2cf& b) { return Packet2cf(pxor(a.v, b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf pandnot(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(pandnot(a.v, b.v)); } template<> EIGEN_STRONG_INLINE void prefetch >(const std::complex * addr) { EIGEN_PPC_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE std::complex pfirst(const Packet2cf& a) { EIGEN_ALIGN16 std::complex res[2]; pstore((float *)&res, a.v); return res[0]; } template<> EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) { Packet4f rev_a; rev_a = vec_perm(a.v, a.v, p16uc_COMPLEX32_REV2); return Packet2cf(rev_a); } template<> EIGEN_STRONG_INLINE std::complex predux(const Packet2cf& a) { Packet4f b; b = vec_sld(a.v, a.v, 8); b = padd(a.v, b); return pfirst(Packet2cf(b)); } template<> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet2cf& a) { Packet4f b; Packet2cf prod; b = vec_sld(a.v, a.v, 8); prod = pmul(a, Packet2cf(b)); return pfirst(prod); } EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf,Packet4f) template<> EIGEN_STRONG_INLINE Packet2cf pdiv(const Packet2cf& a, const Packet2cf& b) { // TODO optimize it for AltiVec Packet2cf res = pmul(a, pconj(b)); Packet4f s = pmul(b.v, b.v); return Packet2cf(pdiv(res.v, padd(s, vec_perm(s, s, p16uc_COMPLEX32_REV)))); } template<> EIGEN_STRONG_INLINE Packet2cf pcplxflip(const Packet2cf& x) { return Packet2cf(vec_perm(x.v, x.v, p16uc_COMPLEX32_REV)); } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { Packet4f tmp = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_HI); kernel.packet[1].v = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_LO); kernel.packet[0].v = tmp; } template<> EIGEN_STRONG_INLINE Packet2cf pcmp_eq(const Packet2cf& a, const Packet2cf& b) { Packet4f eq = reinterpret_cast(vec_cmpeq(a.v,b.v)); return Packet2cf(vec_and(eq, vec_perm(eq, eq, p16uc_COMPLEX32_REV))); } #ifdef __VSX__ template<> EIGEN_STRONG_INLINE Packet2cf pblend(const Selector<2>& ifPacket, const Packet2cf& thenPacket, const Packet2cf& elsePacket) { Packet2cf result; result.v = reinterpret_cast(pblend(ifPacket, reinterpret_cast(thenPacket.v), reinterpret_cast(elsePacket.v))); return result; } #endif template<> EIGEN_STRONG_INLINE Packet2cf psqrt(const Packet2cf& a) { return psqrt_complex(a); } //---------- double ---------- #ifdef __VSX__ struct Packet1cd { EIGEN_STRONG_INLINE Packet1cd() {} EIGEN_STRONG_INLINE explicit Packet1cd(const Packet2d& a) : v(a) {} EIGEN_STRONG_INLINE Packet1cd pmul(const Packet1cd& a, const Packet1cd& b) { Packet2d a_re, a_im, v1, v2; // Permute and multiply the real parts of a and b a_re = vec_perm(a.v, a.v, p16uc_PSET64_HI); // Get the imaginary parts of a a_im = vec_perm(a.v, a.v, p16uc_PSET64_LO); // multiply a_re * b v1 = vec_madd(a_re, b.v, p2d_ZERO); // multiply a_im * b and get the conjugate result v2 = vec_madd(a_im, b.v, p2d_ZERO); v2 = reinterpret_cast(vec_sld(reinterpret_cast(v2), reinterpret_cast(v2), 8)); v2 = pxor(v2, reinterpret_cast(p2ul_CONJ_XOR1)); return Packet1cd(padd(v1, v2)); } EIGEN_STRONG_INLINE Packet1cd& operator*=(const Packet1cd& b) { v = pmul(Packet1cd(*this), b).v; return *this; } EIGEN_STRONG_INLINE Packet1cd operator*(const Packet1cd& b) const { return Packet1cd(*this) *= b; } EIGEN_STRONG_INLINE Packet1cd& operator+=(const Packet1cd& b) { v = padd(v, b.v); return *this; } EIGEN_STRONG_INLINE Packet1cd operator+(const Packet1cd& b) const { return Packet1cd(*this) += b; } EIGEN_STRONG_INLINE Packet1cd& operator-=(const Packet1cd& b) { v = psub(v, b.v); return *this; } EIGEN_STRONG_INLINE Packet1cd operator-(const Packet1cd& b) const { return Packet1cd(*this) -= b; } EIGEN_STRONG_INLINE Packet1cd operator-(void) const { return Packet1cd(-v); } Packet2d v; }; template<> struct packet_traits > : default_packet_traits { typedef Packet1cd type; typedef Packet1cd half; typedef Packet2d as_real; enum { Vectorizable = 1, AlignedOnScalar = 0, size = 1, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, HasMax = 0, HasSetLinear = 0 }; }; template<> struct unpacket_traits { typedef std::complex type; enum {size=1, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet1cd half; typedef Packet2d as_real; }; template<> EIGEN_STRONG_INLINE Packet1cd pload (const std::complex* from) { return Packet1cd(pload((const double*)from)); } template<> EIGEN_STRONG_INLINE Packet1cd ploadu(const std::complex* from) { return Packet1cd(ploadu((const double*)from)); } template<> EIGEN_STRONG_INLINE void pstore >(std::complex * to, const Packet1cd& from) { pstore((double*)to, from.v); } template<> EIGEN_STRONG_INLINE void pstoreu >(std::complex * to, const Packet1cd& from) { pstoreu((double*)to, from.v); } template<> EIGEN_STRONG_INLINE Packet1cd pset1(const std::complex& from) { /* here we really have to use unaligned loads :( */ return ploadu(&from); } template<> EIGEN_DEVICE_FUNC inline Packet1cd pgather, Packet1cd>(const std::complex* from, Index) { return pload(from); } template<> EIGEN_DEVICE_FUNC inline void pscatter, Packet1cd>(std::complex* to, const Packet1cd& from, Index) { pstore >(to, from); } template<> EIGEN_STRONG_INLINE Packet1cd padd(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(a.v + b.v); } template<> EIGEN_STRONG_INLINE Packet1cd psub(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(a.v - b.v); } template<> EIGEN_STRONG_INLINE Packet1cd pnegate(const Packet1cd& a) { return Packet1cd(pnegate(Packet2d(a.v))); } template<> EIGEN_STRONG_INLINE Packet1cd pconj(const Packet1cd& a) { return Packet1cd(pxor(a.v, reinterpret_cast(p2ul_CONJ_XOR2))); } template<> EIGEN_STRONG_INLINE Packet1cd pand (const Packet1cd& a, const Packet1cd& b) { return Packet1cd(pand(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd por (const Packet1cd& a, const Packet1cd& b) { return Packet1cd(por(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd pxor (const Packet1cd& a, const Packet1cd& b) { return Packet1cd(pxor(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd pandnot(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(pandnot(a.v, b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd ploaddup(const std::complex* from) { return pset1(*from); } template<> EIGEN_STRONG_INLINE void prefetch >(const std::complex * addr) { EIGEN_PPC_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE std::complex pfirst(const Packet1cd& a) { EIGEN_ALIGN16 std::complex res[2]; pstore >(res, a); return res[0]; } template<> EIGEN_STRONG_INLINE Packet1cd preverse(const Packet1cd& a) { return a; } template<> EIGEN_STRONG_INLINE std::complex predux(const Packet1cd& a) { return pfirst(a); } template<> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet1cd& a) { return pfirst(a); } EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd,Packet2d) template<> EIGEN_STRONG_INLINE Packet1cd pdiv(const Packet1cd& a, const Packet1cd& b) { // TODO optimize it for AltiVec Packet1cd res = pmul(a,pconj(b)); Packet2d s = pmul(b.v, b.v); return Packet1cd(pdiv(res.v, padd(s, vec_perm(s, s, p16uc_REVERSE64)))); } EIGEN_STRONG_INLINE Packet1cd pcplxflip/**/(const Packet1cd& x) { return Packet1cd(preverse(Packet2d(x.v))); } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { Packet2d tmp = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_HI); kernel.packet[1].v = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_LO); kernel.packet[0].v = tmp; } template<> EIGEN_STRONG_INLINE Packet1cd pcmp_eq(const Packet1cd& a, const Packet1cd& b) { // Compare real and imaginary parts of a and b to get the mask vector: // [re(a)==re(b), im(a)==im(b)] Packet2d eq = reinterpret_cast(vec_cmpeq(a.v,b.v)); // Swap real/imag elements in the mask in to get: // [im(a)==im(b), re(a)==re(b)] Packet2d eq_swapped = reinterpret_cast(vec_sld(reinterpret_cast(eq), reinterpret_cast(eq), 8)); // Return re(a)==re(b) & im(a)==im(b) by computing bitwise AND of eq and eq_swapped return Packet1cd(vec_and(eq, eq_swapped)); } template<> EIGEN_STRONG_INLINE Packet1cd psqrt(const Packet1cd& a) { return psqrt_complex(a); } #endif // __VSX__ } // end namespace internal } // end namespace Eigen #endif // EIGEN_COMPLEX32_ALTIVEC_H RcppEigen/inst/include/Eigen/src/Core/arch/AltiVec/PacketMath.h0000755000176200001440000030777214567757725023746 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2016 Konstantinos Margaritis // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_PACKET_MATH_ALTIVEC_H #define EIGEN_PACKET_MATH_ALTIVEC_H namespace Eigen { namespace internal { #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 4 #endif #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD #endif // NOTE Altivec has 32 registers, but Eigen only accepts a value of 8 or 16 #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32 #endif typedef __vector float Packet4f; typedef __vector int Packet4i; typedef __vector unsigned int Packet4ui; typedef __vector __bool int Packet4bi; typedef __vector short int Packet8s; typedef __vector unsigned short int Packet8us; typedef __vector signed char Packet16c; typedef __vector unsigned char Packet16uc; typedef eigen_packet_wrapper<__vector unsigned short int,0> Packet8bf; // We don't want to write the same code all the time, but we need to reuse the constants // and it doesn't really work to declare them global, so we define macros instead #define _EIGEN_DECLARE_CONST_FAST_Packet4f(NAME,X) \ Packet4f p4f_##NAME = {X, X, X, X} #define _EIGEN_DECLARE_CONST_FAST_Packet4i(NAME,X) \ Packet4i p4i_##NAME = vec_splat_s32(X) #define _EIGEN_DECLARE_CONST_FAST_Packet4ui(NAME,X) \ Packet4ui p4ui_##NAME = {X, X, X, X} #define _EIGEN_DECLARE_CONST_FAST_Packet8us(NAME,X) \ Packet8us p8us_##NAME = {X, X, X, X, X, X, X, X} #define _EIGEN_DECLARE_CONST_FAST_Packet16uc(NAME,X) \ Packet16uc p16uc_##NAME = {X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X} #define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ Packet4f p4f_##NAME = pset1(X) #define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ Packet4i p4i_##NAME = pset1(X) #define _EIGEN_DECLARE_CONST_Packet2d(NAME,X) \ Packet2d p2d_##NAME = pset1(X) #define _EIGEN_DECLARE_CONST_Packet2l(NAME,X) \ Packet2l p2l_##NAME = pset1(X) #define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ const Packet4f p4f_##NAME = reinterpret_cast(pset1(X)) #define DST_CHAN 1 #define DST_CTRL(size, count, stride) (((size) << 24) | ((count) << 16) | (stride)) #define __UNPACK_TYPE__(PACKETNAME) typename unpacket_traits::type // These constants are endian-agnostic static _EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0); //{ 0.0, 0.0, 0.0, 0.0} static _EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0); //{ 0, 0, 0, 0,} static _EIGEN_DECLARE_CONST_FAST_Packet4i(ONE,1); //{ 1, 1, 1, 1} static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS16,-16); //{ -16, -16, -16, -16} static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1,-1); //{ -1, -1, -1, -1} static _EIGEN_DECLARE_CONST_FAST_Packet4ui(SIGN, 0x80000000u); static _EIGEN_DECLARE_CONST_FAST_Packet4ui(PREV0DOT5, 0x3EFFFFFFu); static _EIGEN_DECLARE_CONST_FAST_Packet8us(ONE,1); //{ 1, 1, 1, 1, 1, 1, 1, 1} static _EIGEN_DECLARE_CONST_FAST_Packet16uc(ONE,1); static Packet4f p4f_MZERO = (Packet4f) vec_sl((Packet4ui)p4i_MINUS1, (Packet4ui)p4i_MINUS1); //{ 0x80000000, 0x80000000, 0x80000000, 0x80000000} #ifndef __VSX__ static Packet4f p4f_ONE = vec_ctf(p4i_ONE, 0); //{ 1.0, 1.0, 1.0, 1.0} #endif static Packet4f p4f_COUNTDOWN = { 0.0, 1.0, 2.0, 3.0 }; static Packet4i p4i_COUNTDOWN = { 0, 1, 2, 3 }; static Packet8s p8s_COUNTDOWN = { 0, 1, 2, 3, 4, 5, 6, 7 }; static Packet8us p8us_COUNTDOWN = { 0, 1, 2, 3, 4, 5, 6, 7 }; static Packet16c p16c_COUNTDOWN = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; static Packet16uc p16uc_COUNTDOWN = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; static Packet16uc p16uc_REVERSE32 = { 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3 }; static Packet16uc p16uc_REVERSE16 = { 14,15, 12,13, 10,11, 8,9, 6,7, 4,5, 2,3, 0,1 }; static Packet16uc p16uc_REVERSE8 = { 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 }; static Packet16uc p16uc_DUPLICATE32_HI = { 0,1,2,3, 0,1,2,3, 4,5,6,7, 4,5,6,7 }; static Packet16uc p16uc_DUPLICATE16_HI = { 0,1,0,1, 2,3,2,3, 4,5,4,5, 6,7,6,7 }; static Packet16uc p16uc_DUPLICATE8_HI = { 0,0, 1,1, 2,2, 3,3, 4,4, 5,5, 6,6, 7,7 }; static const Packet16uc p16uc_DUPLICATE16_EVEN= { 0,1 ,0,1, 4,5, 4,5, 8,9, 8,9, 12,13, 12,13 }; static const Packet16uc p16uc_DUPLICATE16_ODD = { 2,3 ,2,3, 6,7, 6,7, 10,11, 10,11, 14,15, 14,15 }; static Packet16uc p16uc_QUADRUPLICATE16_HI = { 0,1,0,1,0,1,0,1, 2,3,2,3,2,3,2,3 }; // Handle endianness properly while loading constants // Define global static constants: #ifdef _BIG_ENDIAN static Packet16uc p16uc_FORWARD = vec_lvsl(0, (float*)0); #ifdef __VSX__ static Packet16uc p16uc_REVERSE64 = { 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 }; #endif static Packet16uc p16uc_PSET32_WODD = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);//{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 }; static Packet16uc p16uc_PSET32_WEVEN = vec_sld(p16uc_DUPLICATE32_HI, (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);//{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 }; static Packet16uc p16uc_HALF64_0_16 = vec_sld((Packet16uc)p4i_ZERO, vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 3), 8); //{ 0,0,0,0, 0,0,0,0, 16,16,16,16, 16,16,16,16}; #else static Packet16uc p16uc_FORWARD = p16uc_REVERSE32; static Packet16uc p16uc_REVERSE64 = { 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 }; static Packet16uc p16uc_PSET32_WODD = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 1), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);//{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 }; static Packet16uc p16uc_PSET32_WEVEN = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);//{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 }; static Packet16uc p16uc_HALF64_0_16 = vec_sld(vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 0), (Packet16uc)p4i_ZERO, 8); //{ 0,0,0,0, 0,0,0,0, 16,16,16,16, 16,16,16,16}; #endif // _BIG_ENDIAN static Packet16uc p16uc_PSET64_HI = (Packet16uc) vec_mergeh((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN); //{ 0,1,2,3, 4,5,6,7, 0,1,2,3, 4,5,6,7 }; static Packet16uc p16uc_PSET64_LO = (Packet16uc) vec_mergel((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN); //{ 8,9,10,11, 12,13,14,15, 8,9,10,11, 12,13,14,15 }; static Packet16uc p16uc_TRANSPOSE64_HI = p16uc_PSET64_HI + p16uc_HALF64_0_16; //{ 0,1,2,3, 4,5,6,7, 16,17,18,19, 20,21,22,23}; static Packet16uc p16uc_TRANSPOSE64_LO = p16uc_PSET64_LO + p16uc_HALF64_0_16; //{ 8,9,10,11, 12,13,14,15, 24,25,26,27, 28,29,30,31}; static Packet16uc p16uc_COMPLEX32_REV = vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8); //{ 4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11 }; #ifdef _BIG_ENDIAN static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_FORWARD, p16uc_FORWARD, 8); //{ 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 }; #else static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_PSET64_HI, p16uc_PSET64_LO, 8); //{ 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 }; #endif // _BIG_ENDIAN #if EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC #define EIGEN_PPC_PREFETCH(ADDR) __builtin_prefetch(ADDR); #else #define EIGEN_PPC_PREFETCH(ADDR) asm( " dcbt [%[addr]]\n" :: [addr] "r" (ADDR) : "cc" ); #endif template <> struct packet_traits : default_packet_traits { typedef Packet4f type; typedef Packet4f half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 4, HasHalfPacket = 1, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasMin = 1, HasMax = 1, HasAbs = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasLog = 1, HasExp = 1, #ifdef __VSX__ HasSqrt = 1, #if !EIGEN_COMP_CLANG HasRsqrt = 1, #else HasRsqrt = 0, #endif #else HasSqrt = 0, HasRsqrt = 0, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, #endif HasRound = 1, HasFloor = 1, HasCeil = 1, HasRint = 1, HasNegate = 1, HasBlend = 1 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet8bf type; typedef Packet8bf half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 8, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasMin = 1, HasMax = 1, HasAbs = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasLog = 1, HasExp = 1, #ifdef __VSX__ HasSqrt = 1, #if !EIGEN_COMP_CLANG HasRsqrt = 1, #else HasRsqrt = 0, #endif #else HasSqrt = 0, HasRsqrt = 0, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, #endif HasRound = 1, HasFloor = 1, HasCeil = 1, HasRint = 1, HasNegate = 1, HasBlend = 1 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet4i type; typedef Packet4i half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 4, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasDiv = 0, HasBlend = 1 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet8s type; typedef Packet8s half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 8, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 0, HasBlend = 1 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet8us type; typedef Packet8us half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 8, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 0, HasBlend = 1 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet16c type; typedef Packet16c half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 16, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 0, HasBlend = 1 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet16uc type; typedef Packet16uc half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 16, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 0, HasBlend = 1 }; }; template<> struct unpacket_traits { typedef float type; typedef Packet4f half; typedef Packet4i integer_packet; enum {size=4, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; }; template<> struct unpacket_traits { typedef int type; typedef Packet4i half; enum {size=4, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; }; template<> struct unpacket_traits { typedef short int type; typedef Packet8s half; enum {size=8, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; }; template<> struct unpacket_traits { typedef unsigned short int type; typedef Packet8us half; enum {size=8, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; }; template<> struct unpacket_traits { typedef signed char type; typedef Packet16c half; enum {size=16, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; }; template<> struct unpacket_traits { typedef unsigned char type; typedef Packet16uc half; enum {size=16, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; }; template<> struct unpacket_traits { typedef bfloat16 type; typedef Packet8bf half; enum {size=8, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; }; inline std::ostream & operator <<(std::ostream & s, const Packet16c & v) { union { Packet16c v; signed char n[16]; } vt; vt.v = v; for (int i=0; i< 16; i++) s << vt.n[i] << ", "; return s; } inline std::ostream & operator <<(std::ostream & s, const Packet16uc & v) { union { Packet16uc v; unsigned char n[16]; } vt; vt.v = v; for (int i=0; i< 16; i++) s << vt.n[i] << ", "; return s; } inline std::ostream & operator <<(std::ostream & s, const Packet4f & v) { union { Packet4f v; float n[4]; } vt; vt.v = v; s << vt.n[0] << ", " << vt.n[1] << ", " << vt.n[2] << ", " << vt.n[3]; return s; } inline std::ostream & operator <<(std::ostream & s, const Packet4i & v) { union { Packet4i v; int n[4]; } vt; vt.v = v; s << vt.n[0] << ", " << vt.n[1] << ", " << vt.n[2] << ", " << vt.n[3]; return s; } inline std::ostream & operator <<(std::ostream & s, const Packet4ui & v) { union { Packet4ui v; unsigned int n[4]; } vt; vt.v = v; s << vt.n[0] << ", " << vt.n[1] << ", " << vt.n[2] << ", " << vt.n[3]; return s; } template EIGEN_STRONG_INLINE Packet pload_common(const __UNPACK_TYPE__(Packet)* from) { // some versions of GCC throw "unused-but-set-parameter". // ignoring these warnings for now. EIGEN_UNUSED_VARIABLE(from); EIGEN_DEBUG_ALIGNED_LOAD #ifdef __VSX__ return vec_xl(0, const_cast<__UNPACK_TYPE__(Packet)*>(from)); #else return vec_ld(0, from); #endif } // Need to define them first or we get specialization after instantiation errors template<> EIGEN_STRONG_INLINE Packet4f pload(const float* from) { return pload_common(from); } template<> EIGEN_STRONG_INLINE Packet4i pload(const int* from) { return pload_common(from); } template<> EIGEN_STRONG_INLINE Packet8s pload(const short int* from) { return pload_common(from); } template<> EIGEN_STRONG_INLINE Packet8us pload(const unsigned short int* from) { return pload_common(from); } template<> EIGEN_STRONG_INLINE Packet16c pload(const signed char* from) { return pload_common(from); } template<> EIGEN_STRONG_INLINE Packet16uc pload(const unsigned char* from) { return pload_common(from); } template<> EIGEN_STRONG_INLINE Packet8bf pload(const bfloat16* from) { return pload_common(reinterpret_cast(from)); } template EIGEN_STRONG_INLINE void pstore_common(__UNPACK_TYPE__(Packet)* to, const Packet& from){ // some versions of GCC throw "unused-but-set-parameter" (float *to). // ignoring these warnings for now. EIGEN_UNUSED_VARIABLE(to); EIGEN_DEBUG_ALIGNED_STORE #ifdef __VSX__ vec_xst(from, 0, to); #else vec_st(from, 0, to); #endif } template<> EIGEN_STRONG_INLINE void pstore(float* to, const Packet4f& from) { pstore_common(to, from); } template<> EIGEN_STRONG_INLINE void pstore(int* to, const Packet4i& from) { pstore_common(to, from); } template<> EIGEN_STRONG_INLINE void pstore(short int* to, const Packet8s& from) { pstore_common(to, from); } template<> EIGEN_STRONG_INLINE void pstore(unsigned short int* to, const Packet8us& from) { pstore_common(to, from); } template<> EIGEN_STRONG_INLINE void pstore(bfloat16* to, const Packet8bf& from) { pstore_common(reinterpret_cast(to), from); } template<> EIGEN_STRONG_INLINE void pstore(signed char* to, const Packet16c& from) { pstore_common(to, from); } template<> EIGEN_STRONG_INLINE void pstore(unsigned char* to, const Packet16uc& from) { pstore_common(to, from); } template EIGEN_STRONG_INLINE Packet pset1_size4(const __UNPACK_TYPE__(Packet)& from) { Packet v = {from, from, from, from}; return v; } template EIGEN_STRONG_INLINE Packet pset1_size8(const __UNPACK_TYPE__(Packet)& from) { Packet v = {from, from, from, from, from, from, from, from}; return v; } template EIGEN_STRONG_INLINE Packet pset1_size16(const __UNPACK_TYPE__(Packet)& from) { Packet v = {from, from, from, from, from, from, from, from, from, from, from, from, from, from, from, from}; return v; } template<> EIGEN_STRONG_INLINE Packet4f pset1(const float& from) { return pset1_size4(from); } template<> EIGEN_STRONG_INLINE Packet4i pset1(const int& from) { return pset1_size4(from); } template<> EIGEN_STRONG_INLINE Packet8s pset1(const short int& from) { return pset1_size8(from); } template<> EIGEN_STRONG_INLINE Packet8us pset1(const unsigned short int& from) { return pset1_size8(from); } template<> EIGEN_STRONG_INLINE Packet16c pset1(const signed char& from) { return pset1_size16(from); } template<> EIGEN_STRONG_INLINE Packet16uc pset1(const unsigned char& from) { return pset1_size16(from); } template<> EIGEN_STRONG_INLINE Packet4f pset1frombits(unsigned int from) { return reinterpret_cast(pset1(from)); } template<> EIGEN_STRONG_INLINE Packet8bf pset1(const bfloat16& from) { return pset1_size8(reinterpret_cast(from)); } template EIGEN_STRONG_INLINE void pbroadcast4_common(const __UNPACK_TYPE__(Packet) *a, Packet& a0, Packet& a1, Packet& a2, Packet& a3) { a3 = pload(a); a0 = vec_splat(a3, 0); a1 = vec_splat(a3, 1); a2 = vec_splat(a3, 2); a3 = vec_splat(a3, 3); } template<> EIGEN_STRONG_INLINE void pbroadcast4(const float *a, Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) { pbroadcast4_common(a, a0, a1, a2, a3); } template<> EIGEN_STRONG_INLINE void pbroadcast4(const int *a, Packet4i& a0, Packet4i& a1, Packet4i& a2, Packet4i& a3) { pbroadcast4_common(a, a0, a1, a2, a3); } template EIGEN_DEVICE_FUNC inline Packet pgather_common(const __UNPACK_TYPE__(Packet)* from, Index stride) { EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[4]; a[0] = from[0*stride]; a[1] = from[1*stride]; a[2] = from[2*stride]; a[3] = from[3*stride]; return pload(a); } template<> EIGEN_DEVICE_FUNC inline Packet4f pgather(const float* from, Index stride) { return pgather_common(from, stride); } template<> EIGEN_DEVICE_FUNC inline Packet4i pgather(const int* from, Index stride) { return pgather_common(from, stride); } template EIGEN_DEVICE_FUNC inline Packet pgather_size8(const __UNPACK_TYPE__(Packet)* from, Index stride) { EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[8]; a[0] = from[0*stride]; a[1] = from[1*stride]; a[2] = from[2*stride]; a[3] = from[3*stride]; a[4] = from[4*stride]; a[5] = from[5*stride]; a[6] = from[6*stride]; a[7] = from[7*stride]; return pload(a); } template<> EIGEN_DEVICE_FUNC inline Packet8s pgather(const short int* from, Index stride) { return pgather_size8(from, stride); } template<> EIGEN_DEVICE_FUNC inline Packet8us pgather(const unsigned short int* from, Index stride) { return pgather_size8(from, stride); } template<> EIGEN_DEVICE_FUNC inline Packet8bf pgather(const bfloat16* from, Index stride) { return pgather_size8(from, stride); } template EIGEN_DEVICE_FUNC inline Packet pgather_size16(const __UNPACK_TYPE__(Packet)* from, Index stride) { EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[16]; a[0] = from[0*stride]; a[1] = from[1*stride]; a[2] = from[2*stride]; a[3] = from[3*stride]; a[4] = from[4*stride]; a[5] = from[5*stride]; a[6] = from[6*stride]; a[7] = from[7*stride]; a[8] = from[8*stride]; a[9] = from[9*stride]; a[10] = from[10*stride]; a[11] = from[11*stride]; a[12] = from[12*stride]; a[13] = from[13*stride]; a[14] = from[14*stride]; a[15] = from[15*stride]; return pload(a); } template<> EIGEN_DEVICE_FUNC inline Packet16c pgather(const signed char* from, Index stride) { return pgather_size16(from, stride); } template<> EIGEN_DEVICE_FUNC inline Packet16uc pgather(const unsigned char* from, Index stride) { return pgather_size16(from, stride); } template EIGEN_DEVICE_FUNC inline void pscatter_size4(__UNPACK_TYPE__(Packet)* to, const Packet& from, Index stride) { EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[4]; pstore<__UNPACK_TYPE__(Packet)>(a, from); to[0*stride] = a[0]; to[1*stride] = a[1]; to[2*stride] = a[2]; to[3*stride] = a[3]; } template<> EIGEN_DEVICE_FUNC inline void pscatter(float* to, const Packet4f& from, Index stride) { pscatter_size4(to, from, stride); } template<> EIGEN_DEVICE_FUNC inline void pscatter(int* to, const Packet4i& from, Index stride) { pscatter_size4(to, from, stride); } template EIGEN_DEVICE_FUNC inline void pscatter_size8(__UNPACK_TYPE__(Packet)* to, const Packet& from, Index stride) { EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[8]; pstore<__UNPACK_TYPE__(Packet)>(a, from); to[0*stride] = a[0]; to[1*stride] = a[1]; to[2*stride] = a[2]; to[3*stride] = a[3]; to[4*stride] = a[4]; to[5*stride] = a[5]; to[6*stride] = a[6]; to[7*stride] = a[7]; } template<> EIGEN_DEVICE_FUNC inline void pscatter(short int* to, const Packet8s& from, Index stride) { pscatter_size8(to, from, stride); } template<> EIGEN_DEVICE_FUNC inline void pscatter(unsigned short int* to, const Packet8us& from, Index stride) { pscatter_size8(to, from, stride); } template<> EIGEN_DEVICE_FUNC inline void pscatter(bfloat16* to, const Packet8bf& from, Index stride) { pscatter_size8(to, from, stride); } template EIGEN_DEVICE_FUNC inline void pscatter_size16(__UNPACK_TYPE__(Packet)* to, const Packet& from, Index stride) { EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[16]; pstore<__UNPACK_TYPE__(Packet)>(a, from); to[0*stride] = a[0]; to[1*stride] = a[1]; to[2*stride] = a[2]; to[3*stride] = a[3]; to[4*stride] = a[4]; to[5*stride] = a[5]; to[6*stride] = a[6]; to[7*stride] = a[7]; to[8*stride] = a[8]; to[9*stride] = a[9]; to[10*stride] = a[10]; to[11*stride] = a[11]; to[12*stride] = a[12]; to[13*stride] = a[13]; to[14*stride] = a[14]; to[15*stride] = a[15]; } template<> EIGEN_DEVICE_FUNC inline void pscatter(signed char* to, const Packet16c& from, Index stride) { pscatter_size16(to, from, stride); } template<> EIGEN_DEVICE_FUNC inline void pscatter(unsigned char* to, const Packet16uc& from, Index stride) { pscatter_size16(to, from, stride); } template<> EIGEN_STRONG_INLINE Packet4f plset(const float& a) { return pset1(a) + p4f_COUNTDOWN; } template<> EIGEN_STRONG_INLINE Packet4i plset(const int& a) { return pset1(a) + p4i_COUNTDOWN; } template<> EIGEN_STRONG_INLINE Packet8s plset(const short int& a) { return pset1(a) + p8s_COUNTDOWN; } template<> EIGEN_STRONG_INLINE Packet8us plset(const unsigned short int& a) { return pset1(a) + p8us_COUNTDOWN; } template<> EIGEN_STRONG_INLINE Packet16c plset(const signed char& a) { return pset1(a) + p16c_COUNTDOWN; } template<> EIGEN_STRONG_INLINE Packet16uc plset(const unsigned char& a) { return pset1(a) + p16uc_COUNTDOWN; } template<> EIGEN_STRONG_INLINE Packet4f padd (const Packet4f& a, const Packet4f& b) { return a + b; } template<> EIGEN_STRONG_INLINE Packet4i padd (const Packet4i& a, const Packet4i& b) { return a + b; } template<> EIGEN_STRONG_INLINE Packet4ui padd (const Packet4ui& a, const Packet4ui& b) { return a + b; } template<> EIGEN_STRONG_INLINE Packet8s padd (const Packet8s& a, const Packet8s& b) { return a + b; } template<> EIGEN_STRONG_INLINE Packet8us padd (const Packet8us& a, const Packet8us& b) { return a + b; } template<> EIGEN_STRONG_INLINE Packet16c padd (const Packet16c& a, const Packet16c& b) { return a + b; } template<> EIGEN_STRONG_INLINE Packet16uc padd(const Packet16uc& a, const Packet16uc& b) { return a + b; } template<> EIGEN_STRONG_INLINE Packet4f psub (const Packet4f& a, const Packet4f& b) { return a - b; } template<> EIGEN_STRONG_INLINE Packet4i psub (const Packet4i& a, const Packet4i& b) { return a - b; } template<> EIGEN_STRONG_INLINE Packet8s psub (const Packet8s& a, const Packet8s& b) { return a - b; } template<> EIGEN_STRONG_INLINE Packet8us psub (const Packet8us& a, const Packet8us& b) { return a - b; } template<> EIGEN_STRONG_INLINE Packet16c psub (const Packet16c& a, const Packet16c& b) { return a - b; } template<> EIGEN_STRONG_INLINE Packet16uc psub(const Packet16uc& a, const Packet16uc& b) { return a - b; } template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { return p4f_ZERO - a; } template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return p4i_ZERO - a; } template<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4f pmul (const Packet4f& a, const Packet4f& b) { return vec_madd(a,b, p4f_MZERO); } template<> EIGEN_STRONG_INLINE Packet4i pmul (const Packet4i& a, const Packet4i& b) { return a * b; } template<> EIGEN_STRONG_INLINE Packet8s pmul (const Packet8s& a, const Packet8s& b) { return vec_mul(a,b); } template<> EIGEN_STRONG_INLINE Packet8us pmul (const Packet8us& a, const Packet8us& b) { return vec_mul(a,b); } template<> EIGEN_STRONG_INLINE Packet16c pmul (const Packet16c& a, const Packet16c& b) { return vec_mul(a,b); } template<> EIGEN_STRONG_INLINE Packet16uc pmul(const Packet16uc& a, const Packet16uc& b) { return vec_mul(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pdiv(const Packet4f& a, const Packet4f& b) { #ifndef __VSX__ // VSX actually provides a div instruction Packet4f t, y_0, y_1; // Altivec does not offer a divide instruction, we have to do a reciprocal approximation y_0 = vec_re(b); // Do one Newton-Raphson iteration to get the needed accuracy t = vec_nmsub(y_0, b, p4f_ONE); y_1 = vec_madd(y_0, t, y_0); return vec_madd(a, y_1, p4f_MZERO); #else return vec_div(a, b); #endif } template<> EIGEN_STRONG_INLINE Packet4i pdiv(const Packet4i& /*a*/, const Packet4i& /*b*/) { eigen_assert(false && "packet integer division are not supported by AltiVec"); return pset1(0); } // for some weird raisons, it has to be overloaded for packet of integers template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return vec_madd(a,b,c); } template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return a*b + c; } template<> EIGEN_STRONG_INLINE Packet8s pmadd(const Packet8s& a, const Packet8s& b, const Packet8s& c) { return vec_madd(a,b,c); } template<> EIGEN_STRONG_INLINE Packet8us pmadd(const Packet8us& a, const Packet8us& b, const Packet8us& c) { return vec_madd(a,b,c); } template<> EIGEN_STRONG_INLINE Packet4f pmin(const Packet4f& a, const Packet4f& b) { #ifdef __VSX__ // NOTE: about 10% slower than vec_min, but consistent with std::min and SSE regarding NaN Packet4f ret; __asm__ ("xvcmpgesp %x0,%x1,%x2\n\txxsel %x0,%x1,%x2,%x0" : "=&wa" (ret) : "wa" (a), "wa" (b)); return ret; #else return vec_min(a, b); #endif } template<> EIGEN_STRONG_INLINE Packet4i pmin(const Packet4i& a, const Packet4i& b) { return vec_min(a, b); } template<> EIGEN_STRONG_INLINE Packet8s pmin(const Packet8s& a, const Packet8s& b) { return vec_min(a, b); } template<> EIGEN_STRONG_INLINE Packet8us pmin(const Packet8us& a, const Packet8us& b) { return vec_min(a, b); } template<> EIGEN_STRONG_INLINE Packet16c pmin(const Packet16c& a, const Packet16c& b) { return vec_min(a, b); } template<> EIGEN_STRONG_INLINE Packet16uc pmin(const Packet16uc& a, const Packet16uc& b) { return vec_min(a, b); } template<> EIGEN_STRONG_INLINE Packet4f pmax(const Packet4f& a, const Packet4f& b) { #ifdef __VSX__ // NOTE: about 10% slower than vec_max, but consistent with std::max and SSE regarding NaN Packet4f ret; __asm__ ("xvcmpgtsp %x0,%x2,%x1\n\txxsel %x0,%x1,%x2,%x0" : "=&wa" (ret) : "wa" (a), "wa" (b)); return ret; #else return vec_max(a, b); #endif } template<> EIGEN_STRONG_INLINE Packet4i pmax(const Packet4i& a, const Packet4i& b) { return vec_max(a, b); } template<> EIGEN_STRONG_INLINE Packet8s pmax(const Packet8s& a, const Packet8s& b) { return vec_max(a, b); } template<> EIGEN_STRONG_INLINE Packet8us pmax(const Packet8us& a, const Packet8us& b) { return vec_max(a, b); } template<> EIGEN_STRONG_INLINE Packet16c pmax(const Packet16c& a, const Packet16c& b) { return vec_max(a, b); } template<> EIGEN_STRONG_INLINE Packet16uc pmax(const Packet16uc& a, const Packet16uc& b) { return vec_max(a, b); } template<> EIGEN_STRONG_INLINE Packet4f pcmp_le(const Packet4f& a, const Packet4f& b) { return reinterpret_cast(vec_cmple(a,b)); } template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt(const Packet4f& a, const Packet4f& b) { return reinterpret_cast(vec_cmplt(a,b)); } template<> EIGEN_STRONG_INLINE Packet4f pcmp_eq(const Packet4f& a, const Packet4f& b) { return reinterpret_cast(vec_cmpeq(a,b)); } template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan(const Packet4f& a, const Packet4f& b) { Packet4f c = reinterpret_cast(vec_cmpge(a,b)); return vec_nor(c,c); } template<> EIGEN_STRONG_INLINE Packet4i pcmp_le(const Packet4i& a, const Packet4i& b) { return reinterpret_cast(vec_cmple(a,b)); } template<> EIGEN_STRONG_INLINE Packet4i pcmp_lt(const Packet4i& a, const Packet4i& b) { return reinterpret_cast(vec_cmplt(a,b)); } template<> EIGEN_STRONG_INLINE Packet4i pcmp_eq(const Packet4i& a, const Packet4i& b) { return reinterpret_cast(vec_cmpeq(a,b)); } template<> EIGEN_STRONG_INLINE Packet8s pcmp_le(const Packet8s& a, const Packet8s& b) { return reinterpret_cast(vec_cmple(a,b)); } template<> EIGEN_STRONG_INLINE Packet8s pcmp_lt(const Packet8s& a, const Packet8s& b) { return reinterpret_cast(vec_cmplt(a,b)); } template<> EIGEN_STRONG_INLINE Packet8s pcmp_eq(const Packet8s& a, const Packet8s& b) { return reinterpret_cast(vec_cmpeq(a,b)); } template<> EIGEN_STRONG_INLINE Packet8us pcmp_le(const Packet8us& a, const Packet8us& b) { return reinterpret_cast(vec_cmple(a,b)); } template<> EIGEN_STRONG_INLINE Packet8us pcmp_lt(const Packet8us& a, const Packet8us& b) { return reinterpret_cast(vec_cmplt(a,b)); } template<> EIGEN_STRONG_INLINE Packet8us pcmp_eq(const Packet8us& a, const Packet8us& b) { return reinterpret_cast(vec_cmpeq(a,b)); } template<> EIGEN_STRONG_INLINE Packet16c pcmp_le(const Packet16c& a, const Packet16c& b) { return reinterpret_cast(vec_cmple(a,b)); } template<> EIGEN_STRONG_INLINE Packet16c pcmp_lt(const Packet16c& a, const Packet16c& b) { return reinterpret_cast(vec_cmplt(a,b)); } template<> EIGEN_STRONG_INLINE Packet16c pcmp_eq(const Packet16c& a, const Packet16c& b) { return reinterpret_cast(vec_cmpeq(a,b)); } template<> EIGEN_STRONG_INLINE Packet16uc pcmp_le(const Packet16uc& a, const Packet16uc& b) { return reinterpret_cast(vec_cmple(a,b)); } template<> EIGEN_STRONG_INLINE Packet16uc pcmp_lt(const Packet16uc& a, const Packet16uc& b) { return reinterpret_cast(vec_cmplt(a,b)); } template<> EIGEN_STRONG_INLINE Packet16uc pcmp_eq(const Packet16uc& a, const Packet16uc& b) { return reinterpret_cast(vec_cmpeq(a,b)); } template<> EIGEN_STRONG_INLINE Packet4f pand(const Packet4f& a, const Packet4f& b) { return vec_and(a, b); } template<> EIGEN_STRONG_INLINE Packet4i pand(const Packet4i& a, const Packet4i& b) { return vec_and(a, b); } template<> EIGEN_STRONG_INLINE Packet4ui pand(const Packet4ui& a, const Packet4ui& b) { return vec_and(a, b); } template<> EIGEN_STRONG_INLINE Packet8us pand(const Packet8us& a, const Packet8us& b) { return vec_and(a, b); } template<> EIGEN_STRONG_INLINE Packet8bf pand(const Packet8bf& a, const Packet8bf& b) { return pand(a, b); } template<> EIGEN_STRONG_INLINE Packet4f por(const Packet4f& a, const Packet4f& b) { return vec_or(a, b); } template<> EIGEN_STRONG_INLINE Packet4i por(const Packet4i& a, const Packet4i& b) { return vec_or(a, b); } template<> EIGEN_STRONG_INLINE Packet8s por(const Packet8s& a, const Packet8s& b) { return vec_or(a, b); } template<> EIGEN_STRONG_INLINE Packet8us por(const Packet8us& a, const Packet8us& b) { return vec_or(a, b); } template<> EIGEN_STRONG_INLINE Packet8bf por(const Packet8bf& a, const Packet8bf& b) { return por(a, b); } template<> EIGEN_STRONG_INLINE Packet4f pxor(const Packet4f& a, const Packet4f& b) { return vec_xor(a, b); } template<> EIGEN_STRONG_INLINE Packet4i pxor(const Packet4i& a, const Packet4i& b) { return vec_xor(a, b); } template<> EIGEN_STRONG_INLINE Packet8bf pxor(const Packet8bf& a, const Packet8bf& b) { return pxor(a, b); } template<> EIGEN_STRONG_INLINE Packet4f pandnot(const Packet4f& a, const Packet4f& b) { return vec_andc(a, b); } template<> EIGEN_STRONG_INLINE Packet4i pandnot(const Packet4i& a, const Packet4i& b) { return vec_andc(a, b); } template<> EIGEN_STRONG_INLINE Packet4f pselect(const Packet4f& mask, const Packet4f& a, const Packet4f& b) { return vec_sel(b, a, reinterpret_cast(mask)); } template<> EIGEN_STRONG_INLINE Packet4f pround(const Packet4f& a) { Packet4f t = vec_add(reinterpret_cast(vec_or(vec_and(reinterpret_cast(a), p4ui_SIGN), p4ui_PREV0DOT5)), a); Packet4f res; #ifdef __VSX__ __asm__("xvrspiz %x0, %x1\n\t" : "=&wa" (res) : "wa" (t)); #else __asm__("vrfiz %0, %1\n\t" : "=v" (res) : "v" (t)); #endif return res; } template<> EIGEN_STRONG_INLINE Packet4f pceil(const Packet4f& a) { return vec_ceil(a); } template<> EIGEN_STRONG_INLINE Packet4f pfloor(const Packet4f& a) { return vec_floor(a); } template<> EIGEN_STRONG_INLINE Packet4f print(const Packet4f& a) { Packet4f res; __asm__("xvrspic %x0, %x1\n\t" : "=&wa" (res) : "wa" (a)); return res; } template EIGEN_STRONG_INLINE Packet ploadu_common(const __UNPACK_TYPE__(Packet)* from) { EIGEN_DEBUG_ALIGNED_LOAD #ifdef _BIG_ENDIAN Packet16uc MSQ, LSQ; Packet16uc mask; MSQ = vec_ld(0, (unsigned char *)from); // most significant quadword LSQ = vec_ld(15, (unsigned char *)from); // least significant quadword mask = vec_lvsl(0, from); // create the permute mask //TODO: Add static_cast here return (Packet) vec_perm(MSQ, LSQ, mask); // align the data #else EIGEN_DEBUG_UNALIGNED_LOAD return vec_xl(0, const_cast<__UNPACK_TYPE__(Packet)*>(from)); #endif } template<> EIGEN_STRONG_INLINE Packet4f ploadu(const float* from) { return ploadu_common(from); } template<> EIGEN_STRONG_INLINE Packet4i ploadu(const int* from) { return ploadu_common(from); } template<> EIGEN_STRONG_INLINE Packet8s ploadu(const short int* from) { return ploadu_common(from); } template<> EIGEN_STRONG_INLINE Packet8us ploadu(const unsigned short int* from) { return ploadu_common(from); } template<> EIGEN_STRONG_INLINE Packet8bf ploadu(const bfloat16* from) { return ploadu_common(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE Packet16c ploadu(const signed char* from) { return ploadu_common(from); } template<> EIGEN_STRONG_INLINE Packet16uc ploadu(const unsigned char* from) { return ploadu_common(from); } template EIGEN_STRONG_INLINE Packet ploaddup_common(const __UNPACK_TYPE__(Packet)* from) { Packet p; if((std::ptrdiff_t(from) % 16) == 0) p = pload(from); else p = ploadu(from); return vec_perm(p, p, p16uc_DUPLICATE32_HI); } template<> EIGEN_STRONG_INLINE Packet4f ploaddup(const float* from) { return ploaddup_common(from); } template<> EIGEN_STRONG_INLINE Packet4i ploaddup(const int* from) { return ploaddup_common(from); } template<> EIGEN_STRONG_INLINE Packet8s ploaddup(const short int* from) { Packet8s p; if((std::ptrdiff_t(from) % 16) == 0) p = pload(from); else p = ploadu(from); return vec_perm(p, p, p16uc_DUPLICATE16_HI); } template<> EIGEN_STRONG_INLINE Packet8us ploaddup(const unsigned short int* from) { Packet8us p; if((std::ptrdiff_t(from) % 16) == 0) p = pload(from); else p = ploadu(from); return vec_perm(p, p, p16uc_DUPLICATE16_HI); } template<> EIGEN_STRONG_INLINE Packet8s ploadquad(const short int* from) { Packet8s p; if((std::ptrdiff_t(from) % 16) == 0) p = pload(from); else p = ploadu(from); return vec_perm(p, p, p16uc_QUADRUPLICATE16_HI); } template<> EIGEN_STRONG_INLINE Packet8us ploadquad(const unsigned short int* from) { Packet8us p; if((std::ptrdiff_t(from) % 16) == 0) p = pload(from); else p = ploadu(from); return vec_perm(p, p, p16uc_QUADRUPLICATE16_HI); } template<> EIGEN_STRONG_INLINE Packet8bf ploadquad(const bfloat16* from) { return ploadquad(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE Packet16c ploaddup(const signed char* from) { Packet16c p; if((std::ptrdiff_t(from) % 16) == 0) p = pload(from); else p = ploadu(from); return vec_perm(p, p, p16uc_DUPLICATE8_HI); } template<> EIGEN_STRONG_INLINE Packet16uc ploaddup(const unsigned char* from) { Packet16uc p; if((std::ptrdiff_t(from) % 16) == 0) p = pload(from); else p = ploadu(from); return vec_perm(p, p, p16uc_DUPLICATE8_HI); } template EIGEN_STRONG_INLINE void pstoreu_common(__UNPACK_TYPE__(Packet)* to, const Packet& from) { EIGEN_DEBUG_UNALIGNED_STORE #ifdef _BIG_ENDIAN // Taken from http://developer.apple.com/hardwaredrivers/ve/alignment.html // Warning: not thread safe! Packet16uc MSQ, LSQ, edges; Packet16uc edgeAlign, align; MSQ = vec_ld(0, (unsigned char *)to); // most significant quadword LSQ = vec_ld(15, (unsigned char *)to); // least significant quadword edgeAlign = vec_lvsl(0, to); // permute map to extract edges edges=vec_perm(LSQ,MSQ,edgeAlign); // extract the edges align = vec_lvsr( 0, to ); // permute map to misalign data MSQ = vec_perm(edges,(Packet16uc)from,align); // misalign the data (MSQ) LSQ = vec_perm((Packet16uc)from,edges,align); // misalign the data (LSQ) vec_st( LSQ, 15, (unsigned char *)to ); // Store the LSQ part first vec_st( MSQ, 0, (unsigned char *)to ); // Store the MSQ part second #else vec_xst(from, 0, to); #endif } template<> EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet4f& from) { pstoreu_common(to, from); } template<> EIGEN_STRONG_INLINE void pstoreu(int* to, const Packet4i& from) { pstoreu_common(to, from); } template<> EIGEN_STRONG_INLINE void pstoreu(short int* to, const Packet8s& from) { pstoreu_common(to, from); } template<> EIGEN_STRONG_INLINE void pstoreu(unsigned short int* to, const Packet8us& from) { pstoreu_common(to, from); } template<> EIGEN_STRONG_INLINE void pstoreu(bfloat16* to, const Packet8bf& from) { pstoreu_common(reinterpret_cast(to), from); } template<> EIGEN_STRONG_INLINE void pstoreu(signed char* to, const Packet16c& from) { pstoreu_common(to, from); } template<> EIGEN_STRONG_INLINE void pstoreu(unsigned char* to, const Packet16uc& from) { pstoreu_common(to, from); } template<> EIGEN_STRONG_INLINE void prefetch(const float* addr) { EIGEN_PPC_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE void prefetch(const int* addr) { EIGEN_PPC_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE float pfirst(const Packet4f& a) { EIGEN_ALIGN16 float x; vec_ste(a, 0, &x); return x; } template<> EIGEN_STRONG_INLINE int pfirst(const Packet4i& a) { EIGEN_ALIGN16 int x; vec_ste(a, 0, &x); return x; } template EIGEN_STRONG_INLINE __UNPACK_TYPE__(Packet) pfirst_common(const Packet& a) { EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) x; vec_ste(a, 0, &x); return x; } template<> EIGEN_STRONG_INLINE short int pfirst(const Packet8s& a) { return pfirst_common(a); } template<> EIGEN_STRONG_INLINE unsigned short int pfirst(const Packet8us& a) { return pfirst_common(a); } template<> EIGEN_STRONG_INLINE signed char pfirst(const Packet16c& a) { return pfirst_common(a); } template<> EIGEN_STRONG_INLINE unsigned char pfirst(const Packet16uc& a) { return pfirst_common(a); } template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) { return reinterpret_cast(vec_perm(reinterpret_cast(a), reinterpret_cast(a), p16uc_REVERSE32)); } template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) { return reinterpret_cast(vec_perm(reinterpret_cast(a), reinterpret_cast(a), p16uc_REVERSE32)); } template<> EIGEN_STRONG_INLINE Packet8s preverse(const Packet8s& a) { return reinterpret_cast(vec_perm(reinterpret_cast(a), reinterpret_cast(a), p16uc_REVERSE16)); } template<> EIGEN_STRONG_INLINE Packet8us preverse(const Packet8us& a) { return reinterpret_cast(vec_perm(reinterpret_cast(a), reinterpret_cast(a), p16uc_REVERSE16)); } template<> EIGEN_STRONG_INLINE Packet16c preverse(const Packet16c& a) { return vec_perm(a, a, p16uc_REVERSE8); } template<> EIGEN_STRONG_INLINE Packet16uc preverse(const Packet16uc& a) { return vec_perm(a, a, p16uc_REVERSE8); } template<> EIGEN_STRONG_INLINE Packet8bf preverse(const Packet8bf& a) { return preverse(a); } template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { return vec_abs(a); } template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { return vec_abs(a); } template<> EIGEN_STRONG_INLINE Packet8s pabs(const Packet8s& a) { return vec_abs(a); } template<> EIGEN_STRONG_INLINE Packet8us pabs(const Packet8us& a) { return a; } template<> EIGEN_STRONG_INLINE Packet16c pabs(const Packet16c& a) { return vec_abs(a); } template<> EIGEN_STRONG_INLINE Packet16uc pabs(const Packet16uc& a) { return a; } template<> EIGEN_STRONG_INLINE Packet8bf pabs(const Packet8bf& a) { _EIGEN_DECLARE_CONST_FAST_Packet8us(abs_mask,0x7FFF); return pand(p8us_abs_mask, a); } template EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(const Packet4i& a) { return vec_sra(a,reinterpret_cast(pset1(N))); } template EIGEN_STRONG_INLINE Packet4i plogical_shift_right(const Packet4i& a) { return vec_sr(a,reinterpret_cast(pset1(N))); } template EIGEN_STRONG_INLINE Packet4i plogical_shift_left(const Packet4i& a) { return vec_sl(a,reinterpret_cast(pset1(N))); } template EIGEN_STRONG_INLINE Packet4f plogical_shift_left(const Packet4f& a) { const _EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N); Packet4ui r = vec_sl(reinterpret_cast(a), p4ui_mask); return reinterpret_cast(r); } template EIGEN_STRONG_INLINE Packet4f plogical_shift_right(const Packet4f& a) { const _EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N); Packet4ui r = vec_sr(reinterpret_cast(a), p4ui_mask); return reinterpret_cast(r); } template EIGEN_STRONG_INLINE Packet4ui plogical_shift_right(const Packet4ui& a) { const _EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N); return vec_sr(a, p4ui_mask); } template EIGEN_STRONG_INLINE Packet4ui plogical_shift_left(const Packet4ui& a) { const _EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N); return vec_sl(a, p4ui_mask); } template EIGEN_STRONG_INLINE Packet8us plogical_shift_left(const Packet8us& a) { const _EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N); return vec_sl(a, p8us_mask); } template EIGEN_STRONG_INLINE Packet8us plogical_shift_right(const Packet8us& a) { const _EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N); return vec_sr(a, p8us_mask); } EIGEN_STRONG_INLINE Packet4f Bf16ToF32Even(const Packet8bf& bf){ return plogical_shift_left<16>(reinterpret_cast(bf.m_val)); } EIGEN_STRONG_INLINE Packet4f Bf16ToF32Odd(const Packet8bf& bf){ const _EIGEN_DECLARE_CONST_FAST_Packet4ui(high_mask, 0xFFFF0000); return pand( reinterpret_cast(bf.m_val), reinterpret_cast(p4ui_high_mask) ); } // Simple interleaving of bool masks, prevents true values from being // converted to NaNs. EIGEN_STRONG_INLINE Packet8bf F32ToBf16Bool(Packet4f even, Packet4f odd) { const _EIGEN_DECLARE_CONST_FAST_Packet4ui(high_mask, 0xFFFF0000); Packet4f bf_odd, bf_even; bf_odd = pand(reinterpret_cast(p4ui_high_mask), odd); bf_even = plogical_shift_right<16>(even); return reinterpret_cast(por(bf_even, bf_odd)); } EIGEN_STRONG_INLINE Packet8bf F32ToBf16(Packet4f p4f){ Packet4ui input = reinterpret_cast(p4f); Packet4ui lsb = plogical_shift_right<16>(input); lsb = pand(lsb, reinterpret_cast(p4i_ONE)); _EIGEN_DECLARE_CONST_FAST_Packet4ui(BIAS,0x7FFFu); Packet4ui rounding_bias = padd(lsb, p4ui_BIAS); input = padd(input, rounding_bias); //Test NaN and Subnormal - Begin const _EIGEN_DECLARE_CONST_FAST_Packet4ui(exp_mask, 0x7F800000); Packet4ui exp = pand(p4ui_exp_mask, reinterpret_cast(p4f)); const _EIGEN_DECLARE_CONST_FAST_Packet4ui(mantissa_mask, 0x7FFFFF); Packet4ui mantissa = pand(p4ui_mantissa_mask, reinterpret_cast(p4f)); const _EIGEN_DECLARE_CONST_FAST_Packet4ui(max_exp, 0x7F800000); Packet4bi is_max_exp = vec_cmpeq(exp, p4ui_max_exp); Packet4bi is_zero_exp = vec_cmpeq(exp, reinterpret_cast(p4i_ZERO)); Packet4bi is_mant_zero = vec_cmpeq(mantissa, reinterpret_cast(p4i_ZERO)); Packet4ui nan_selector = pandnot( reinterpret_cast(is_max_exp), reinterpret_cast(is_mant_zero) ); Packet4ui subnormal_selector = pandnot( reinterpret_cast(is_zero_exp), reinterpret_cast(is_mant_zero) ); const _EIGEN_DECLARE_CONST_FAST_Packet4ui(nan, 0x7FC00000); input = vec_sel(input, p4ui_nan, nan_selector); input = vec_sel(input, reinterpret_cast(p4f), subnormal_selector); //Test NaN and Subnormal - End input = plogical_shift_right<16>(input); return reinterpret_cast(input); } EIGEN_STRONG_INLINE Packet8bf F32ToBf16(Packet4f even, Packet4f odd){ Packet4f bf_odd, bf_even; bf_odd = reinterpret_cast(F32ToBf16(odd).m_val); bf_odd = plogical_shift_left<16>(bf_odd); bf_even = reinterpret_cast(F32ToBf16(even).m_val); return reinterpret_cast(por(bf_even, bf_odd)); } #define BF16_TO_F32_UNARY_OP_WRAPPER(OP, A) \ Packet4f a_even = Bf16ToF32Even(A);\ Packet4f a_odd = Bf16ToF32Odd(A);\ Packet4f op_even = OP(a_even);\ Packet4f op_odd = OP(a_odd);\ return F32ToBf16(op_even, op_odd);\ #define BF16_TO_F32_BINARY_OP_WRAPPER(OP, A, B) \ Packet4f a_even = Bf16ToF32Even(A);\ Packet4f a_odd = Bf16ToF32Odd(A);\ Packet4f b_even = Bf16ToF32Even(B);\ Packet4f b_odd = Bf16ToF32Odd(B);\ Packet4f op_even = OP(a_even, b_even);\ Packet4f op_odd = OP(a_odd, b_odd);\ return F32ToBf16(op_even, op_odd);\ #define BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(OP, A, B) \ Packet4f a_even = Bf16ToF32Even(A);\ Packet4f a_odd = Bf16ToF32Odd(A);\ Packet4f b_even = Bf16ToF32Even(B);\ Packet4f b_odd = Bf16ToF32Odd(B);\ Packet4f op_even = OP(a_even, b_even);\ Packet4f op_odd = OP(a_odd, b_odd);\ return F32ToBf16Bool(op_even, op_odd);\ template<> EIGEN_STRONG_INLINE Packet8bf padd(const Packet8bf& a, const Packet8bf& b) { BF16_TO_F32_BINARY_OP_WRAPPER(padd, a, b); } template<> EIGEN_STRONG_INLINE Packet8bf pmul(const Packet8bf& a, const Packet8bf& b) { BF16_TO_F32_BINARY_OP_WRAPPER(pmul, a, b); } template<> EIGEN_STRONG_INLINE Packet8bf pdiv(const Packet8bf& a, const Packet8bf& b) { BF16_TO_F32_BINARY_OP_WRAPPER(pdiv, a, b); } template<> EIGEN_STRONG_INLINE Packet8bf pnegate(const Packet8bf& a) { BF16_TO_F32_UNARY_OP_WRAPPER(pnegate, a); } template<> EIGEN_STRONG_INLINE Packet8bf psub(const Packet8bf& a, const Packet8bf& b) { BF16_TO_F32_BINARY_OP_WRAPPER(psub, a, b); } template<> EIGEN_STRONG_INLINE Packet8bf psqrt (const Packet8bf& a){ BF16_TO_F32_UNARY_OP_WRAPPER(vec_sqrt, a); } template<> EIGEN_STRONG_INLINE Packet8bf prsqrt (const Packet8bf& a){ BF16_TO_F32_UNARY_OP_WRAPPER(prsqrt, a); } template<> EIGEN_STRONG_INLINE Packet8bf pexp (const Packet8bf& a){ BF16_TO_F32_UNARY_OP_WRAPPER(pexp_float, a); } template<> EIGEN_STRONG_INLINE Packet4f pldexp(const Packet4f& a, const Packet4f& exponent) { return pldexp_generic(a,exponent); } template<> EIGEN_STRONG_INLINE Packet8bf pldexp (const Packet8bf& a, const Packet8bf& exponent){ BF16_TO_F32_BINARY_OP_WRAPPER(pldexp, a, exponent); } template<> EIGEN_STRONG_INLINE Packet4f pfrexp(const Packet4f& a, Packet4f& exponent) { return pfrexp_generic(a,exponent); } template<> EIGEN_STRONG_INLINE Packet8bf pfrexp (const Packet8bf& a, Packet8bf& e){ Packet4f a_even = Bf16ToF32Even(a); Packet4f a_odd = Bf16ToF32Odd(a); Packet4f e_even; Packet4f e_odd; Packet4f op_even = pfrexp(a_even, e_even); Packet4f op_odd = pfrexp(a_odd, e_odd); e = F32ToBf16(e_even, e_odd); return F32ToBf16(op_even, op_odd); } template<> EIGEN_STRONG_INLINE Packet8bf psin (const Packet8bf& a){ BF16_TO_F32_UNARY_OP_WRAPPER(psin_float, a); } template<> EIGEN_STRONG_INLINE Packet8bf pcos (const Packet8bf& a){ BF16_TO_F32_UNARY_OP_WRAPPER(pcos_float, a); } template<> EIGEN_STRONG_INLINE Packet8bf plog (const Packet8bf& a){ BF16_TO_F32_UNARY_OP_WRAPPER(plog_float, a); } template<> EIGEN_STRONG_INLINE Packet8bf pfloor (const Packet8bf& a){ BF16_TO_F32_UNARY_OP_WRAPPER(pfloor, a); } template<> EIGEN_STRONG_INLINE Packet8bf pceil (const Packet8bf& a){ BF16_TO_F32_UNARY_OP_WRAPPER(pceil, a); } template<> EIGEN_STRONG_INLINE Packet8bf pround (const Packet8bf& a){ BF16_TO_F32_UNARY_OP_WRAPPER(pround, a); } template<> EIGEN_STRONG_INLINE Packet8bf print (const Packet8bf& a){ BF16_TO_F32_UNARY_OP_WRAPPER(print, a); } template<> EIGEN_STRONG_INLINE Packet8bf pmadd(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) { Packet4f a_even = Bf16ToF32Even(a); Packet4f a_odd = Bf16ToF32Odd(a); Packet4f b_even = Bf16ToF32Even(b); Packet4f b_odd = Bf16ToF32Odd(b); Packet4f c_even = Bf16ToF32Even(c); Packet4f c_odd = Bf16ToF32Odd(c); Packet4f pmadd_even = pmadd(a_even, b_even, c_even); Packet4f pmadd_odd = pmadd(a_odd, b_odd, c_odd); return F32ToBf16(pmadd_even, pmadd_odd); } template<> EIGEN_STRONG_INLINE Packet8bf pmin(const Packet8bf& a, const Packet8bf& b) { BF16_TO_F32_BINARY_OP_WRAPPER(pmin, a, b); } template<> EIGEN_STRONG_INLINE Packet8bf pmax(const Packet8bf& a, const Packet8bf& b) { BF16_TO_F32_BINARY_OP_WRAPPER(pmax, a, b); } template<> EIGEN_STRONG_INLINE Packet8bf pcmp_lt(const Packet8bf& a, const Packet8bf& b) { BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_lt, a, b); } template<> EIGEN_STRONG_INLINE Packet8bf pcmp_lt_or_nan(const Packet8bf& a, const Packet8bf& b) { BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_lt_or_nan, a, b); } template<> EIGEN_STRONG_INLINE Packet8bf pcmp_le(const Packet8bf& a, const Packet8bf& b) { BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_le, a, b); } template<> EIGEN_STRONG_INLINE Packet8bf pcmp_eq(const Packet8bf& a, const Packet8bf& b) { BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_eq, a, b); } template<> EIGEN_STRONG_INLINE bfloat16 pfirst(const Packet8bf& a) { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16((pfirst(a))); } template<> EIGEN_STRONG_INLINE Packet8bf ploaddup(const bfloat16* from) { return ploaddup(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE Packet8bf plset(const bfloat16& a) { bfloat16 countdown[8] = { bfloat16(0), bfloat16(1), bfloat16(2), bfloat16(3), bfloat16(4), bfloat16(5), bfloat16(6), bfloat16(7) }; return padd(pset1(a), pload(countdown)); } template<> EIGEN_STRONG_INLINE float predux(const Packet4f& a) { Packet4f b, sum; b = vec_sld(a, a, 8); sum = a + b; b = vec_sld(sum, sum, 4); sum += b; return pfirst(sum); } template<> EIGEN_STRONG_INLINE int predux(const Packet4i& a) { Packet4i sum; sum = vec_sums(a, p4i_ZERO); #ifdef _BIG_ENDIAN sum = vec_sld(sum, p4i_ZERO, 12); #else sum = vec_sld(p4i_ZERO, sum, 4); #endif return pfirst(sum); } template<> EIGEN_STRONG_INLINE bfloat16 predux(const Packet8bf& a) { float redux_even = predux(Bf16ToF32Even(a)); float redux_odd = predux(Bf16ToF32Odd(a)); float f32_result = redux_even + redux_odd; return bfloat16(f32_result); } template EIGEN_STRONG_INLINE __UNPACK_TYPE__(Packet) predux_size8(const Packet& a) { union{ Packet v; __UNPACK_TYPE__(Packet) n[8]; } vt; vt.v = a; EIGEN_ALIGN16 int first_loader[4] = { vt.n[0], vt.n[1], vt.n[2], vt.n[3] }; EIGEN_ALIGN16 int second_loader[4] = { vt.n[4], vt.n[5], vt.n[6], vt.n[7] }; Packet4i first_half = pload(first_loader); Packet4i second_half = pload(second_loader); return static_cast<__UNPACK_TYPE__(Packet)>(predux(first_half) + predux(second_half)); } template<> EIGEN_STRONG_INLINE short int predux(const Packet8s& a) { return predux_size8(a); } template<> EIGEN_STRONG_INLINE unsigned short int predux(const Packet8us& a) { return predux_size8(a); } template EIGEN_STRONG_INLINE __UNPACK_TYPE__(Packet) predux_size16(const Packet& a) { union{ Packet v; __UNPACK_TYPE__(Packet) n[16]; } vt; vt.v = a; EIGEN_ALIGN16 int first_loader[4] = { vt.n[0], vt.n[1], vt.n[2], vt.n[3] }; EIGEN_ALIGN16 int second_loader[4] = { vt.n[4], vt.n[5], vt.n[6], vt.n[7] }; EIGEN_ALIGN16 int third_loader[4] = { vt.n[8], vt.n[9], vt.n[10], vt.n[11] }; EIGEN_ALIGN16 int fourth_loader[4] = { vt.n[12], vt.n[13], vt.n[14], vt.n[15] }; Packet4i first_quarter = pload(first_loader); Packet4i second_quarter = pload(second_loader); Packet4i third_quarter = pload(third_loader); Packet4i fourth_quarter = pload(fourth_loader); return static_cast<__UNPACK_TYPE__(Packet)>(predux(first_quarter) + predux(second_quarter) + predux(third_quarter) + predux(fourth_quarter)); } template<> EIGEN_STRONG_INLINE signed char predux(const Packet16c& a) { return predux_size16(a); } template<> EIGEN_STRONG_INLINE unsigned char predux(const Packet16uc& a) { return predux_size16(a); } // Other reduction functions: // mul template<> EIGEN_STRONG_INLINE float predux_mul(const Packet4f& a) { Packet4f prod; prod = pmul(a, vec_sld(a, a, 8)); return pfirst(pmul(prod, vec_sld(prod, prod, 4))); } template<> EIGEN_STRONG_INLINE int predux_mul(const Packet4i& a) { EIGEN_ALIGN16 int aux[4]; pstore(aux, a); return aux[0] * aux[1] * aux[2] * aux[3]; } template<> EIGEN_STRONG_INLINE short int predux_mul(const Packet8s& a) { Packet8s pair, quad, octo; pair = vec_mul(a, vec_sld(a, a, 8)); quad = vec_mul(pair, vec_sld(pair, pair, 4)); octo = vec_mul(quad, vec_sld(quad, quad, 2)); return pfirst(octo); } template<> EIGEN_STRONG_INLINE unsigned short int predux_mul(const Packet8us& a) { Packet8us pair, quad, octo; pair = vec_mul(a, vec_sld(a, a, 8)); quad = vec_mul(pair, vec_sld(pair, pair, 4)); octo = vec_mul(quad, vec_sld(quad, quad, 2)); return pfirst(octo); } template<> EIGEN_STRONG_INLINE bfloat16 predux_mul(const Packet8bf& a) { float redux_even = predux_mul(Bf16ToF32Even(a)); float redux_odd = predux_mul(Bf16ToF32Odd(a)); float f32_result = redux_even * redux_odd; return bfloat16(f32_result); } template<> EIGEN_STRONG_INLINE signed char predux_mul(const Packet16c& a) { Packet16c pair, quad, octo, result; pair = vec_mul(a, vec_sld(a, a, 8)); quad = vec_mul(pair, vec_sld(pair, pair, 4)); octo = vec_mul(quad, vec_sld(quad, quad, 2)); result = vec_mul(octo, vec_sld(octo, octo, 1)); return pfirst(result); } template<> EIGEN_STRONG_INLINE unsigned char predux_mul(const Packet16uc& a) { Packet16uc pair, quad, octo, result; pair = vec_mul(a, vec_sld(a, a, 8)); quad = vec_mul(pair, vec_sld(pair, pair, 4)); octo = vec_mul(quad, vec_sld(quad, quad, 2)); result = vec_mul(octo, vec_sld(octo, octo, 1)); return pfirst(result); } // min template EIGEN_STRONG_INLINE __UNPACK_TYPE__(Packet) predux_min4(const Packet& a) { Packet b, res; b = vec_min(a, vec_sld(a, a, 8)); res = vec_min(b, vec_sld(b, b, 4)); return pfirst(res); } template<> EIGEN_STRONG_INLINE float predux_min(const Packet4f& a) { return predux_min4(a); } template<> EIGEN_STRONG_INLINE int predux_min(const Packet4i& a) { return predux_min4(a); } template<> EIGEN_STRONG_INLINE bfloat16 predux_min(const Packet8bf& a) { float redux_even = predux_min(Bf16ToF32Even(a)); float redux_odd = predux_min(Bf16ToF32Odd(a)); float f32_result = (std::min)(redux_even, redux_odd); return bfloat16(f32_result); } template<> EIGEN_STRONG_INLINE short int predux_min(const Packet8s& a) { Packet8s pair, quad, octo; //pair = { Min(a0,a4), Min(a1,a5), Min(a2,a6), Min(a3,a7) } pair = vec_min(a, vec_sld(a, a, 8)); //quad = { Min(a0, a4, a2, a6), Min(a1, a5, a3, a7) } quad = vec_min(pair, vec_sld(pair, pair, 4)); //octo = { Min(a0, a4, a2, a6, a1, a5, a3, a7) } octo = vec_min(quad, vec_sld(quad, quad, 2)); return pfirst(octo); } template<> EIGEN_STRONG_INLINE unsigned short int predux_min(const Packet8us& a) { Packet8us pair, quad, octo; //pair = { Min(a0,a4), Min(a1,a5), Min(a2,a6), Min(a3,a7) } pair = vec_min(a, vec_sld(a, a, 8)); //quad = { Min(a0, a4, a2, a6), Min(a1, a5, a3, a7) } quad = vec_min(pair, vec_sld(pair, pair, 4)); //octo = { Min(a0, a4, a2, a6, a1, a5, a3, a7) } octo = vec_min(quad, vec_sld(quad, quad, 2)); return pfirst(octo); } template<> EIGEN_STRONG_INLINE signed char predux_min(const Packet16c& a) { Packet16c pair, quad, octo, result; pair = vec_min(a, vec_sld(a, a, 8)); quad = vec_min(pair, vec_sld(pair, pair, 4)); octo = vec_min(quad, vec_sld(quad, quad, 2)); result = vec_min(octo, vec_sld(octo, octo, 1)); return pfirst(result); } template<> EIGEN_STRONG_INLINE unsigned char predux_min(const Packet16uc& a) { Packet16uc pair, quad, octo, result; pair = vec_min(a, vec_sld(a, a, 8)); quad = vec_min(pair, vec_sld(pair, pair, 4)); octo = vec_min(quad, vec_sld(quad, quad, 2)); result = vec_min(octo, vec_sld(octo, octo, 1)); return pfirst(result); } // max template EIGEN_STRONG_INLINE __UNPACK_TYPE__(Packet) predux_max4(const Packet& a) { Packet b, res; b = vec_max(a, vec_sld(a, a, 8)); res = vec_max(b, vec_sld(b, b, 4)); return pfirst(res); } template<> EIGEN_STRONG_INLINE float predux_max(const Packet4f& a) { return predux_max4(a); } template<> EIGEN_STRONG_INLINE int predux_max(const Packet4i& a) { return predux_max4(a); } template<> EIGEN_STRONG_INLINE bfloat16 predux_max(const Packet8bf& a) { float redux_even = predux_max(Bf16ToF32Even(a)); float redux_odd = predux_max(Bf16ToF32Odd(a)); float f32_result = (std::max)(redux_even, redux_odd); return bfloat16(f32_result); } template<> EIGEN_STRONG_INLINE short int predux_max(const Packet8s& a) { Packet8s pair, quad, octo; //pair = { Max(a0,a4), Max(a1,a5), Max(a2,a6), Max(a3,a7) } pair = vec_max(a, vec_sld(a, a, 8)); //quad = { Max(a0, a4, a2, a6), Max(a1, a5, a3, a7) } quad = vec_max(pair, vec_sld(pair, pair, 4)); //octo = { Max(a0, a4, a2, a6, a1, a5, a3, a7) } octo = vec_max(quad, vec_sld(quad, quad, 2)); return pfirst(octo); } template<> EIGEN_STRONG_INLINE unsigned short int predux_max(const Packet8us& a) { Packet8us pair, quad, octo; //pair = { Max(a0,a4), Max(a1,a5), Max(a2,a6), Max(a3,a7) } pair = vec_max(a, vec_sld(a, a, 8)); //quad = { Max(a0, a4, a2, a6), Max(a1, a5, a3, a7) } quad = vec_max(pair, vec_sld(pair, pair, 4)); //octo = { Max(a0, a4, a2, a6, a1, a5, a3, a7) } octo = vec_max(quad, vec_sld(quad, quad, 2)); return pfirst(octo); } template<> EIGEN_STRONG_INLINE signed char predux_max(const Packet16c& a) { Packet16c pair, quad, octo, result; pair = vec_max(a, vec_sld(a, a, 8)); quad = vec_max(pair, vec_sld(pair, pair, 4)); octo = vec_max(quad, vec_sld(quad, quad, 2)); result = vec_max(octo, vec_sld(octo, octo, 1)); return pfirst(result); } template<> EIGEN_STRONG_INLINE unsigned char predux_max(const Packet16uc& a) { Packet16uc pair, quad, octo, result; pair = vec_max(a, vec_sld(a, a, 8)); quad = vec_max(pair, vec_sld(pair, pair, 4)); octo = vec_max(quad, vec_sld(quad, quad, 2)); result = vec_max(octo, vec_sld(octo, octo, 1)); return pfirst(result); } template<> EIGEN_STRONG_INLINE bool predux_any(const Packet4f& x) { return vec_any_ne(x, pzero(x)); } template EIGEN_DEVICE_FUNC inline void ptranpose_common(PacketBlock& kernel){ T t0, t1, t2, t3; t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]); t1 = vec_mergel(kernel.packet[0], kernel.packet[2]); t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]); t3 = vec_mergel(kernel.packet[1], kernel.packet[3]); kernel.packet[0] = vec_mergeh(t0, t2); kernel.packet[1] = vec_mergel(t0, t2); kernel.packet[2] = vec_mergeh(t1, t3); kernel.packet[3] = vec_mergel(t1, t3); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { ptranpose_common(kernel); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { ptranpose_common(kernel); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { Packet8s t0, t1, t2, t3; t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]); t1 = vec_mergel(kernel.packet[0], kernel.packet[2]); t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]); t3 = vec_mergel(kernel.packet[1], kernel.packet[3]); kernel.packet[0] = vec_mergeh(t0, t2); kernel.packet[1] = vec_mergel(t0, t2); kernel.packet[2] = vec_mergeh(t1, t3); kernel.packet[3] = vec_mergel(t1, t3); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { Packet8us t0, t1, t2, t3; t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]); t1 = vec_mergel(kernel.packet[0], kernel.packet[2]); t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]); t3 = vec_mergel(kernel.packet[1], kernel.packet[3]); kernel.packet[0] = vec_mergeh(t0, t2); kernel.packet[1] = vec_mergel(t0, t2); kernel.packet[2] = vec_mergeh(t1, t3); kernel.packet[3] = vec_mergel(t1, t3); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { Packet8us t0, t1, t2, t3; t0 = vec_mergeh(kernel.packet[0].m_val, kernel.packet[2].m_val); t1 = vec_mergel(kernel.packet[0].m_val, kernel.packet[2].m_val); t2 = vec_mergeh(kernel.packet[1].m_val, kernel.packet[3].m_val); t3 = vec_mergel(kernel.packet[1].m_val, kernel.packet[3].m_val); kernel.packet[0] = vec_mergeh(t0, t2); kernel.packet[1] = vec_mergel(t0, t2); kernel.packet[2] = vec_mergeh(t1, t3); kernel.packet[3] = vec_mergel(t1, t3); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { Packet16c t0, t1, t2, t3; t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]); t1 = vec_mergel(kernel.packet[0], kernel.packet[2]); t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]); t3 = vec_mergel(kernel.packet[1], kernel.packet[3]); kernel.packet[0] = vec_mergeh(t0, t2); kernel.packet[1] = vec_mergel(t0, t2); kernel.packet[2] = vec_mergeh(t1, t3); kernel.packet[3] = vec_mergel(t1, t3); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { Packet16uc t0, t1, t2, t3; t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]); t1 = vec_mergel(kernel.packet[0], kernel.packet[2]); t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]); t3 = vec_mergel(kernel.packet[1], kernel.packet[3]); kernel.packet[0] = vec_mergeh(t0, t2); kernel.packet[1] = vec_mergel(t0, t2); kernel.packet[2] = vec_mergeh(t1, t3); kernel.packet[3] = vec_mergel(t1, t3); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { Packet8s v[8], sum[8]; v[0] = vec_mergeh(kernel.packet[0], kernel.packet[4]); v[1] = vec_mergel(kernel.packet[0], kernel.packet[4]); v[2] = vec_mergeh(kernel.packet[1], kernel.packet[5]); v[3] = vec_mergel(kernel.packet[1], kernel.packet[5]); v[4] = vec_mergeh(kernel.packet[2], kernel.packet[6]); v[5] = vec_mergel(kernel.packet[2], kernel.packet[6]); v[6] = vec_mergeh(kernel.packet[3], kernel.packet[7]); v[7] = vec_mergel(kernel.packet[3], kernel.packet[7]); sum[0] = vec_mergeh(v[0], v[4]); sum[1] = vec_mergel(v[0], v[4]); sum[2] = vec_mergeh(v[1], v[5]); sum[3] = vec_mergel(v[1], v[5]); sum[4] = vec_mergeh(v[2], v[6]); sum[5] = vec_mergel(v[2], v[6]); sum[6] = vec_mergeh(v[3], v[7]); sum[7] = vec_mergel(v[3], v[7]); kernel.packet[0] = vec_mergeh(sum[0], sum[4]); kernel.packet[1] = vec_mergel(sum[0], sum[4]); kernel.packet[2] = vec_mergeh(sum[1], sum[5]); kernel.packet[3] = vec_mergel(sum[1], sum[5]); kernel.packet[4] = vec_mergeh(sum[2], sum[6]); kernel.packet[5] = vec_mergel(sum[2], sum[6]); kernel.packet[6] = vec_mergeh(sum[3], sum[7]); kernel.packet[7] = vec_mergel(sum[3], sum[7]); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { Packet8us v[8], sum[8]; v[0] = vec_mergeh(kernel.packet[0], kernel.packet[4]); v[1] = vec_mergel(kernel.packet[0], kernel.packet[4]); v[2] = vec_mergeh(kernel.packet[1], kernel.packet[5]); v[3] = vec_mergel(kernel.packet[1], kernel.packet[5]); v[4] = vec_mergeh(kernel.packet[2], kernel.packet[6]); v[5] = vec_mergel(kernel.packet[2], kernel.packet[6]); v[6] = vec_mergeh(kernel.packet[3], kernel.packet[7]); v[7] = vec_mergel(kernel.packet[3], kernel.packet[7]); sum[0] = vec_mergeh(v[0], v[4]); sum[1] = vec_mergel(v[0], v[4]); sum[2] = vec_mergeh(v[1], v[5]); sum[3] = vec_mergel(v[1], v[5]); sum[4] = vec_mergeh(v[2], v[6]); sum[5] = vec_mergel(v[2], v[6]); sum[6] = vec_mergeh(v[3], v[7]); sum[7] = vec_mergel(v[3], v[7]); kernel.packet[0] = vec_mergeh(sum[0], sum[4]); kernel.packet[1] = vec_mergel(sum[0], sum[4]); kernel.packet[2] = vec_mergeh(sum[1], sum[5]); kernel.packet[3] = vec_mergel(sum[1], sum[5]); kernel.packet[4] = vec_mergeh(sum[2], sum[6]); kernel.packet[5] = vec_mergel(sum[2], sum[6]); kernel.packet[6] = vec_mergeh(sum[3], sum[7]); kernel.packet[7] = vec_mergel(sum[3], sum[7]); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { Packet8bf v[8], sum[8]; v[0] = vec_mergeh(kernel.packet[0].m_val, kernel.packet[4].m_val); v[1] = vec_mergel(kernel.packet[0].m_val, kernel.packet[4].m_val); v[2] = vec_mergeh(kernel.packet[1].m_val, kernel.packet[5].m_val); v[3] = vec_mergel(kernel.packet[1].m_val, kernel.packet[5].m_val); v[4] = vec_mergeh(kernel.packet[2].m_val, kernel.packet[6].m_val); v[5] = vec_mergel(kernel.packet[2].m_val, kernel.packet[6].m_val); v[6] = vec_mergeh(kernel.packet[3].m_val, kernel.packet[7].m_val); v[7] = vec_mergel(kernel.packet[3].m_val, kernel.packet[7].m_val); sum[0] = vec_mergeh(v[0].m_val, v[4].m_val); sum[1] = vec_mergel(v[0].m_val, v[4].m_val); sum[2] = vec_mergeh(v[1].m_val, v[5].m_val); sum[3] = vec_mergel(v[1].m_val, v[5].m_val); sum[4] = vec_mergeh(v[2].m_val, v[6].m_val); sum[5] = vec_mergel(v[2].m_val, v[6].m_val); sum[6] = vec_mergeh(v[3].m_val, v[7].m_val); sum[7] = vec_mergel(v[3].m_val, v[7].m_val); kernel.packet[0] = vec_mergeh(sum[0].m_val, sum[4].m_val); kernel.packet[1] = vec_mergel(sum[0].m_val, sum[4].m_val); kernel.packet[2] = vec_mergeh(sum[1].m_val, sum[5].m_val); kernel.packet[3] = vec_mergel(sum[1].m_val, sum[5].m_val); kernel.packet[4] = vec_mergeh(sum[2].m_val, sum[6].m_val); kernel.packet[5] = vec_mergel(sum[2].m_val, sum[6].m_val); kernel.packet[6] = vec_mergeh(sum[3].m_val, sum[7].m_val); kernel.packet[7] = vec_mergel(sum[3].m_val, sum[7].m_val); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { Packet16c step1[16], step2[16], step3[16]; step1[0] = vec_mergeh(kernel.packet[0], kernel.packet[8]); step1[1] = vec_mergel(kernel.packet[0], kernel.packet[8]); step1[2] = vec_mergeh(kernel.packet[1], kernel.packet[9]); step1[3] = vec_mergel(kernel.packet[1], kernel.packet[9]); step1[4] = vec_mergeh(kernel.packet[2], kernel.packet[10]); step1[5] = vec_mergel(kernel.packet[2], kernel.packet[10]); step1[6] = vec_mergeh(kernel.packet[3], kernel.packet[11]); step1[7] = vec_mergel(kernel.packet[3], kernel.packet[11]); step1[8] = vec_mergeh(kernel.packet[4], kernel.packet[12]); step1[9] = vec_mergel(kernel.packet[4], kernel.packet[12]); step1[10] = vec_mergeh(kernel.packet[5], kernel.packet[13]); step1[11] = vec_mergel(kernel.packet[5], kernel.packet[13]); step1[12] = vec_mergeh(kernel.packet[6], kernel.packet[14]); step1[13] = vec_mergel(kernel.packet[6], kernel.packet[14]); step1[14] = vec_mergeh(kernel.packet[7], kernel.packet[15]); step1[15] = vec_mergel(kernel.packet[7], kernel.packet[15]); step2[0] = vec_mergeh(step1[0], step1[8]); step2[1] = vec_mergel(step1[0], step1[8]); step2[2] = vec_mergeh(step1[1], step1[9]); step2[3] = vec_mergel(step1[1], step1[9]); step2[4] = vec_mergeh(step1[2], step1[10]); step2[5] = vec_mergel(step1[2], step1[10]); step2[6] = vec_mergeh(step1[3], step1[11]); step2[7] = vec_mergel(step1[3], step1[11]); step2[8] = vec_mergeh(step1[4], step1[12]); step2[9] = vec_mergel(step1[4], step1[12]); step2[10] = vec_mergeh(step1[5], step1[13]); step2[11] = vec_mergel(step1[5], step1[13]); step2[12] = vec_mergeh(step1[6], step1[14]); step2[13] = vec_mergel(step1[6], step1[14]); step2[14] = vec_mergeh(step1[7], step1[15]); step2[15] = vec_mergel(step1[7], step1[15]); step3[0] = vec_mergeh(step2[0], step2[8]); step3[1] = vec_mergel(step2[0], step2[8]); step3[2] = vec_mergeh(step2[1], step2[9]); step3[3] = vec_mergel(step2[1], step2[9]); step3[4] = vec_mergeh(step2[2], step2[10]); step3[5] = vec_mergel(step2[2], step2[10]); step3[6] = vec_mergeh(step2[3], step2[11]); step3[7] = vec_mergel(step2[3], step2[11]); step3[8] = vec_mergeh(step2[4], step2[12]); step3[9] = vec_mergel(step2[4], step2[12]); step3[10] = vec_mergeh(step2[5], step2[13]); step3[11] = vec_mergel(step2[5], step2[13]); step3[12] = vec_mergeh(step2[6], step2[14]); step3[13] = vec_mergel(step2[6], step2[14]); step3[14] = vec_mergeh(step2[7], step2[15]); step3[15] = vec_mergel(step2[7], step2[15]); kernel.packet[0] = vec_mergeh(step3[0], step3[8]); kernel.packet[1] = vec_mergel(step3[0], step3[8]); kernel.packet[2] = vec_mergeh(step3[1], step3[9]); kernel.packet[3] = vec_mergel(step3[1], step3[9]); kernel.packet[4] = vec_mergeh(step3[2], step3[10]); kernel.packet[5] = vec_mergel(step3[2], step3[10]); kernel.packet[6] = vec_mergeh(step3[3], step3[11]); kernel.packet[7] = vec_mergel(step3[3], step3[11]); kernel.packet[8] = vec_mergeh(step3[4], step3[12]); kernel.packet[9] = vec_mergel(step3[4], step3[12]); kernel.packet[10] = vec_mergeh(step3[5], step3[13]); kernel.packet[11] = vec_mergel(step3[5], step3[13]); kernel.packet[12] = vec_mergeh(step3[6], step3[14]); kernel.packet[13] = vec_mergel(step3[6], step3[14]); kernel.packet[14] = vec_mergeh(step3[7], step3[15]); kernel.packet[15] = vec_mergel(step3[7], step3[15]); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { Packet16uc step1[16], step2[16], step3[16]; step1[0] = vec_mergeh(kernel.packet[0], kernel.packet[8]); step1[1] = vec_mergel(kernel.packet[0], kernel.packet[8]); step1[2] = vec_mergeh(kernel.packet[1], kernel.packet[9]); step1[3] = vec_mergel(kernel.packet[1], kernel.packet[9]); step1[4] = vec_mergeh(kernel.packet[2], kernel.packet[10]); step1[5] = vec_mergel(kernel.packet[2], kernel.packet[10]); step1[6] = vec_mergeh(kernel.packet[3], kernel.packet[11]); step1[7] = vec_mergel(kernel.packet[3], kernel.packet[11]); step1[8] = vec_mergeh(kernel.packet[4], kernel.packet[12]); step1[9] = vec_mergel(kernel.packet[4], kernel.packet[12]); step1[10] = vec_mergeh(kernel.packet[5], kernel.packet[13]); step1[11] = vec_mergel(kernel.packet[5], kernel.packet[13]); step1[12] = vec_mergeh(kernel.packet[6], kernel.packet[14]); step1[13] = vec_mergel(kernel.packet[6], kernel.packet[14]); step1[14] = vec_mergeh(kernel.packet[7], kernel.packet[15]); step1[15] = vec_mergel(kernel.packet[7], kernel.packet[15]); step2[0] = vec_mergeh(step1[0], step1[8]); step2[1] = vec_mergel(step1[0], step1[8]); step2[2] = vec_mergeh(step1[1], step1[9]); step2[3] = vec_mergel(step1[1], step1[9]); step2[4] = vec_mergeh(step1[2], step1[10]); step2[5] = vec_mergel(step1[2], step1[10]); step2[6] = vec_mergeh(step1[3], step1[11]); step2[7] = vec_mergel(step1[3], step1[11]); step2[8] = vec_mergeh(step1[4], step1[12]); step2[9] = vec_mergel(step1[4], step1[12]); step2[10] = vec_mergeh(step1[5], step1[13]); step2[11] = vec_mergel(step1[5], step1[13]); step2[12] = vec_mergeh(step1[6], step1[14]); step2[13] = vec_mergel(step1[6], step1[14]); step2[14] = vec_mergeh(step1[7], step1[15]); step2[15] = vec_mergel(step1[7], step1[15]); step3[0] = vec_mergeh(step2[0], step2[8]); step3[1] = vec_mergel(step2[0], step2[8]); step3[2] = vec_mergeh(step2[1], step2[9]); step3[3] = vec_mergel(step2[1], step2[9]); step3[4] = vec_mergeh(step2[2], step2[10]); step3[5] = vec_mergel(step2[2], step2[10]); step3[6] = vec_mergeh(step2[3], step2[11]); step3[7] = vec_mergel(step2[3], step2[11]); step3[8] = vec_mergeh(step2[4], step2[12]); step3[9] = vec_mergel(step2[4], step2[12]); step3[10] = vec_mergeh(step2[5], step2[13]); step3[11] = vec_mergel(step2[5], step2[13]); step3[12] = vec_mergeh(step2[6], step2[14]); step3[13] = vec_mergel(step2[6], step2[14]); step3[14] = vec_mergeh(step2[7], step2[15]); step3[15] = vec_mergel(step2[7], step2[15]); kernel.packet[0] = vec_mergeh(step3[0], step3[8]); kernel.packet[1] = vec_mergel(step3[0], step3[8]); kernel.packet[2] = vec_mergeh(step3[1], step3[9]); kernel.packet[3] = vec_mergel(step3[1], step3[9]); kernel.packet[4] = vec_mergeh(step3[2], step3[10]); kernel.packet[5] = vec_mergel(step3[2], step3[10]); kernel.packet[6] = vec_mergeh(step3[3], step3[11]); kernel.packet[7] = vec_mergel(step3[3], step3[11]); kernel.packet[8] = vec_mergeh(step3[4], step3[12]); kernel.packet[9] = vec_mergel(step3[4], step3[12]); kernel.packet[10] = vec_mergeh(step3[5], step3[13]); kernel.packet[11] = vec_mergel(step3[5], step3[13]); kernel.packet[12] = vec_mergeh(step3[6], step3[14]); kernel.packet[13] = vec_mergel(step3[6], step3[14]); kernel.packet[14] = vec_mergeh(step3[7], step3[15]); kernel.packet[15] = vec_mergel(step3[7], step3[15]); } template EIGEN_STRONG_INLINE Packet pblend4(const Selector<4>& ifPacket, const Packet& thenPacket, const Packet& elsePacket) { Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] }; Packet4ui mask = reinterpret_cast(vec_cmpeq(reinterpret_cast(select), reinterpret_cast(p4i_ONE))); return vec_sel(elsePacket, thenPacket, mask); } template<> EIGEN_STRONG_INLINE Packet4i pblend(const Selector<4>& ifPacket, const Packet4i& thenPacket, const Packet4i& elsePacket) { return pblend4(ifPacket, thenPacket, elsePacket); } template<> EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket, const Packet4f& elsePacket) { return pblend4(ifPacket, thenPacket, elsePacket); } template<> EIGEN_STRONG_INLINE Packet8s pblend(const Selector<8>& ifPacket, const Packet8s& thenPacket, const Packet8s& elsePacket) { Packet8us select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3], ifPacket.select[4], ifPacket.select[5], ifPacket.select[6], ifPacket.select[7] }; Packet8us mask = reinterpret_cast(vec_cmpeq(select, p8us_ONE)); Packet8s result = vec_sel(elsePacket, thenPacket, mask); return result; } template<> EIGEN_STRONG_INLINE Packet8us pblend(const Selector<8>& ifPacket, const Packet8us& thenPacket, const Packet8us& elsePacket) { Packet8us select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3], ifPacket.select[4], ifPacket.select[5], ifPacket.select[6], ifPacket.select[7] }; Packet8us mask = reinterpret_cast(vec_cmpeq(reinterpret_cast(select), p8us_ONE)); return vec_sel(elsePacket, thenPacket, mask); } template<> EIGEN_STRONG_INLINE Packet8bf pblend(const Selector<8>& ifPacket, const Packet8bf& thenPacket, const Packet8bf& elsePacket) { return pblend(ifPacket, thenPacket, elsePacket); } template<> EIGEN_STRONG_INLINE Packet16c pblend(const Selector<16>& ifPacket, const Packet16c& thenPacket, const Packet16c& elsePacket) { Packet16uc select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3], ifPacket.select[4], ifPacket.select[5], ifPacket.select[6], ifPacket.select[7], ifPacket.select[8], ifPacket.select[9], ifPacket.select[10], ifPacket.select[11], ifPacket.select[12], ifPacket.select[13], ifPacket.select[14], ifPacket.select[15] }; Packet16uc mask = reinterpret_cast(vec_cmpeq(reinterpret_cast(select), p16uc_ONE)); return vec_sel(elsePacket, thenPacket, mask); } template<> EIGEN_STRONG_INLINE Packet16uc pblend(const Selector<16>& ifPacket, const Packet16uc& thenPacket, const Packet16uc& elsePacket) { Packet16uc select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3], ifPacket.select[4], ifPacket.select[5], ifPacket.select[6], ifPacket.select[7], ifPacket.select[8], ifPacket.select[9], ifPacket.select[10], ifPacket.select[11], ifPacket.select[12], ifPacket.select[13], ifPacket.select[14], ifPacket.select[15] }; Packet16uc mask = reinterpret_cast(vec_cmpeq(reinterpret_cast(select), p16uc_ONE)); return vec_sel(elsePacket, thenPacket, mask); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template<> EIGEN_STRONG_INLINE Packet4i pcast(const Packet4f& a) { return vec_cts(a,0); } template<> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet4f& a) { return vec_ctu(a,0); } template<> EIGEN_STRONG_INLINE Packet4f pcast(const Packet4i& a) { return vec_ctf(a,0); } template<> EIGEN_STRONG_INLINE Packet4f pcast(const Packet4ui& a) { return vec_ctf(a,0); } template<> EIGEN_STRONG_INLINE Packet8us pcast(const Packet8bf& a) { Packet4f float_even = Bf16ToF32Even(a); Packet4f float_odd = Bf16ToF32Odd(a); Packet4ui int_even = pcast(float_even); Packet4ui int_odd = pcast(float_odd); const _EIGEN_DECLARE_CONST_FAST_Packet4ui(low_mask, 0x0000FFFF); Packet4ui low_even = pand(int_even, p4ui_low_mask); Packet4ui low_odd = pand(int_odd, p4ui_low_mask); //Check values that are bigger than USHRT_MAX (0xFFFF) Packet4bi overflow_selector; if(vec_any_gt(int_even, p4ui_low_mask)){ overflow_selector = vec_cmpgt(int_even, p4ui_low_mask); low_even = vec_sel(low_even, p4ui_low_mask, overflow_selector); } if(vec_any_gt(int_odd, p4ui_low_mask)){ overflow_selector = vec_cmpgt(int_odd, p4ui_low_mask); low_odd = vec_sel(low_even, p4ui_low_mask, overflow_selector); } low_odd = plogical_shift_left<16>(low_odd); Packet4ui int_final = por(low_even, low_odd); return reinterpret_cast(int_final); } template<> EIGEN_STRONG_INLINE Packet8bf pcast(const Packet8us& a) { //short -> int -> float -> bfloat16 const _EIGEN_DECLARE_CONST_FAST_Packet4ui(low_mask, 0x0000FFFF); Packet4ui int_cast = reinterpret_cast(a); Packet4ui int_even = pand(int_cast, p4ui_low_mask); Packet4ui int_odd = plogical_shift_right<16>(int_cast); Packet4f float_even = pcast(int_even); Packet4f float_odd = pcast(int_odd); return F32ToBf16(float_even, float_odd); } template<> EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet4f& a) { return reinterpret_cast(a); } template<> EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet4i& a) { return reinterpret_cast(a); } //---------- double ---------- #ifdef __VSX__ typedef __vector double Packet2d; typedef __vector unsigned long long Packet2ul; typedef __vector long long Packet2l; #if EIGEN_COMP_CLANG typedef Packet2ul Packet2bl; #else typedef __vector __bool long Packet2bl; #endif static Packet2l p2l_ONE = { 1, 1 }; static Packet2l p2l_ZERO = reinterpret_cast(p4i_ZERO); static Packet2ul p2ul_SIGN = { 0x8000000000000000ull, 0x8000000000000000ull }; static Packet2ul p2ul_PREV0DOT5 = { 0x3FDFFFFFFFFFFFFFull, 0x3FDFFFFFFFFFFFFFull }; static Packet2d p2d_ONE = { 1.0, 1.0 }; static Packet2d p2d_ZERO = reinterpret_cast(p4f_ZERO); static Packet2d p2d_MZERO = { numext::bit_cast(0x8000000000000000ull), numext::bit_cast(0x8000000000000000ull) }; #ifdef _BIG_ENDIAN static Packet2d p2d_COUNTDOWN = reinterpret_cast(vec_sld(reinterpret_cast(p2d_ZERO), reinterpret_cast(p2d_ONE), 8)); #else static Packet2d p2d_COUNTDOWN = reinterpret_cast(vec_sld(reinterpret_cast(p2d_ONE), reinterpret_cast(p2d_ZERO), 8)); #endif template Packet2d vec_splat_dbl(Packet2d& a) { return vec_splat(a, index); } template<> struct packet_traits : default_packet_traits { typedef Packet2d type; typedef Packet2d half; enum { Vectorizable = 1, AlignedOnScalar = 1, size=2, HasHalfPacket = 1, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasMin = 1, HasMax = 1, HasAbs = 1, HasSin = 0, HasCos = 0, HasLog = 0, HasExp = 1, HasSqrt = 1, HasRsqrt = 1, HasRound = 1, HasFloor = 1, HasCeil = 1, HasRint = 1, HasNegate = 1, HasBlend = 1 }; }; template<> struct unpacket_traits { typedef double type; enum {size=2, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet2d half; }; inline std::ostream & operator <<(std::ostream & s, const Packet2l & v) { union { Packet2l v; int64_t n[2]; } vt; vt.v = v; s << vt.n[0] << ", " << vt.n[1]; return s; } inline std::ostream & operator <<(std::ostream & s, const Packet2d & v) { union { Packet2d v; double n[2]; } vt; vt.v = v; s << vt.n[0] << ", " << vt.n[1]; return s; } // Need to define them first or we get specialization after instantiation errors template<> EIGEN_STRONG_INLINE Packet2d pload(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return vec_xl(0, const_cast(from)); // cast needed by Clang } template<> EIGEN_STRONG_INLINE void pstore(double* to, const Packet2d& from) { EIGEN_DEBUG_ALIGNED_STORE vec_xst(from, 0, to); } template<> EIGEN_STRONG_INLINE Packet2d pset1(const double& from) { Packet2d v = {from, from}; return v; } template<> EIGEN_STRONG_INLINE Packet2d pset1frombits(unsigned long from) { Packet2l v = {static_cast(from), static_cast(from)}; return reinterpret_cast(v); } template<> EIGEN_STRONG_INLINE void pbroadcast4(const double *a, Packet2d& a0, Packet2d& a1, Packet2d& a2, Packet2d& a3) { //This way is faster than vec_splat (at least for doubles in Power 9) a0 = pset1(a[0]); a1 = pset1(a[1]); a2 = pset1(a[2]); a3 = pset1(a[3]); } template<> EIGEN_DEVICE_FUNC inline Packet2d pgather(const double* from, Index stride) { EIGEN_ALIGN16 double af[2]; af[0] = from[0*stride]; af[1] = from[1*stride]; return pload(af); } template<> EIGEN_DEVICE_FUNC inline void pscatter(double* to, const Packet2d& from, Index stride) { EIGEN_ALIGN16 double af[2]; pstore(af, from); to[0*stride] = af[0]; to[1*stride] = af[1]; } template<> EIGEN_STRONG_INLINE Packet2d plset(const double& a) { return pset1(a) + p2d_COUNTDOWN; } template<> EIGEN_STRONG_INLINE Packet2d padd(const Packet2d& a, const Packet2d& b) { return a + b; } template<> EIGEN_STRONG_INLINE Packet2d psub(const Packet2d& a, const Packet2d& b) { return a - b; } template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) { return p2d_ZERO - a; } template<> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { return a; } template<> EIGEN_STRONG_INLINE Packet2d pmul(const Packet2d& a, const Packet2d& b) { return vec_madd(a,b,p2d_MZERO); } template<> EIGEN_STRONG_INLINE Packet2d pdiv(const Packet2d& a, const Packet2d& b) { return vec_div(a,b); } // for some weird raisons, it has to be overloaded for packet of integers template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return vec_madd(a, b, c); } template<> EIGEN_STRONG_INLINE Packet2d pmin(const Packet2d& a, const Packet2d& b) { // NOTE: about 10% slower than vec_min, but consistent with std::min and SSE regarding NaN Packet2d ret; __asm__ ("xvcmpgedp %x0,%x1,%x2\n\txxsel %x0,%x1,%x2,%x0" : "=&wa" (ret) : "wa" (a), "wa" (b)); return ret; } template<> EIGEN_STRONG_INLINE Packet2d pmax(const Packet2d& a, const Packet2d& b) { // NOTE: about 10% slower than vec_max, but consistent with std::max and SSE regarding NaN Packet2d ret; __asm__ ("xvcmpgtdp %x0,%x2,%x1\n\txxsel %x0,%x1,%x2,%x0" : "=&wa" (ret) : "wa" (a), "wa" (b)); return ret; } template<> EIGEN_STRONG_INLINE Packet2d pcmp_le(const Packet2d& a, const Packet2d& b) { return reinterpret_cast(vec_cmple(a,b)); } template<> EIGEN_STRONG_INLINE Packet2d pcmp_lt(const Packet2d& a, const Packet2d& b) { return reinterpret_cast(vec_cmplt(a,b)); } template<> EIGEN_STRONG_INLINE Packet2d pcmp_eq(const Packet2d& a, const Packet2d& b) { return reinterpret_cast(vec_cmpeq(a,b)); } template<> EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan(const Packet2d& a, const Packet2d& b) { Packet2d c = reinterpret_cast(vec_cmpge(a,b)); return vec_nor(c,c); } template<> EIGEN_STRONG_INLINE Packet2d pand(const Packet2d& a, const Packet2d& b) { return vec_and(a, b); } template<> EIGEN_STRONG_INLINE Packet2d por(const Packet2d& a, const Packet2d& b) { return vec_or(a, b); } template<> EIGEN_STRONG_INLINE Packet2d pxor(const Packet2d& a, const Packet2d& b) { return vec_xor(a, b); } template<> EIGEN_STRONG_INLINE Packet2d pandnot(const Packet2d& a, const Packet2d& b) { return vec_and(a, vec_nor(b, b)); } template<> EIGEN_STRONG_INLINE Packet2d pround(const Packet2d& a) { Packet2d t = vec_add(reinterpret_cast(vec_or(vec_and(reinterpret_cast(a), p2ul_SIGN), p2ul_PREV0DOT5)), a); Packet2d res; __asm__("xvrdpiz %x0, %x1\n\t" : "=&wa" (res) : "wa" (t)); return res; } template<> EIGEN_STRONG_INLINE Packet2d pceil(const Packet2d& a) { return vec_ceil(a); } template<> EIGEN_STRONG_INLINE Packet2d pfloor(const Packet2d& a) { return vec_floor(a); } template<> EIGEN_STRONG_INLINE Packet2d print(const Packet2d& a) { Packet2d res; __asm__("xvrdpic %x0, %x1\n\t" : "=&wa" (res) : "wa" (a)); return res; } template<> EIGEN_STRONG_INLINE Packet2d ploadu(const double* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vec_xl(0, const_cast(from)); } template<> EIGEN_STRONG_INLINE Packet2d ploaddup(const double* from) { Packet2d p; if((std::ptrdiff_t(from) % 16) == 0) p = pload(from); else p = ploadu(from); return vec_splat_dbl<0>(p); } template<> EIGEN_STRONG_INLINE void pstoreu(double* to, const Packet2d& from) { EIGEN_DEBUG_UNALIGNED_STORE vec_xst(from, 0, to); } template<> EIGEN_STRONG_INLINE void prefetch(const double* addr) { EIGEN_PPC_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE double pfirst(const Packet2d& a) { EIGEN_ALIGN16 double x[2]; pstore(x, a); return x[0]; } template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) { return reinterpret_cast(vec_perm(reinterpret_cast(a), reinterpret_cast(a), p16uc_REVERSE64)); } template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { return vec_abs(a); } // VSX support varies between different compilers and even different // versions of the same compiler. For gcc version >= 4.9.3, we can use // vec_cts to efficiently convert Packet2d to Packet2l. Otherwise, use // a slow version that works with older compilers. // Update: apparently vec_cts/vec_ctf intrinsics for 64-bit doubles // are buggy, https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70963 template<> inline Packet2l pcast(const Packet2d& x) { #if EIGEN_GNUC_AT_LEAST(5, 4) || \ (EIGEN_GNUC_AT(6, 1) && __GNUC_PATCHLEVEL__ >= 1) return vec_cts(x, 0); // TODO: check clang version. #else double tmp[2]; memcpy(tmp, &x, sizeof(tmp)); Packet2l l = { static_cast(tmp[0]), static_cast(tmp[1]) }; return l; #endif } template<> inline Packet2d pcast(const Packet2l& x) { unsigned long long tmp[2]; memcpy(tmp, &x, sizeof(tmp)); Packet2d d = { static_cast(tmp[0]), static_cast(tmp[1]) }; return d; } // Packet2l shifts. // For POWER8 we simply use vec_sr/l. // // Things are more complicated for POWER7. There is actually a // vec_xxsxdi intrinsic but it is not supported by some gcc versions. // So we need to shift by N % 32 and rearrage bytes. #ifdef __POWER8_VECTOR__ template EIGEN_STRONG_INLINE Packet2l plogical_shift_left(const Packet2l& a) { const Packet2ul shift = { N, N }; return vec_sl(a, shift); } template EIGEN_STRONG_INLINE Packet2l plogical_shift_right(const Packet2l& a) { const Packet2ul shift = { N, N }; return vec_sr(a, shift); } #else // Shifts [A, B, C, D] to [B, 0, D, 0]. // Used to implement left shifts for Packet2l. EIGEN_ALWAYS_INLINE Packet4i shift_even_left(const Packet4i& a) { static const Packet16uc perm = { 0x14, 0x15, 0x16, 0x17, 0x00, 0x01, 0x02, 0x03, 0x1c, 0x1d, 0x1e, 0x1f, 0x08, 0x09, 0x0a, 0x0b }; #ifdef _BIG_ENDIAN return vec_perm(p4i_ZERO, a, perm); #else return vec_perm(a, p4i_ZERO, perm); #endif } // Shifts [A, B, C, D] to [0, A, 0, C]. // Used to implement right shifts for Packet2l. EIGEN_ALWAYS_INLINE Packet4i shift_odd_right(const Packet4i& a) { static const Packet16uc perm = { 0x04, 0x05, 0x06, 0x07, 0x10, 0x11, 0x12, 0x13, 0x0c, 0x0d, 0x0e, 0x0f, 0x18, 0x19, 0x1a, 0x1b }; #ifdef _BIG_ENDIAN return vec_perm(p4i_ZERO, a, perm); #else return vec_perm(a, p4i_ZERO, perm); #endif } template struct plogical_shift_left_impl; template struct plogical_shift_left_impl= 0)>::type> { static EIGEN_STRONG_INLINE Packet2l run(const Packet2l& a) { static const unsigned n = static_cast(N); const Packet4ui shift = {n, n, n, n}; const Packet4i ai = reinterpret_cast(a); static const unsigned m = static_cast(32 - N); const Packet4ui shift_right = {m, m, m, m}; const Packet4i out_hi = vec_sl(ai, shift); const Packet4i out_lo = shift_even_left(vec_sr(ai, shift_right)); return reinterpret_cast(por(out_hi, out_lo)); } }; template struct plogical_shift_left_impl= 32)>::type> { static EIGEN_STRONG_INLINE Packet2l run(const Packet2l& a) { static const unsigned m = static_cast(N - 32); const Packet4ui shift = {m, m, m, m}; const Packet4i ai = reinterpret_cast(a); return reinterpret_cast(shift_even_left(vec_sl(ai, shift))); } }; template EIGEN_STRONG_INLINE Packet2l plogical_shift_left(const Packet2l& a) { return plogical_shift_left_impl::run(a); } template struct plogical_shift_right_impl; template struct plogical_shift_right_impl= 0)>::type> { static EIGEN_STRONG_INLINE Packet2l run(const Packet2l& a) { static const unsigned n = static_cast(N); const Packet4ui shift = {n, n, n, n}; const Packet4i ai = reinterpret_cast(a); static const unsigned m = static_cast(32 - N); const Packet4ui shift_left = {m, m, m, m}; const Packet4i out_lo = vec_sr(ai, shift); const Packet4i out_hi = shift_odd_right(vec_sl(ai, shift_left)); return reinterpret_cast(por(out_hi, out_lo)); } }; template struct plogical_shift_right_impl= 32)>::type> { static EIGEN_STRONG_INLINE Packet2l run(const Packet2l& a) { static const unsigned m = static_cast(N - 32); const Packet4ui shift = {m, m, m, m}; const Packet4i ai = reinterpret_cast(a); return reinterpret_cast(shift_odd_right(vec_sr(ai, shift))); } }; template EIGEN_STRONG_INLINE Packet2l plogical_shift_right(const Packet2l& a) { return plogical_shift_right_impl::run(a); } #endif template<> EIGEN_STRONG_INLINE Packet2d pldexp(const Packet2d& a, const Packet2d& exponent) { // Clamp exponent to [-2099, 2099] const Packet2d max_exponent = pset1(2099.0); const Packet2l e = pcast(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent)); // Split 2^e into four factors and multiply: const Packet2l bias = { 1023, 1023 }; Packet2l b = plogical_shift_right<2>(e); // floor(e/4) Packet2d c = reinterpret_cast(plogical_shift_left<52>(b + bias)); Packet2d out = pmul(pmul(pmul(a, c), c), c); // a * 2^(3b) b = psub(psub(psub(e, b), b), b); // e - 3b c = reinterpret_cast(plogical_shift_left<52>(b + bias)); // 2^(e - 3b) out = pmul(out, c); // a * 2^e return out; } // Extract exponent without existence of Packet2l. template<> EIGEN_STRONG_INLINE Packet2d pfrexp_generic_get_biased_exponent(const Packet2d& a) { return pcast(plogical_shift_right<52>(reinterpret_cast(pabs(a)))); } template<> EIGEN_STRONG_INLINE Packet2d pfrexp (const Packet2d& a, Packet2d& exponent) { return pfrexp_generic(a, exponent); } template<> EIGEN_STRONG_INLINE double predux(const Packet2d& a) { Packet2d b, sum; b = reinterpret_cast(vec_sld(reinterpret_cast(a), reinterpret_cast(a), 8)); sum = a + b; return pfirst(sum); } // Other reduction functions: // mul template<> EIGEN_STRONG_INLINE double predux_mul(const Packet2d& a) { return pfirst(pmul(a, reinterpret_cast(vec_sld(reinterpret_cast(a), reinterpret_cast(a), 8)))); } // min template<> EIGEN_STRONG_INLINE double predux_min(const Packet2d& a) { return pfirst(pmin(a, reinterpret_cast(vec_sld(reinterpret_cast(a), reinterpret_cast(a), 8)))); } // max template<> EIGEN_STRONG_INLINE double predux_max(const Packet2d& a) { return pfirst(pmax(a, reinterpret_cast(vec_sld(reinterpret_cast(a), reinterpret_cast(a), 8)))); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { Packet2d t0, t1; t0 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_HI); t1 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_LO); kernel.packet[0] = t0; kernel.packet[1] = t1; } template<> EIGEN_STRONG_INLINE Packet2d pblend(const Selector<2>& ifPacket, const Packet2d& thenPacket, const Packet2d& elsePacket) { Packet2l select = { ifPacket.select[0], ifPacket.select[1] }; Packet2bl mask = reinterpret_cast( vec_cmpeq(reinterpret_cast(select), reinterpret_cast(p2l_ONE)) ); return vec_sel(elsePacket, thenPacket, mask); } #endif // __VSX__ } // end namespace internal } // end namespace Eigen #endif // EIGEN_PACKET_MATH_ALTIVEC_H RcppEigen/inst/include/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h0000644000176200001440000006036414567757725025053 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2020 Everton Constantino (everton.constantino@ibm.com) // Copyright (C) 2021 Chip Kerchner (chip.kerchner@ibm.com) // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H #define EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H #pragma GCC target("cpu=power10") #ifdef __has_builtin #if !__has_builtin(__builtin_vsx_assemble_pair) #define __builtin_vsx_assemble_pair __builtin_mma_assemble_pair #endif #endif namespace Eigen { namespace internal { template EIGEN_ALWAYS_INLINE void bsetzeroMMA(__vector_quad* acc) { __builtin_mma_xxsetaccz(acc); } template EIGEN_ALWAYS_INLINE void storeAccumulator(Index i, Index j, const DataMapper& data, const Packet& alpha, __vector_quad* acc) { PacketBlock result; __builtin_mma_disassemble_acc(&result.packet, acc); PacketBlock tRes; bload(tRes, data, i, j); bscale(tRes, result, alpha); data.template storePacketBlock(i, j, tRes); } template EIGEN_ALWAYS_INLINE void storeComplexAccumulator(Index i, Index j, const DataMapper& data, const Packet& alphaReal, const Packet& alphaImag, __vector_quad* accReal, __vector_quad* accImag) { PacketBlock resultReal, resultImag; __builtin_mma_disassemble_acc(&resultReal.packet, accReal); __builtin_mma_disassemble_acc(&resultImag.packet, accImag); PacketBlock tRes; bload(tRes, data, i, j); PacketBlock taccReal, taccImag; bscalec(resultReal, resultImag, alphaReal, alphaImag, taccReal, taccImag); PacketBlock acc1, acc2; bcouple(taccReal, taccImag, tRes, acc1, acc2); data.template storePacketBlock(i + N*accColsC, j, acc1); data.template storePacketBlock(i + (N+1)*accColsC, j, acc2); } // Defaults to float32, since Eigen still supports C++03 we can't use default template arguments template EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const RhsPacket& a, const LhsPacket& b) { if(NegativeAccumulate) { __builtin_mma_xvf32gernp(acc, (__vector unsigned char)a, (__vector unsigned char)b); } else { __builtin_mma_xvf32gerpp(acc, (__vector unsigned char)a, (__vector unsigned char)b); } } template EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const PacketBlock& a, const Packet2d& b) { __vector_pair* a0 = (__vector_pair *)(&a.packet[0]); if(NegativeAccumulate) { __builtin_mma_xvf64gernp(acc, *a0, (__vector unsigned char)b); } else { __builtin_mma_xvf64gerpp(acc, *a0, (__vector unsigned char)b); } } template EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const __vector_pair& a, const Packet2d& b) { if(NegativeAccumulate) { __builtin_mma_xvf64gernp(acc, (__vector_pair)a, (__vector unsigned char)b); } else { __builtin_mma_xvf64gerpp(acc, (__vector_pair)a, (__vector unsigned char)b); } } template EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad*, const __vector_pair&, const Packet4f&) { // Just for compilation } template EIGEN_ALWAYS_INLINE void pgercMMA(__vector_quad* accReal, __vector_quad* accImag, const Packet& lhsV, const Packet& lhsVi, const RhsPacket& rhsV, const RhsPacket& rhsVi) { pgerMMA(accReal, rhsV, lhsV); if(LhsIsReal) { pgerMMA(accImag, rhsVi, lhsV); } else { if(!RhsIsReal) { pgerMMA(accReal, rhsVi, lhsVi); pgerMMA(accImag, rhsVi, lhsV); } else { EIGEN_UNUSED_VARIABLE(rhsVi); } pgerMMA(accImag, rhsV, lhsVi); } } // This is necessary because ploadRhs for double returns a pair of vectors when MMA is enabled. template EIGEN_ALWAYS_INLINE void ploadRhsMMA(const Scalar* rhs, Packet& rhsV) { rhsV = ploadRhs((const Scalar*)(rhs)); } template<> EIGEN_ALWAYS_INLINE void ploadRhsMMA >(const double* rhs, PacketBlock& rhsV) { rhsV.packet[0] = ploadRhs((const double *)((Packet2d *)rhs )); rhsV.packet[1] = ploadRhs((const double *)(((Packet2d *)rhs) + 1)); } template<> EIGEN_ALWAYS_INLINE void ploadRhsMMA(const double* rhs, __vector_pair& rhsV) { #if EIGEN_COMP_LLVM __builtin_vsx_assemble_pair(&rhsV, (__vector unsigned char)(ploadRhs((const double *)(((Packet2d *)rhs) + 1))), (__vector unsigned char)(ploadRhs((const double *)((Packet2d *)rhs )))); #else __asm__ ("lxvp %x0,%1" : "=wa" (rhsV) : "Y" (*rhs)); #endif } template<> EIGEN_ALWAYS_INLINE void ploadRhsMMA(const float*, __vector_pair&) { // Just for compilation } // PEEL_MMA loop factor. #define PEEL_MMA 7 #define MICRO_MMA_UNROLL(func) \ func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7) #define MICRO_MMA_LOAD_ONE(iter) \ if (unroll_factor > iter) { \ lhsV##iter = ploadLhs(lhs_ptr##iter); \ lhs_ptr##iter += accCols; \ } else { \ EIGEN_UNUSED_VARIABLE(lhsV##iter); \ } #define MICRO_MMA_WORK_ONE(iter, type, peel) \ if (unroll_factor > iter) { \ pgerMMA(&accZero##iter, rhsV##peel, lhsV##iter); \ } #define MICRO_MMA_TYPE_PEEL(func, func2, type, peel) \ if (PEEL_MMA > peel) { \ Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \ ploadRhsMMA(rhs_ptr + (accRows * peel), rhsV##peel); \ MICRO_MMA_UNROLL(func2); \ func(0,type,peel) func(1,type,peel) func(2,type,peel) func(3,type,peel) \ func(4,type,peel) func(5,type,peel) func(6,type,peel) func(7,type,peel) \ } else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ } #define MICRO_MMA_UNROLL_TYPE_PEEL(func, func2, type) \ type rhsV0, rhsV1, rhsV2, rhsV3, rhsV4, rhsV5, rhsV6, rhsV7, rhsV8, rhsV9; \ MICRO_MMA_TYPE_PEEL(func,func2,type,0); MICRO_MMA_TYPE_PEEL(func,func2,type,1); \ MICRO_MMA_TYPE_PEEL(func,func2,type,2); MICRO_MMA_TYPE_PEEL(func,func2,type,3); \ MICRO_MMA_TYPE_PEEL(func,func2,type,4); MICRO_MMA_TYPE_PEEL(func,func2,type,5); \ MICRO_MMA_TYPE_PEEL(func,func2,type,6); MICRO_MMA_TYPE_PEEL(func,func2,type,7); \ MICRO_MMA_TYPE_PEEL(func,func2,type,8); MICRO_MMA_TYPE_PEEL(func,func2,type,9); #define MICRO_MMA_UNROLL_TYPE_ONE(func, func2, type) \ type rhsV0; \ MICRO_MMA_TYPE_PEEL(func,func2,type,0); #define MICRO_MMA_ONE_PEEL \ if (sizeof(Scalar) == sizeof(float)) { \ MICRO_MMA_UNROLL_TYPE_PEEL(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, RhsPacket); \ } else { \ MICRO_MMA_UNROLL_TYPE_PEEL(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, __vector_pair); \ } \ rhs_ptr += (accRows * PEEL_MMA); #define MICRO_MMA_ONE \ if (sizeof(Scalar) == sizeof(float)) { \ MICRO_MMA_UNROLL_TYPE_ONE(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, RhsPacket); \ } else { \ MICRO_MMA_UNROLL_TYPE_ONE(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, __vector_pair); \ } \ rhs_ptr += accRows; #define MICRO_MMA_DST_PTR_ONE(iter) \ if (unroll_factor > iter) { \ bsetzeroMMA(&accZero##iter); \ } else { \ EIGEN_UNUSED_VARIABLE(accZero##iter); \ } #define MICRO_MMA_DST_PTR MICRO_MMA_UNROLL(MICRO_MMA_DST_PTR_ONE) #define MICRO_MMA_SRC_PTR_ONE(iter) \ if (unroll_factor > iter) { \ lhs_ptr##iter = lhs_base + ( (row/accCols) + iter )*strideA*accCols + accCols*offsetA; \ } else { \ EIGEN_UNUSED_VARIABLE(lhs_ptr##iter); \ } #define MICRO_MMA_SRC_PTR MICRO_MMA_UNROLL(MICRO_MMA_SRC_PTR_ONE) #define MICRO_MMA_PREFETCH_ONE(iter) \ if (unroll_factor > iter) { \ EIGEN_POWER_PREFETCH(lhs_ptr##iter); \ } #define MICRO_MMA_PREFETCH MICRO_MMA_UNROLL(MICRO_MMA_PREFETCH_ONE) #define MICRO_MMA_STORE_ONE(iter) \ if (unroll_factor > iter) { \ storeAccumulator(row + iter*accCols, col, res, pAlpha, &accZero##iter); \ } #define MICRO_MMA_STORE MICRO_MMA_UNROLL(MICRO_MMA_STORE_ONE) template EIGEN_STRONG_INLINE void gemm_unrolled_MMA_iteration( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index& row, Index col, const Packet& pAlpha) { const Scalar* rhs_ptr = rhs_base; const Scalar* lhs_ptr0 = NULL, * lhs_ptr1 = NULL, * lhs_ptr2 = NULL, * lhs_ptr3 = NULL, * lhs_ptr4 = NULL, * lhs_ptr5 = NULL, * lhs_ptr6 = NULL, * lhs_ptr7 = NULL; __vector_quad accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7; MICRO_MMA_SRC_PTR MICRO_MMA_DST_PTR Index k = 0; for(; k + PEEL_MMA <= depth; k+= PEEL_MMA) { EIGEN_POWER_PREFETCH(rhs_ptr); MICRO_MMA_PREFETCH MICRO_MMA_ONE_PEEL } for(; k < depth; k++) { MICRO_MMA_ONE } MICRO_MMA_STORE row += unroll_factor*accCols; } template void gemmMMA(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { const Index remaining_rows = rows % accCols; const Index remaining_cols = cols % accRows; if( strideA == -1 ) strideA = depth; if( strideB == -1 ) strideB = depth; const Packet pAlpha = pset1(alpha); const Packet pMask = bmask((const int)(remaining_rows)); Index col = 0; for(; col + accRows <= cols; col += accRows) { const Scalar* rhs_base = blockB + col*strideB + accRows*offsetB; const Scalar* lhs_base = blockA; Index row = 0; #define MAX_MMA_UNROLL 7 while(row + MAX_MMA_UNROLL*accCols <= rows) { gemm_unrolled_MMA_iteration(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha); } switch( (rows-row)/accCols ) { #if MAX_MMA_UNROLL > 7 case 7: gemm_unrolled_MMA_iteration<7, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha); break; #endif #if MAX_MMA_UNROLL > 6 case 6: gemm_unrolled_MMA_iteration<6, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha); break; #endif #if MAX_MMA_UNROLL > 5 case 5: gemm_unrolled_MMA_iteration<5, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha); break; #endif #if MAX_MMA_UNROLL > 4 case 4: gemm_unrolled_MMA_iteration<4, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha); break; #endif #if MAX_MMA_UNROLL > 3 case 3: gemm_unrolled_MMA_iteration<3, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha); break; #endif #if MAX_MMA_UNROLL > 2 case 2: gemm_unrolled_MMA_iteration<2, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha); break; #endif #if MAX_MMA_UNROLL > 1 case 1: gemm_unrolled_MMA_iteration<1, Scalar, Packet, RhsPacket, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha); break; #endif default: break; } #undef MAX_MMA_UNROLL if(remaining_rows > 0) { gemm_extra_row(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, rows, cols, remaining_rows, pAlpha, pMask); } } if(remaining_cols > 0) { const Scalar* rhs_base = blockB + col*strideB + remaining_cols*offsetB; const Scalar* lhs_base = blockA; for(; col < cols; col++) { Index row = 0; gemm_unrolled_col(res, lhs_base, rhs_base, depth, strideA, offsetA, row, rows, col, remaining_cols, pAlpha); if (remaining_rows > 0) { gemm_extra_col(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_rows, remaining_cols, pAlpha); } rhs_base++; } } } #define accColsC (accCols / 2) #define advanceRows ((LhsIsReal) ? 1 : 2) #define advanceCols ((RhsIsReal) ? 1 : 2) // PEEL_COMPLEX_MMA loop factor. #define PEEL_COMPLEX_MMA 7 #define MICRO_COMPLEX_MMA_UNROLL(func) \ func(0) func(1) func(2) func(3) func(4) #define MICRO_COMPLEX_MMA_LOAD_ONE(iter) \ if (unroll_factor > iter) { \ lhsV##iter = ploadLhs(lhs_ptr_real##iter); \ lhs_ptr_real##iter += accCols; \ if(!LhsIsReal) { \ lhsVi##iter = ploadLhs(lhs_ptr_imag##iter); \ lhs_ptr_imag##iter += accCols; \ } else { \ EIGEN_UNUSED_VARIABLE(lhsVi##iter); \ } \ } else { \ EIGEN_UNUSED_VARIABLE(lhsV##iter); \ EIGEN_UNUSED_VARIABLE(lhsVi##iter); \ } #define MICRO_COMPLEX_MMA_WORK_ONE(iter, type, peel) \ if (unroll_factor > iter) { \ pgercMMA(&accReal##iter, &accImag##iter, lhsV##iter, lhsVi##iter, rhsV##peel, rhsVi##peel); \ } #define MICRO_COMPLEX_MMA_TYPE_PEEL(func, func2, type, peel) \ if (PEEL_COMPLEX_MMA > peel) { \ Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4; \ Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3, lhsVi4; \ ploadRhsMMA(rhs_ptr_real + (accRows * peel), rhsV##peel); \ if(!RhsIsReal) { \ ploadRhsMMA(rhs_ptr_imag + (accRows * peel), rhsVi##peel); \ } else { \ EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ } \ MICRO_COMPLEX_MMA_UNROLL(func2); \ func(0,type,peel) func(1,type,peel) func(2,type,peel) func(3,type,peel) func(4,type,peel) \ } else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ } #define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(func, func2, type) \ type rhsV0, rhsV1, rhsV2, rhsV3, rhsV4, rhsV5, rhsV6, rhsV7, rhsV8, rhsV9; \ type rhsVi0, rhsVi1, rhsVi2, rhsVi3, rhsVi4, rhsVi5, rhsVi6, rhsVi7, rhsVi8, rhsVi9; \ MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,0); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,1); \ MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,2); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,3); \ MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,4); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,5); \ MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,6); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,7); \ MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,8); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,9); #define MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(func, func2, type) \ type rhsV0, rhsVi0; \ MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,0); #define MICRO_COMPLEX_MMA_ONE_PEEL \ if (sizeof(Scalar) == sizeof(float)) { \ MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, RhsPacket); \ } else { \ MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, __vector_pair); \ } \ rhs_ptr_real += (accRows * PEEL_COMPLEX_MMA); \ if(!RhsIsReal) rhs_ptr_imag += (accRows * PEEL_COMPLEX_MMA); #define MICRO_COMPLEX_MMA_ONE \ if (sizeof(Scalar) == sizeof(float)) { \ MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, RhsPacket); \ } else { \ MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, __vector_pair); \ } \ rhs_ptr_real += accRows; \ if(!RhsIsReal) rhs_ptr_imag += accRows; #define MICRO_COMPLEX_MMA_DST_PTR_ONE(iter) \ if (unroll_factor > iter) { \ bsetzeroMMA(&accReal##iter); \ bsetzeroMMA(&accImag##iter); \ } else { \ EIGEN_UNUSED_VARIABLE(accReal##iter); \ EIGEN_UNUSED_VARIABLE(accImag##iter); \ } #define MICRO_COMPLEX_MMA_DST_PTR MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_DST_PTR_ONE) #define MICRO_COMPLEX_MMA_SRC_PTR_ONE(iter) \ if (unroll_factor > iter) { \ lhs_ptr_real##iter = lhs_base + ( ((advanceRows*row)/accCols) + iter*advanceRows )*strideA*accCols + accCols*offsetA; \ if(!LhsIsReal) { \ lhs_ptr_imag##iter = lhs_ptr_real##iter + accCols*strideA; \ } else { \ EIGEN_UNUSED_VARIABLE(lhs_ptr_imag##iter); \ } \ } else { \ EIGEN_UNUSED_VARIABLE(lhs_ptr_real##iter); \ EIGEN_UNUSED_VARIABLE(lhs_ptr_imag##iter); \ } #define MICRO_COMPLEX_MMA_SRC_PTR MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_SRC_PTR_ONE) #define MICRO_COMPLEX_MMA_PREFETCH_ONE(iter) \ if (unroll_factor > iter) { \ EIGEN_POWER_PREFETCH(lhs_ptr_real##iter); \ if(!LhsIsReal) { \ EIGEN_POWER_PREFETCH(lhs_ptr_imag##iter); \ } \ } #define MICRO_COMPLEX_MMA_PREFETCH MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_PREFETCH_ONE) #define MICRO_COMPLEX_MMA_STORE_ONE(iter) \ if (unroll_factor > iter) { \ storeComplexAccumulator(row + iter*accCols, col, res, pAlphaReal, pAlphaImag, &accReal##iter, &accImag##iter); \ } #define MICRO_COMPLEX_MMA_STORE MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_STORE_ONE) template EIGEN_STRONG_INLINE void gemm_complex_unrolled_MMA_iteration( const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index& row, Index col, const Packet& pAlphaReal, const Packet& pAlphaImag) { const Scalar* rhs_ptr_real = rhs_base; const Scalar* rhs_ptr_imag; if(!RhsIsReal) { rhs_ptr_imag = rhs_base + accRows*strideB; } else { EIGEN_UNUSED_VARIABLE(rhs_ptr_imag); } const Scalar* lhs_ptr_real0 = NULL, * lhs_ptr_imag0 = NULL, * lhs_ptr_real1 = NULL, * lhs_ptr_imag1 = NULL; const Scalar* lhs_ptr_real2 = NULL, * lhs_ptr_imag2 = NULL, * lhs_ptr_real3 = NULL, * lhs_ptr_imag3 = NULL; const Scalar* lhs_ptr_real4 = NULL, * lhs_ptr_imag4 = NULL; __vector_quad accReal0, accImag0, accReal1, accImag1, accReal2, accImag2, accReal3, accImag3, accReal4, accImag4; MICRO_COMPLEX_MMA_SRC_PTR MICRO_COMPLEX_MMA_DST_PTR Index k = 0; for(; k + PEEL_COMPLEX_MMA <= depth; k+= PEEL_COMPLEX_MMA) { EIGEN_POWER_PREFETCH(rhs_ptr_real); if(!RhsIsReal) { EIGEN_POWER_PREFETCH(rhs_ptr_imag); } MICRO_COMPLEX_MMA_PREFETCH MICRO_COMPLEX_MMA_ONE_PEEL } for(; k < depth; k++) { MICRO_COMPLEX_MMA_ONE } MICRO_COMPLEX_MMA_STORE row += unroll_factor*accCols; } template void gemm_complexMMA(const DataMapper& res, const LhsScalar* blockAc, const RhsScalar* blockBc, Index rows, Index depth, Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { const Index remaining_rows = rows % accCols; const Index remaining_cols = cols % accRows; if( strideA == -1 ) strideA = depth; if( strideB == -1 ) strideB = depth; const Packet pAlphaReal = pset1(alpha.real()); const Packet pAlphaImag = pset1(alpha.imag()); const Packet pMask = bmask((const int)(remaining_rows)); const Scalar* blockA = (Scalar *) blockAc; const Scalar* blockB = (Scalar *) blockBc; Index col = 0; for(; col + accRows <= cols; col += accRows) { const Scalar* rhs_base = blockB + advanceCols*col*strideB + accRows*offsetB; const Scalar* lhs_base = blockA; Index row = 0; #define MAX_COMPLEX_MMA_UNROLL 4 while(row + MAX_COMPLEX_MMA_UNROLL*accCols <= rows) { gemm_complex_unrolled_MMA_iteration(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag); } switch( (rows-row)/accCols ) { #if MAX_COMPLEX_MMA_UNROLL > 4 case 4: gemm_complex_unrolled_MMA_iteration<4, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag); break; #endif #if MAX_COMPLEX_MMA_UNROLL > 3 case 3: gemm_complex_unrolled_MMA_iteration<3, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag); break; #endif #if MAX_COMPLEX_MMA_UNROLL > 2 case 2: gemm_complex_unrolled_MMA_iteration<2, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag); break; #endif #if MAX_COMPLEX_MMA_UNROLL > 1 case 1: gemm_complex_unrolled_MMA_iteration<1, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag); break; #endif default: break; } #undef MAX_COMPLEX_MMA_UNROLL if(remaining_rows > 0) { gemm_complex_extra_row(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, rows, cols, remaining_rows, pAlphaReal, pAlphaImag, pMask); } } if(remaining_cols > 0) { const Scalar* rhs_base = blockB + advanceCols*col*strideB + remaining_cols*offsetB; const Scalar* lhs_base = blockA; for(; col < cols; col++) { Index row = 0; gemm_complex_unrolled_col(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, rows, col, remaining_cols, pAlphaReal, pAlphaImag); if (remaining_rows > 0) { gemm_complex_extra_col(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, remaining_rows, remaining_cols, pAlphaReal, pAlphaImag); } rhs_base++; } } } #undef accColsC #undef advanceRows #undef advanceCols #pragma GCC reset_options } // end namespace internal } // end namespace Eigen #endif // EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H RcppEigen/inst/include/Eigen/src/Core/arch/AltiVec/MathFunctions.h0000644000176200001440000000442314567757725024467 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2007 Julien Pommier // Copyright (C) 2009 Gael Guennebaud // Copyright (C) 2016 Konstantinos Margaritis // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_MATH_FUNCTIONS_ALTIVEC_H #define EIGEN_MATH_FUNCTIONS_ALTIVEC_H namespace Eigen { namespace internal { template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f plog(const Packet4f& _x) { return plog_float(_x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pexp(const Packet4f& _x) { return pexp_float(_x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f psin(const Packet4f& _x) { return psin_float(_x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pcos(const Packet4f& _x) { return pcos_float(_x); } #ifndef EIGEN_COMP_CLANG template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f prsqrt(const Packet4f& x) { return vec_rsqrt(x); } #endif #ifdef __VSX__ #ifndef EIGEN_COMP_CLANG template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d prsqrt(const Packet2d& x) { return vec_rsqrt(x); } #endif template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f psqrt(const Packet4f& x) { return vec_sqrt(x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d psqrt(const Packet2d& x) { return vec_sqrt(x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d pexp(const Packet2d& _x) { return pexp_double(_x); } #endif // Hyperbolic Tangent function. template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f ptanh(const Packet4f& x) { return internal::generic_fast_tanh_float(x); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_MATH_FUNCTIONS_ALTIVEC_H RcppEigen/inst/include/Eigen/src/Core/arch/SSE/0000755000176200001440000000000014567757725020634 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/arch/SSE/TypeCasting.h0000644000176200001440000000710214567757725023237 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2015 Benoit Steiner // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_TYPE_CASTING_SSE_H #define EIGEN_TYPE_CASTING_SSE_H namespace Eigen { namespace internal { #ifndef EIGEN_VECTORIZE_AVX template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; #endif template<> EIGEN_STRONG_INLINE Packet4i pcast(const Packet4f& a) { return _mm_cvttps_epi32(a); } template<> EIGEN_STRONG_INLINE Packet4f pcast(const Packet4i& a) { return _mm_cvtepi32_ps(a); } template<> EIGEN_STRONG_INLINE Packet4f pcast(const Packet2d& a, const Packet2d& b) { return _mm_shuffle_ps(_mm_cvtpd_ps(a), _mm_cvtpd_ps(b), (1 << 2) | (1 << 6)); } template<> EIGEN_STRONG_INLINE Packet2d pcast(const Packet4f& a) { // Simply discard the second half of the input return _mm_cvtps_pd(a); } template<> EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet4f& a) { return _mm_castps_si128(a); } template<> EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet4i& a) { return _mm_castsi128_ps(a); } template<> EIGEN_STRONG_INLINE Packet2d preinterpret(const Packet4i& a) { return _mm_castsi128_pd(a); } template<> EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet2d& a) { return _mm_castpd_si128(a); } // Disable the following code since it's broken on too many platforms / compilers. //#elif defined(EIGEN_VECTORIZE_SSE) && (!EIGEN_ARCH_x86_64) && (!EIGEN_COMP_MSVC) #if 0 template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template<> EIGEN_STRONG_INLINE Packet4f pcast(const Packet4h& a) { __int64_t a64 = _mm_cvtm64_si64(a.x); Eigen::half h = raw_uint16_to_half(static_cast(a64)); float f1 = static_cast(h); h = raw_uint16_to_half(static_cast(a64 >> 16)); float f2 = static_cast(h); h = raw_uint16_to_half(static_cast(a64 >> 32)); float f3 = static_cast(h); h = raw_uint16_to_half(static_cast(a64 >> 48)); float f4 = static_cast(h); return _mm_set_ps(f4, f3, f2, f1); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template<> EIGEN_STRONG_INLINE Packet4h pcast(const Packet4f& a) { EIGEN_ALIGN16 float aux[4]; pstore(aux, a); Eigen::half h0(aux[0]); Eigen::half h1(aux[1]); Eigen::half h2(aux[2]); Eigen::half h3(aux[3]); Packet4h result; result.x = _mm_set_pi16(h3.x, h2.x, h1.x, h0.x); return result; } #endif } // end namespace internal } // end namespace Eigen #endif // EIGEN_TYPE_CASTING_SSE_H RcppEigen/inst/include/Eigen/src/Core/arch/SSE/Complex.h0000644000176200001440000003365314567757725022426 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2010 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_COMPLEX_SSE_H #define EIGEN_COMPLEX_SSE_H namespace Eigen { namespace internal { //---------- float ---------- struct Packet2cf { EIGEN_STRONG_INLINE Packet2cf() {} EIGEN_STRONG_INLINE explicit Packet2cf(const __m128& a) : v(a) {} Packet4f v; }; // Use the packet_traits defined in AVX/PacketMath.h instead if we're going // to leverage AVX instructions. #ifndef EIGEN_VECTORIZE_AVX template<> struct packet_traits > : default_packet_traits { typedef Packet2cf type; typedef Packet2cf half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 2, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, HasSqrt = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, HasMax = 0, HasSetLinear = 0, HasBlend = 1 }; }; #endif template<> struct unpacket_traits { typedef std::complex type; typedef Packet2cf half; typedef Packet4f as_real; enum { size=2, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false }; }; template<> EIGEN_STRONG_INLINE Packet2cf padd(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_add_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf psub(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_sub_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf& a) { const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x80000000,0x80000000,0x80000000,0x80000000)); return Packet2cf(_mm_xor_ps(a.v,mask)); } template<> EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) { const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000)); return Packet2cf(_mm_xor_ps(a.v,mask)); } template<> EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) { #ifdef EIGEN_VECTORIZE_SSE3 return Packet2cf(_mm_addsub_ps(_mm_mul_ps(_mm_moveldup_ps(a.v), b.v), _mm_mul_ps(_mm_movehdup_ps(a.v), vec4f_swizzle1(b.v, 1, 0, 3, 2)))); // return Packet2cf(_mm_addsub_ps(_mm_mul_ps(vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v), // _mm_mul_ps(vec4f_swizzle1(a.v, 1, 1, 3, 3), // vec4f_swizzle1(b.v, 1, 0, 3, 2)))); #else const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x80000000,0x00000000,0x80000000,0x00000000)); return Packet2cf(_mm_add_ps(_mm_mul_ps(vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v), _mm_xor_ps(_mm_mul_ps(vec4f_swizzle1(a.v, 1, 1, 3, 3), vec4f_swizzle1(b.v, 1, 0, 3, 2)), mask))); #endif } template<> EIGEN_STRONG_INLINE Packet2cf ptrue (const Packet2cf& a) { return Packet2cf(ptrue(Packet4f(a.v))); } template<> EIGEN_STRONG_INLINE Packet2cf pand (const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_and_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf por (const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_or_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf pxor (const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_xor_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf pandnot(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_andnot_ps(b.v,a.v)); } template<> EIGEN_STRONG_INLINE Packet2cf pload (const std::complex* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet2cf(pload(&numext::real_ref(*from))); } template<> EIGEN_STRONG_INLINE Packet2cf ploadu(const std::complex* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cf(ploadu(&numext::real_ref(*from))); } template<> EIGEN_STRONG_INLINE Packet2cf pset1(const std::complex& from) { Packet2cf res; #ifdef EIGEN_VECTORIZE_SSE3 res.v = _mm_castpd_ps(_mm_loaddup_pd(reinterpret_cast(&from))); #else res.v = _mm_castpd_ps(_mm_load_sd(reinterpret_cast(&from))); res.v = _mm_movelh_ps(res.v, res.v); #endif return res; } template<> EIGEN_STRONG_INLINE Packet2cf ploaddup(const std::complex* from) { return pset1(*from); } template<> EIGEN_STRONG_INLINE void pstore >(std::complex * to, const Packet2cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), Packet4f(from.v)); } template<> EIGEN_STRONG_INLINE void pstoreu >(std::complex * to, const Packet2cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), Packet4f(from.v)); } template<> EIGEN_DEVICE_FUNC inline Packet2cf pgather, Packet2cf>(const std::complex* from, Index stride) { return Packet2cf(_mm_set_ps(std::imag(from[1*stride]), std::real(from[1*stride]), std::imag(from[0*stride]), std::real(from[0*stride]))); } template<> EIGEN_DEVICE_FUNC inline void pscatter, Packet2cf>(std::complex* to, const Packet2cf& from, Index stride) { to[stride*0] = std::complex(_mm_cvtss_f32(_mm_shuffle_ps(from.v, from.v, 0)), _mm_cvtss_f32(_mm_shuffle_ps(from.v, from.v, 1))); to[stride*1] = std::complex(_mm_cvtss_f32(_mm_shuffle_ps(from.v, from.v, 2)), _mm_cvtss_f32(_mm_shuffle_ps(from.v, from.v, 3))); } template<> EIGEN_STRONG_INLINE void prefetch >(const std::complex * addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } template<> EIGEN_STRONG_INLINE std::complex pfirst(const Packet2cf& a) { #if EIGEN_GNUC_AT_MOST(4,3) // Workaround gcc 4.2 ICE - this is not performance wise ideal, but who cares... // This workaround also fix invalid code generation with gcc 4.3 EIGEN_ALIGN16 std::complex res[2]; _mm_store_ps((float*)res, a.v); return res[0]; #else std::complex res; _mm_storel_pi((__m64*)&res, a.v); return res; #endif } template<> EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) { return Packet2cf(_mm_castpd_ps(preverse(Packet2d(_mm_castps_pd(a.v))))); } template<> EIGEN_STRONG_INLINE std::complex predux(const Packet2cf& a) { return pfirst(Packet2cf(_mm_add_ps(a.v, _mm_movehl_ps(a.v,a.v)))); } template<> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet2cf& a) { return pfirst(pmul(a, Packet2cf(_mm_movehl_ps(a.v,a.v)))); } EIGEN_STRONG_INLINE Packet2cf pcplxflip/* */(const Packet2cf& x) { return Packet2cf(vec4f_swizzle1(x.v, 1, 0, 3, 2)); } EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf,Packet4f) template<> EIGEN_STRONG_INLINE Packet2cf pdiv(const Packet2cf& a, const Packet2cf& b) { // TODO optimize it for SSE3 and 4 Packet2cf res = pmul(a, pconj(b)); __m128 s = _mm_mul_ps(b.v,b.v); return Packet2cf(_mm_div_ps(res.v,_mm_add_ps(s,vec4f_swizzle1(s, 1, 0, 3, 2)))); } //---------- double ---------- struct Packet1cd { EIGEN_STRONG_INLINE Packet1cd() {} EIGEN_STRONG_INLINE explicit Packet1cd(const __m128d& a) : v(a) {} Packet2d v; }; // Use the packet_traits defined in AVX/PacketMath.h instead if we're going // to leverage AVX instructions. #ifndef EIGEN_VECTORIZE_AVX template<> struct packet_traits > : default_packet_traits { typedef Packet1cd type; typedef Packet1cd half; enum { Vectorizable = 1, AlignedOnScalar = 0, size = 1, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, HasSqrt = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, HasMax = 0, HasSetLinear = 0 }; }; #endif template<> struct unpacket_traits { typedef std::complex type; typedef Packet1cd half; typedef Packet2d as_real; enum { size=1, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false }; }; template<> EIGEN_STRONG_INLINE Packet1cd padd(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_add_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd psub(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_sub_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd pnegate(const Packet1cd& a) { return Packet1cd(pnegate(Packet2d(a.v))); } template<> EIGEN_STRONG_INLINE Packet1cd pconj(const Packet1cd& a) { const __m128d mask = _mm_castsi128_pd(_mm_set_epi32(0x80000000,0x0,0x0,0x0)); return Packet1cd(_mm_xor_pd(a.v,mask)); } template<> EIGEN_STRONG_INLINE Packet1cd pmul(const Packet1cd& a, const Packet1cd& b) { #ifdef EIGEN_VECTORIZE_SSE3 return Packet1cd(_mm_addsub_pd(_mm_mul_pd(_mm_movedup_pd(a.v), b.v), _mm_mul_pd(vec2d_swizzle1(a.v, 1, 1), vec2d_swizzle1(b.v, 1, 0)))); #else const __m128d mask = _mm_castsi128_pd(_mm_set_epi32(0x0,0x0,0x80000000,0x0)); return Packet1cd(_mm_add_pd(_mm_mul_pd(vec2d_swizzle1(a.v, 0, 0), b.v), _mm_xor_pd(_mm_mul_pd(vec2d_swizzle1(a.v, 1, 1), vec2d_swizzle1(b.v, 1, 0)), mask))); #endif } template<> EIGEN_STRONG_INLINE Packet1cd ptrue (const Packet1cd& a) { return Packet1cd(ptrue(Packet2d(a.v))); } template<> EIGEN_STRONG_INLINE Packet1cd pand (const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_and_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd por (const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_or_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd pxor (const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_xor_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd pandnot(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(_mm_andnot_pd(b.v,a.v)); } // FIXME force unaligned load, this is a temporary fix template<> EIGEN_STRONG_INLINE Packet1cd pload (const std::complex* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet1cd(pload((const double*)from)); } template<> EIGEN_STRONG_INLINE Packet1cd ploadu(const std::complex* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet1cd(ploadu((const double*)from)); } template<> EIGEN_STRONG_INLINE Packet1cd pset1(const std::complex& from) { /* here we really have to use unaligned loads :( */ return ploadu(&from); } template<> EIGEN_STRONG_INLINE Packet1cd ploaddup(const std::complex* from) { return pset1(*from); } // FIXME force unaligned store, this is a temporary fix template<> EIGEN_STRONG_INLINE void pstore >(std::complex * to, const Packet1cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, Packet2d(from.v)); } template<> EIGEN_STRONG_INLINE void pstoreu >(std::complex * to, const Packet1cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, Packet2d(from.v)); } template<> EIGEN_STRONG_INLINE void prefetch >(const std::complex * addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } template<> EIGEN_STRONG_INLINE std::complex pfirst(const Packet1cd& a) { EIGEN_ALIGN16 double res[2]; _mm_store_pd(res, a.v); return std::complex(res[0],res[1]); } template<> EIGEN_STRONG_INLINE Packet1cd preverse(const Packet1cd& a) { return a; } template<> EIGEN_STRONG_INLINE std::complex predux(const Packet1cd& a) { return pfirst(a); } template<> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet1cd& a) { return pfirst(a); } EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd,Packet2d) template<> EIGEN_STRONG_INLINE Packet1cd pdiv(const Packet1cd& a, const Packet1cd& b) { // TODO optimize it for SSE3 and 4 Packet1cd res = pmul(a,pconj(b)); __m128d s = _mm_mul_pd(b.v,b.v); return Packet1cd(_mm_div_pd(res.v, _mm_add_pd(s,_mm_shuffle_pd(s, s, 0x1)))); } EIGEN_STRONG_INLINE Packet1cd pcplxflip/* */(const Packet1cd& x) { return Packet1cd(preverse(Packet2d(x.v))); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { __m128d w1 = _mm_castps_pd(kernel.packet[0].v); __m128d w2 = _mm_castps_pd(kernel.packet[1].v); __m128 tmp = _mm_castpd_ps(_mm_unpackhi_pd(w1, w2)); kernel.packet[0].v = _mm_castpd_ps(_mm_unpacklo_pd(w1, w2)); kernel.packet[1].v = tmp; } template<> EIGEN_STRONG_INLINE Packet2cf pcmp_eq(const Packet2cf& a, const Packet2cf& b) { __m128 eq = _mm_cmpeq_ps(a.v, b.v); return Packet2cf(pand(eq, vec4f_swizzle1(eq, 1, 0, 3, 2))); } template<> EIGEN_STRONG_INLINE Packet1cd pcmp_eq(const Packet1cd& a, const Packet1cd& b) { __m128d eq = _mm_cmpeq_pd(a.v, b.v); return Packet1cd(pand(eq, vec2d_swizzle1(eq, 1, 0))); } template<> EIGEN_STRONG_INLINE Packet2cf pblend(const Selector<2>& ifPacket, const Packet2cf& thenPacket, const Packet2cf& elsePacket) { __m128d result = pblend(ifPacket, _mm_castps_pd(thenPacket.v), _mm_castps_pd(elsePacket.v)); return Packet2cf(_mm_castpd_ps(result)); } template<> EIGEN_STRONG_INLINE Packet1cd psqrt(const Packet1cd& a) { return psqrt_complex(a); } template<> EIGEN_STRONG_INLINE Packet2cf psqrt(const Packet2cf& a) { return psqrt_complex(a); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_COMPLEX_SSE_H RcppEigen/inst/include/Eigen/src/Core/arch/SSE/PacketMath.h0000755000176200001440000017572114567757725023046 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2009 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_PACKET_MATH_SSE_H #define EIGEN_PACKET_MATH_SSE_H namespace Eigen { namespace internal { #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 #endif #if !defined(EIGEN_VECTORIZE_AVX) && !defined(EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS) // 32 bits => 8 registers // 64 bits => 16 registers #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS (2*sizeof(void*)) #endif #ifdef EIGEN_VECTORIZE_FMA #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD #endif #endif #if ((defined EIGEN_VECTORIZE_AVX) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_MINGW) && (__GXX_ABI_VERSION < 1004)) || EIGEN_OS_QNX // With GCC's default ABI version, a __m128 or __m256 are the same types and therefore we cannot // have overloads for both types without linking error. // One solution is to increase ABI version using -fabi-version=4 (or greater). // Otherwise, we workaround this inconvenience by wrapping 128bit types into the following helper // structure: typedef eigen_packet_wrapper<__m128> Packet4f; typedef eigen_packet_wrapper<__m128d> Packet2d; #else typedef __m128 Packet4f; typedef __m128d Packet2d; #endif typedef eigen_packet_wrapper<__m128i, 0> Packet4i; typedef eigen_packet_wrapper<__m128i, 1> Packet16b; template<> struct is_arithmetic<__m128> { enum { value = true }; }; template<> struct is_arithmetic<__m128i> { enum { value = true }; }; template<> struct is_arithmetic<__m128d> { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template struct shuffle_mask{ enum { mask = (s)<<6|(r)<<4|(q)<<2|(p) }; }; // TODO: change the implementation of all swizzle* ops from macro to template, #define vec4f_swizzle1(v,p,q,r,s) \ Packet4f(_mm_castsi128_ps(_mm_shuffle_epi32( _mm_castps_si128(v), (shuffle_mask::mask)))) #define vec4i_swizzle1(v,p,q,r,s) \ Packet4i(_mm_shuffle_epi32( v, (shuffle_mask::mask))) #define vec2d_swizzle1(v,p,q) \ Packet2d(_mm_castsi128_pd(_mm_shuffle_epi32( _mm_castpd_si128(v), (shuffle_mask<2*p,2*p+1,2*q,2*q+1>::mask)))) #define vec4f_swizzle2(a,b,p,q,r,s) \ Packet4f(_mm_shuffle_ps( (a), (b), (shuffle_mask::mask))) #define vec4i_swizzle2(a,b,p,q,r,s) \ Packet4i(_mm_castps_si128( (_mm_shuffle_ps( _mm_castsi128_ps(a), _mm_castsi128_ps(b), (shuffle_mask::mask))))) EIGEN_STRONG_INLINE Packet4f vec4f_movelh(const Packet4f& a, const Packet4f& b) { return Packet4f(_mm_movelh_ps(a,b)); } EIGEN_STRONG_INLINE Packet4f vec4f_movehl(const Packet4f& a, const Packet4f& b) { return Packet4f(_mm_movehl_ps(a,b)); } EIGEN_STRONG_INLINE Packet4f vec4f_unpacklo(const Packet4f& a, const Packet4f& b) { return Packet4f(_mm_unpacklo_ps(a,b)); } EIGEN_STRONG_INLINE Packet4f vec4f_unpackhi(const Packet4f& a, const Packet4f& b) { return Packet4f(_mm_unpackhi_ps(a,b)); } #define vec4f_duplane(a,p) \ vec4f_swizzle2(a,a,p,p,p,p) #define vec2d_swizzle2(a,b,mask) \ Packet2d(_mm_shuffle_pd(a,b,mask)) EIGEN_STRONG_INLINE Packet2d vec2d_unpacklo(const Packet2d& a, const Packet2d& b) { return Packet2d(_mm_unpacklo_pd(a,b)); } EIGEN_STRONG_INLINE Packet2d vec2d_unpackhi(const Packet2d& a, const Packet2d& b) { return Packet2d(_mm_unpackhi_pd(a,b)); } #define vec2d_duplane(a,p) \ vec2d_swizzle2(a,a,(p<<1)|p) #define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ const Packet4f p4f_##NAME = pset1(X) #define _EIGEN_DECLARE_CONST_Packet2d(NAME,X) \ const Packet2d p2d_##NAME = pset1(X) #define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ const Packet4f p4f_##NAME = pset1frombits(X) #define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ const Packet4i p4i_##NAME = pset1(X) // Use the packet_traits defined in AVX/PacketMath.h instead if we're going // to leverage AVX instructions. #ifndef EIGEN_VECTORIZE_AVX template <> struct packet_traits : default_packet_traits { typedef Packet4f type; typedef Packet4f half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 4, HasHalfPacket = 0, HasCmp = 1, HasDiv = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasLog = 1, HasLog1p = 1, HasExpm1 = 1, HasNdtri = 1, HasExp = 1, HasBessel = 1, HasSqrt = 1, HasRsqrt = 1, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, HasBlend = 1, HasCeil = 1, HasFloor = 1, #ifdef EIGEN_VECTORIZE_SSE4_1 HasRound = 1, #endif HasRint = 1 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet2d type; typedef Packet2d half; enum { Vectorizable = 1, AlignedOnScalar = 1, size=2, HasHalfPacket = 0, HasCmp = 1, HasDiv = 1, HasLog = 1, HasExp = 1, HasSqrt = 1, HasRsqrt = 1, HasBlend = 1, HasFloor = 1, HasCeil = 1, #ifdef EIGEN_VECTORIZE_SSE4_1 HasRound = 1, #endif HasRint = 1 }; }; #endif template<> struct packet_traits : default_packet_traits { typedef Packet4i type; typedef Packet4i half; enum { Vectorizable = 1, AlignedOnScalar = 1, size=4, HasShift = 1, HasBlend = 1 }; }; template<> struct packet_traits : default_packet_traits { typedef Packet16b type; typedef Packet16b half; enum { Vectorizable = 1, AlignedOnScalar = 1, HasHalfPacket = 0, size=16, HasAdd = 1, HasSub = 1, HasShift = 0, HasMul = 1, HasNegate = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, HasMax = 0, HasConj = 0, HasSqrt = 1 }; }; template<> struct unpacket_traits { typedef float type; typedef Packet4f half; typedef Packet4i integer_packet; enum {size=4, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; }; template<> struct unpacket_traits { typedef double type; typedef Packet2d half; enum {size=2, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; }; template<> struct unpacket_traits { typedef int type; typedef Packet4i half; enum {size=4, alignment=Aligned16, vectorizable=false, masked_load_available=false, masked_store_available=false}; }; template<> struct unpacket_traits { typedef bool type; typedef Packet16b half; enum {size=16, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; }; #ifndef EIGEN_VECTORIZE_AVX template<> struct scalar_div_cost { enum { value = 7 }; }; template<> struct scalar_div_cost { enum { value = 8 }; }; #endif #if EIGEN_COMP_MSVC==1500 // Workaround MSVC 9 internal compiler error. // TODO: It has been detected with win64 builds (amd64), so let's check whether it also happens in 32bits+SSE mode // TODO: let's check whether there does not exist a better fix, like adding a pset0() function. (it crashed on pset1(0)). template<> EIGEN_STRONG_INLINE Packet4f pset1(const float& from) { return _mm_set_ps(from,from,from,from); } template<> EIGEN_STRONG_INLINE Packet2d pset1(const double& from) { return _mm_set_pd(from,from); } template<> EIGEN_STRONG_INLINE Packet4i pset1(const int& from) { return _mm_set_epi32(from,from,from,from); } #else template<> EIGEN_STRONG_INLINE Packet4f pset1(const float& from) { return _mm_set_ps1(from); } template<> EIGEN_STRONG_INLINE Packet2d pset1(const double& from) { return _mm_set1_pd(from); } template<> EIGEN_STRONG_INLINE Packet4i pset1(const int& from) { return _mm_set1_epi32(from); } #endif template<> EIGEN_STRONG_INLINE Packet16b pset1(const bool& from) { return _mm_set1_epi8(static_cast(from)); } template<> EIGEN_STRONG_INLINE Packet4f pset1frombits(unsigned int from) { return _mm_castsi128_ps(pset1(from)); } template<> EIGEN_STRONG_INLINE Packet2d pset1frombits(uint64_t from) { return _mm_castsi128_pd(_mm_set1_epi64x(from)); } template<> EIGEN_STRONG_INLINE Packet4f peven_mask(const Packet4f& /*a*/) { return _mm_castsi128_ps(_mm_set_epi32(0, -1, 0, -1)); } template<> EIGEN_STRONG_INLINE Packet4i peven_mask(const Packet4i& /*a*/) { return _mm_set_epi32(0, -1, 0, -1); } template<> EIGEN_STRONG_INLINE Packet2d peven_mask(const Packet2d& /*a*/) { return _mm_castsi128_pd(_mm_set_epi32(0, 0, -1, -1)); } template<> EIGEN_STRONG_INLINE Packet4f pzero(const Packet4f& /*a*/) { return _mm_setzero_ps(); } template<> EIGEN_STRONG_INLINE Packet2d pzero(const Packet2d& /*a*/) { return _mm_setzero_pd(); } template<> EIGEN_STRONG_INLINE Packet4i pzero(const Packet4i& /*a*/) { return _mm_setzero_si128(); } // GCC generates a shufps instruction for _mm_set1_ps/_mm_load1_ps instead of the more efficient pshufd instruction. // However, using inrinsics for pset1 makes gcc to generate crappy code in some cases (see bug 203) // Using inline assembly is also not an option because then gcc fails to reorder properly the instructions. // Therefore, we introduced the pload1 functions to be used in product kernels for which bug 203 does not apply. // Also note that with AVX, we want it to generate a vbroadcastss. #if EIGEN_COMP_GNUC_STRICT && (!defined __AVX__) template<> EIGEN_STRONG_INLINE Packet4f pload1(const float *from) { return vec4f_swizzle1(_mm_load_ss(from),0,0,0,0); } #endif template<> EIGEN_STRONG_INLINE Packet4f plset(const float& a) { return _mm_add_ps(pset1(a), _mm_set_ps(3,2,1,0)); } template<> EIGEN_STRONG_INLINE Packet2d plset(const double& a) { return _mm_add_pd(pset1(a),_mm_set_pd(1,0)); } template<> EIGEN_STRONG_INLINE Packet4i plset(const int& a) { return _mm_add_epi32(pset1(a),_mm_set_epi32(3,2,1,0)); } template<> EIGEN_STRONG_INLINE Packet4f padd(const Packet4f& a, const Packet4f& b) { return _mm_add_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d padd(const Packet2d& a, const Packet2d& b) { return _mm_add_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet4i padd(const Packet4i& a, const Packet4i& b) { return _mm_add_epi32(a,b); } template<> EIGEN_STRONG_INLINE Packet16b padd(const Packet16b& a, const Packet16b& b) { return _mm_or_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet4f psub(const Packet4f& a, const Packet4f& b) { return _mm_sub_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d psub(const Packet2d& a, const Packet2d& b) { return _mm_sub_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet4i psub(const Packet4i& a, const Packet4i& b) { return _mm_sub_epi32(a,b); } template<> EIGEN_STRONG_INLINE Packet16b psub(const Packet16b& a, const Packet16b& b) { return _mm_xor_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pxor(const Packet4f& a, const Packet4f& b); template<> EIGEN_STRONG_INLINE Packet4f paddsub(const Packet4f& a, const Packet4f& b) { #ifdef EIGEN_VECTORIZE_SSE3 return _mm_addsub_ps(a,b); #else const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x80000000,0x0,0x80000000,0x0)); return padd(a, pxor(mask, b)); #endif } template<> EIGEN_STRONG_INLINE Packet2d pxor(const Packet2d& , const Packet2d& ); template<> EIGEN_STRONG_INLINE Packet2d paddsub(const Packet2d& a, const Packet2d& b) { #ifdef EIGEN_VECTORIZE_SSE3 return _mm_addsub_pd(a,b); #else const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0x0,0x80000000,0x0,0x0)); return padd(a, pxor(mask, b)); #endif } template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x80000000,0x80000000,0x80000000,0x80000000)); return _mm_xor_ps(a,mask); } template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) { const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0x0,0x80000000,0x0,0x80000000)); return _mm_xor_pd(a,mask); } template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return psub(Packet4i(_mm_setr_epi32(0,0,0,0)), a); } template<> EIGEN_STRONG_INLINE Packet16b pnegate(const Packet16b& a) { return psub(pset1(false), a); } template<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; } template<> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4f pmul(const Packet4f& a, const Packet4f& b) { return _mm_mul_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pmul(const Packet2d& a, const Packet2d& b) { return _mm_mul_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pmul(const Packet4i& a, const Packet4i& b) { #ifdef EIGEN_VECTORIZE_SSE4_1 return _mm_mullo_epi32(a,b); #else // this version is slightly faster than 4 scalar products return vec4i_swizzle1( vec4i_swizzle2( _mm_mul_epu32(a,b), _mm_mul_epu32(vec4i_swizzle1(a,1,0,3,2), vec4i_swizzle1(b,1,0,3,2)), 0,2,0,2), 0,2,1,3); #endif } template<> EIGEN_STRONG_INLINE Packet16b pmul(const Packet16b& a, const Packet16b& b) { return _mm_and_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pdiv(const Packet4f& a, const Packet4f& b) { return _mm_div_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pdiv(const Packet2d& a, const Packet2d& b) { return _mm_div_pd(a,b); } // for some weird raisons, it has to be overloaded for packet of integers template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return padd(pmul(a,b), c); } #ifdef EIGEN_VECTORIZE_FMA template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return _mm_fmadd_ps(a,b,c); } template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return _mm_fmadd_pd(a,b,c); } #endif #ifdef EIGEN_VECTORIZE_SSE4_1 template<> EIGEN_DEVICE_FUNC inline Packet4f pselect(const Packet4f& mask, const Packet4f& a, const Packet4f& b) { return _mm_blendv_ps(b,a,mask); } template<> EIGEN_DEVICE_FUNC inline Packet4i pselect(const Packet4i& mask, const Packet4i& a, const Packet4i& b) { return _mm_castps_si128(_mm_blendv_ps(_mm_castsi128_ps(b),_mm_castsi128_ps(a),_mm_castsi128_ps(mask))); } template<> EIGEN_DEVICE_FUNC inline Packet2d pselect(const Packet2d& mask, const Packet2d& a, const Packet2d& b) { return _mm_blendv_pd(b,a,mask); } template<> EIGEN_DEVICE_FUNC inline Packet16b pselect(const Packet16b& mask, const Packet16b& a, const Packet16b& b) { return _mm_blendv_epi8(b,a,mask); } #else template<> EIGEN_DEVICE_FUNC inline Packet16b pselect(const Packet16b& mask, const Packet16b& a, const Packet16b& b) { Packet16b a_part = _mm_and_si128(mask, a); Packet16b b_part = _mm_andnot_si128(mask, b); return _mm_or_si128(a_part, b_part); } #endif template<> EIGEN_STRONG_INLINE Packet4i ptrue(const Packet4i& a) { return _mm_cmpeq_epi32(a, a); } template<> EIGEN_STRONG_INLINE Packet16b ptrue(const Packet16b& a) { return _mm_cmpeq_epi8(a, a); } template<> EIGEN_STRONG_INLINE Packet4f ptrue(const Packet4f& a) { Packet4i b = _mm_castps_si128(a); return _mm_castsi128_ps(_mm_cmpeq_epi32(b, b)); } template<> EIGEN_STRONG_INLINE Packet2d ptrue(const Packet2d& a) { Packet4i b = _mm_castpd_si128(a); return _mm_castsi128_pd(_mm_cmpeq_epi32(b, b)); } template<> EIGEN_STRONG_INLINE Packet4f pand(const Packet4f& a, const Packet4f& b) { return _mm_and_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pand(const Packet2d& a, const Packet2d& b) { return _mm_and_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pand(const Packet4i& a, const Packet4i& b) { return _mm_and_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet16b pand(const Packet16b& a, const Packet16b& b) { return _mm_and_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet4f por(const Packet4f& a, const Packet4f& b) { return _mm_or_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d por(const Packet2d& a, const Packet2d& b) { return _mm_or_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet4i por(const Packet4i& a, const Packet4i& b) { return _mm_or_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet16b por(const Packet16b& a, const Packet16b& b) { return _mm_or_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pxor(const Packet4f& a, const Packet4f& b) { return _mm_xor_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pxor(const Packet2d& a, const Packet2d& b) { return _mm_xor_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pxor(const Packet4i& a, const Packet4i& b) { return _mm_xor_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet16b pxor(const Packet16b& a, const Packet16b& b) { return _mm_xor_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pandnot(const Packet4f& a, const Packet4f& b) { return _mm_andnot_ps(b,a); } template<> EIGEN_STRONG_INLINE Packet2d pandnot(const Packet2d& a, const Packet2d& b) { return _mm_andnot_pd(b,a); } template<> EIGEN_STRONG_INLINE Packet4i pandnot(const Packet4i& a, const Packet4i& b) { return _mm_andnot_si128(b,a); } template<> EIGEN_STRONG_INLINE Packet4f pcmp_le(const Packet4f& a, const Packet4f& b) { return _mm_cmple_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt(const Packet4f& a, const Packet4f& b) { return _mm_cmplt_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan(const Packet4f& a, const Packet4f& b) { return _mm_cmpnge_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pcmp_eq(const Packet4f& a, const Packet4f& b) { return _mm_cmpeq_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pcmp_le(const Packet2d& a, const Packet2d& b) { return _mm_cmple_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pcmp_lt(const Packet2d& a, const Packet2d& b) { return _mm_cmplt_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan(const Packet2d& a, const Packet2d& b) { return _mm_cmpnge_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pcmp_eq(const Packet2d& a, const Packet2d& b) { return _mm_cmpeq_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pcmp_lt(const Packet4i& a, const Packet4i& b) { return _mm_cmplt_epi32(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pcmp_eq(const Packet4i& a, const Packet4i& b) { return _mm_cmpeq_epi32(a,b); } template<> EIGEN_STRONG_INLINE Packet16b pcmp_eq(const Packet16b& a, const Packet16b& b) { return _mm_cmpeq_epi8(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pcmp_le(const Packet4i& a, const Packet4i& b) { return por(pcmp_lt(a,b), pcmp_eq(a,b)); } template<> EIGEN_STRONG_INLINE Packet4f pmin(const Packet4f& a, const Packet4f& b) { #if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 // There appears to be a bug in GCC, by which the optimizer may // flip the argument order in calls to _mm_min_ps, so we have to // resort to inline ASM here. This is supposed to be fixed in gcc6.3, // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 #ifdef EIGEN_VECTORIZE_AVX Packet4f res; asm("vminps %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); #else Packet4f res = b; asm("minps %[a], %[res]" : [res] "+x" (res) : [a] "x" (a)); #endif return res; #else // Arguments are reversed to match NaN propagation behavior of std::min. return _mm_min_ps(b, a); #endif } template<> EIGEN_STRONG_INLINE Packet2d pmin(const Packet2d& a, const Packet2d& b) { #if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 // There appears to be a bug in GCC, by which the optimizer may // flip the argument order in calls to _mm_min_pd, so we have to // resort to inline ASM here. This is supposed to be fixed in gcc6.3, // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 #ifdef EIGEN_VECTORIZE_AVX Packet2d res; asm("vminpd %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); #else Packet2d res = b; asm("minpd %[a], %[res]" : [res] "+x" (res) : [a] "x" (a)); #endif return res; #else // Arguments are reversed to match NaN propagation behavior of std::min. return _mm_min_pd(b, a); #endif } template<> EIGEN_STRONG_INLINE Packet4i pmin(const Packet4i& a, const Packet4i& b) { #ifdef EIGEN_VECTORIZE_SSE4_1 return _mm_min_epi32(a,b); #else // after some bench, this version *is* faster than a scalar implementation Packet4i mask = _mm_cmplt_epi32(a,b); return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b)); #endif } template<> EIGEN_STRONG_INLINE Packet4f pmax(const Packet4f& a, const Packet4f& b) { #if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 // There appears to be a bug in GCC, by which the optimizer may // flip the argument order in calls to _mm_max_ps, so we have to // resort to inline ASM here. This is supposed to be fixed in gcc6.3, // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 #ifdef EIGEN_VECTORIZE_AVX Packet4f res; asm("vmaxps %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); #else Packet4f res = b; asm("maxps %[a], %[res]" : [res] "+x" (res) : [a] "x" (a)); #endif return res; #else // Arguments are reversed to match NaN propagation behavior of std::max. return _mm_max_ps(b, a); #endif } template<> EIGEN_STRONG_INLINE Packet2d pmax(const Packet2d& a, const Packet2d& b) { #if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 // There appears to be a bug in GCC, by which the optimizer may // flip the argument order in calls to _mm_max_pd, so we have to // resort to inline ASM here. This is supposed to be fixed in gcc6.3, // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 #ifdef EIGEN_VECTORIZE_AVX Packet2d res; asm("vmaxpd %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); #else Packet2d res = b; asm("maxpd %[a], %[res]" : [res] "+x" (res) : [a] "x" (a)); #endif return res; #else // Arguments are reversed to match NaN propagation behavior of std::max. return _mm_max_pd(b, a); #endif } template<> EIGEN_STRONG_INLINE Packet4i pmax(const Packet4i& a, const Packet4i& b) { #ifdef EIGEN_VECTORIZE_SSE4_1 return _mm_max_epi32(a,b); #else // after some bench, this version *is* faster than a scalar implementation Packet4i mask = _mm_cmpgt_epi32(a,b); return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b)); #endif } template EIGEN_STRONG_INLINE Packet pminmax_propagate_numbers(const Packet& a, const Packet& b, Op op) { // In this implementation, we take advantage of the fact that pmin/pmax for SSE // always return a if either a or b is NaN. Packet not_nan_mask_a = pcmp_eq(a, a); Packet m = op(a, b); return pselect(not_nan_mask_a, m, b); } template EIGEN_STRONG_INLINE Packet pminmax_propagate_nan(const Packet& a, const Packet& b, Op op) { // In this implementation, we take advantage of the fact that pmin/pmax for SSE // always return a if either a or b is NaN. Packet not_nan_mask_a = pcmp_eq(a, a); Packet m = op(b, a); return pselect(not_nan_mask_a, m, a); } // Add specializations for min/max with prescribed NaN progation. template<> EIGEN_STRONG_INLINE Packet4f pmin(const Packet4f& a, const Packet4f& b) { return pminmax_propagate_numbers(a, b, pmin); } template<> EIGEN_STRONG_INLINE Packet2d pmin(const Packet2d& a, const Packet2d& b) { return pminmax_propagate_numbers(a, b, pmin); } template<> EIGEN_STRONG_INLINE Packet4f pmax(const Packet4f& a, const Packet4f& b) { return pminmax_propagate_numbers(a, b, pmax); } template<> EIGEN_STRONG_INLINE Packet2d pmax(const Packet2d& a, const Packet2d& b) { return pminmax_propagate_numbers(a, b, pmax); } template<> EIGEN_STRONG_INLINE Packet4f pmin(const Packet4f& a, const Packet4f& b) { return pminmax_propagate_nan(a, b, pmin); } template<> EIGEN_STRONG_INLINE Packet2d pmin(const Packet2d& a, const Packet2d& b) { return pminmax_propagate_nan(a, b, pmin); } template<> EIGEN_STRONG_INLINE Packet4f pmax(const Packet4f& a, const Packet4f& b) { return pminmax_propagate_nan(a, b, pmax); } template<> EIGEN_STRONG_INLINE Packet2d pmax(const Packet2d& a, const Packet2d& b) { return pminmax_propagate_nan(a, b, pmax); } template EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(const Packet4i& a) { return _mm_srai_epi32(a,N); } template EIGEN_STRONG_INLINE Packet4i plogical_shift_right (const Packet4i& a) { return _mm_srli_epi32(a,N); } template EIGEN_STRONG_INLINE Packet4i plogical_shift_left (const Packet4i& a) { return _mm_slli_epi32(a,N); } template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF)); return _mm_and_ps(a,mask); } template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0xFFFFFFFF,0x7FFFFFFF,0xFFFFFFFF,0x7FFFFFFF)); return _mm_and_pd(a,mask); } template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { #ifdef EIGEN_VECTORIZE_SSSE3 return _mm_abs_epi32(a); #else Packet4i aux = _mm_srai_epi32(a,31); return _mm_sub_epi32(_mm_xor_si128(a,aux),aux); #endif } #ifdef EIGEN_VECTORIZE_SSE4_1 template<> EIGEN_STRONG_INLINE Packet4f pround(const Packet4f& a) { // Unfortunatly _mm_round_ps doesn't have a rounding mode to implement numext::round. const Packet4f mask = pset1frombits(0x80000000u); const Packet4f prev0dot5 = pset1frombits(0x3EFFFFFFu); return _mm_round_ps(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO); } template<> EIGEN_STRONG_INLINE Packet2d pround(const Packet2d& a) { const Packet2d mask = _mm_castsi128_pd(_mm_set_epi64x(0x8000000000000000ull, 0x8000000000000000ull)); const Packet2d prev0dot5 = _mm_castsi128_pd(_mm_set_epi64x(0x3FDFFFFFFFFFFFFFull, 0x3FDFFFFFFFFFFFFFull)); return _mm_round_pd(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO); } template<> EIGEN_STRONG_INLINE Packet4f print(const Packet4f& a) { return _mm_round_ps(a, _MM_FROUND_CUR_DIRECTION); } template<> EIGEN_STRONG_INLINE Packet2d print(const Packet2d& a) { return _mm_round_pd(a, _MM_FROUND_CUR_DIRECTION); } template<> EIGEN_STRONG_INLINE Packet4f pceil(const Packet4f& a) { return _mm_ceil_ps(a); } template<> EIGEN_STRONG_INLINE Packet2d pceil(const Packet2d& a) { return _mm_ceil_pd(a); } template<> EIGEN_STRONG_INLINE Packet4f pfloor(const Packet4f& a) { return _mm_floor_ps(a); } template<> EIGEN_STRONG_INLINE Packet2d pfloor(const Packet2d& a) { return _mm_floor_pd(a); } #else template<> EIGEN_STRONG_INLINE Packet4f print(const Packet4f& a) { // Adds and subtracts signum(a) * 2^23 to force rounding. const Packet4f limit = pset1(static_cast(1<<23)); const Packet4f abs_a = pabs(a); Packet4f r = padd(abs_a, limit); // Don't compile-away addition and subtraction. EIGEN_OPTIMIZATION_BARRIER(r); r = psub(r, limit); // If greater than limit, simply return a. Otherwise, account for sign. r = pselect(pcmp_lt(abs_a, limit), pselect(pcmp_lt(a, pzero(a)), pnegate(r), r), a); return r; } template<> EIGEN_STRONG_INLINE Packet2d print(const Packet2d& a) { // Adds and subtracts signum(a) * 2^52 to force rounding. const Packet2d limit = pset1(static_cast(1ull<<52)); const Packet2d abs_a = pabs(a); Packet2d r = padd(abs_a, limit); // Don't compile-away addition and subtraction. EIGEN_OPTIMIZATION_BARRIER(r); r = psub(r, limit); // If greater than limit, simply return a. Otherwise, account for sign. r = pselect(pcmp_lt(abs_a, limit), pselect(pcmp_lt(a, pzero(a)), pnegate(r), r), a); return r; } template<> EIGEN_STRONG_INLINE Packet4f pfloor(const Packet4f& a) { const Packet4f cst_1 = pset1(1.0f); Packet4f tmp = print(a); // If greater, subtract one. Packet4f mask = _mm_cmpgt_ps(tmp, a); mask = pand(mask, cst_1); return psub(tmp, mask); } template<> EIGEN_STRONG_INLINE Packet2d pfloor(const Packet2d& a) { const Packet2d cst_1 = pset1(1.0); Packet2d tmp = print(a); // If greater, subtract one. Packet2d mask = _mm_cmpgt_pd(tmp, a); mask = pand(mask, cst_1); return psub(tmp, mask); } template<> EIGEN_STRONG_INLINE Packet4f pceil(const Packet4f& a) { const Packet4f cst_1 = pset1(1.0f); Packet4f tmp = print(a); // If smaller, add one. Packet4f mask = _mm_cmplt_ps(tmp, a); mask = pand(mask, cst_1); return padd(tmp, mask); } template<> EIGEN_STRONG_INLINE Packet2d pceil(const Packet2d& a) { const Packet2d cst_1 = pset1(1.0); Packet2d tmp = print(a); // If smaller, add one. Packet2d mask = _mm_cmplt_pd(tmp, a); mask = pand(mask, cst_1); return padd(tmp, mask); } #endif template<> EIGEN_STRONG_INLINE Packet4f pload(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_ps(from); } template<> EIGEN_STRONG_INLINE Packet2d pload(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_pd(from); } template<> EIGEN_STRONG_INLINE Packet4i pload(const int* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE Packet16b pload(const bool* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast(from)); } #if EIGEN_COMP_MSVC template<> EIGEN_STRONG_INLINE Packet4f ploadu(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD #if (EIGEN_COMP_MSVC==1600) // NOTE Some version of MSVC10 generates bad code when using _mm_loadu_ps // (i.e., it does not generate an unaligned load!! __m128 res = _mm_loadl_pi(_mm_set1_ps(0.0f), (const __m64*)(from)); res = _mm_loadh_pi(res, (const __m64*)(from+2)); return res; #else return _mm_loadu_ps(from); #endif } #else // NOTE: with the code below, MSVC's compiler crashes! template<> EIGEN_STRONG_INLINE Packet4f ploadu(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_ps(from); } #endif template<> EIGEN_STRONG_INLINE Packet2d ploadu(const double* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_pd(from); } template<> EIGEN_STRONG_INLINE Packet4i ploadu(const int* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_si128(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE Packet16b ploadu(const bool* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_si128(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE Packet4f ploaddup(const float* from) { return vec4f_swizzle1(_mm_castpd_ps(_mm_load_sd(reinterpret_cast(from))), 0, 0, 1, 1); } template<> EIGEN_STRONG_INLINE Packet2d ploaddup(const double* from) { return pset1(from[0]); } template<> EIGEN_STRONG_INLINE Packet4i ploaddup(const int* from) { Packet4i tmp; tmp = _mm_loadl_epi64(reinterpret_cast(from)); return vec4i_swizzle1(tmp, 0, 0, 1, 1); } // Loads 8 bools from memory and returns the packet // {b0, b0, b1, b1, b2, b2, b3, b3, b4, b4, b5, b5, b6, b6, b7, b7} template<> EIGEN_STRONG_INLINE Packet16b ploaddup(const bool* from) { __m128i tmp = _mm_castpd_si128(pload1(reinterpret_cast(from))); return _mm_unpacklo_epi8(tmp, tmp); } // Loads 4 bools from memory and returns the packet // {b0, b0 b0, b0, b1, b1, b1, b1, b2, b2, b2, b2, b3, b3, b3, b3} template<> EIGEN_STRONG_INLINE Packet16b ploadquad(const bool* from) { __m128i tmp = _mm_castps_si128(pload1(reinterpret_cast(from))); tmp = _mm_unpacklo_epi8(tmp, tmp); return _mm_unpacklo_epi16(tmp, tmp); } template<> EIGEN_STRONG_INLINE void pstore(float* to, const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_ps(to, from); } template<> EIGEN_STRONG_INLINE void pstore(double* to, const Packet2d& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_pd(to, from); } template<> EIGEN_STRONG_INLINE void pstore(int* to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<__m128i*>(to), from); } template<> EIGEN_STRONG_INLINE void pstore(bool* to, const Packet16b& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<__m128i*>(to), from); } template<> EIGEN_STRONG_INLINE void pstoreu(double* to, const Packet2d& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_pd(to, from); } template<> EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_ps(to, from); } template<> EIGEN_STRONG_INLINE void pstoreu(int* to, const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from); } template<> EIGEN_STRONG_INLINE void pstoreu(bool* to, const Packet16b& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from); } template<> EIGEN_DEVICE_FUNC inline Packet4f pgather(const float* from, Index stride) { return _mm_set_ps(from[3*stride], from[2*stride], from[1*stride], from[0*stride]); } template<> EIGEN_DEVICE_FUNC inline Packet2d pgather(const double* from, Index stride) { return _mm_set_pd(from[1*stride], from[0*stride]); } template<> EIGEN_DEVICE_FUNC inline Packet4i pgather(const int* from, Index stride) { return _mm_set_epi32(from[3*stride], from[2*stride], from[1*stride], from[0*stride]); } template<> EIGEN_DEVICE_FUNC inline Packet16b pgather(const bool* from, Index stride) { return _mm_set_epi8(from[15*stride], from[14*stride], from[13*stride], from[12*stride], from[11*stride], from[10*stride], from[9*stride], from[8*stride], from[7*stride], from[6*stride], from[5*stride], from[4*stride], from[3*stride], from[2*stride], from[1*stride], from[0*stride]); } template<> EIGEN_DEVICE_FUNC inline void pscatter(float* to, const Packet4f& from, Index stride) { to[stride*0] = _mm_cvtss_f32(from); to[stride*1] = _mm_cvtss_f32(_mm_shuffle_ps(from, from, 1)); to[stride*2] = _mm_cvtss_f32(_mm_shuffle_ps(from, from, 2)); to[stride*3] = _mm_cvtss_f32(_mm_shuffle_ps(from, from, 3)); } template<> EIGEN_DEVICE_FUNC inline void pscatter(double* to, const Packet2d& from, Index stride) { to[stride*0] = _mm_cvtsd_f64(from); to[stride*1] = _mm_cvtsd_f64(_mm_shuffle_pd(from, from, 1)); } template<> EIGEN_DEVICE_FUNC inline void pscatter(int* to, const Packet4i& from, Index stride) { to[stride*0] = _mm_cvtsi128_si32(from); to[stride*1] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 1)); to[stride*2] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 2)); to[stride*3] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 3)); } template<> EIGEN_DEVICE_FUNC inline void pscatter(bool* to, const Packet16b& from, Index stride) { to[4*stride*0] = _mm_cvtsi128_si32(from); to[4*stride*1] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 1)); to[4*stride*2] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 2)); to[4*stride*3] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 3)); } // some compilers might be tempted to perform multiple moves instead of using a vector path. template<> EIGEN_STRONG_INLINE void pstore1(float* to, const float& a) { Packet4f pa = _mm_set_ss(a); pstore(to, Packet4f(vec4f_swizzle1(pa,0,0,0,0))); } // some compilers might be tempted to perform multiple moves instead of using a vector path. template<> EIGEN_STRONG_INLINE void pstore1(double* to, const double& a) { Packet2d pa = _mm_set_sd(a); pstore(to, Packet2d(vec2d_swizzle1(pa,0,0))); } #if EIGEN_COMP_PGI && EIGEN_COMP_PGI < 1900 typedef const void * SsePrefetchPtrType; #else typedef const char * SsePrefetchPtrType; #endif #ifndef EIGEN_VECTORIZE_AVX template<> EIGEN_STRONG_INLINE void prefetch(const float* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } template<> EIGEN_STRONG_INLINE void prefetch(const double* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } template<> EIGEN_STRONG_INLINE void prefetch(const int* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } #endif #if EIGEN_COMP_MSVC_STRICT && EIGEN_OS_WIN64 // The temporary variable fixes an internal compilation error in vs <= 2008 and a wrong-result bug in vs 2010 // Direct of the struct members fixed bug #62. template<> EIGEN_STRONG_INLINE float pfirst(const Packet4f& a) { return a.m128_f32[0]; } template<> EIGEN_STRONG_INLINE double pfirst(const Packet2d& a) { return a.m128d_f64[0]; } template<> EIGEN_STRONG_INLINE int pfirst(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; } #elif EIGEN_COMP_MSVC_STRICT // The temporary variable fixes an internal compilation error in vs <= 2008 and a wrong-result bug in vs 2010 template<> EIGEN_STRONG_INLINE float pfirst(const Packet4f& a) { float x = _mm_cvtss_f32(a); return x; } template<> EIGEN_STRONG_INLINE double pfirst(const Packet2d& a) { double x = _mm_cvtsd_f64(a); return x; } template<> EIGEN_STRONG_INLINE int pfirst(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; } #else template<> EIGEN_STRONG_INLINE float pfirst(const Packet4f& a) { return _mm_cvtss_f32(a); } template<> EIGEN_STRONG_INLINE double pfirst(const Packet2d& a) { return _mm_cvtsd_f64(a); } template<> EIGEN_STRONG_INLINE int pfirst(const Packet4i& a) { return _mm_cvtsi128_si32(a); } #endif template<> EIGEN_STRONG_INLINE bool pfirst(const Packet16b& a) { int x = _mm_cvtsi128_si32(a); return static_cast(x & 1); } template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) { return _mm_shuffle_ps(a,a,0x1B); } template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) { return _mm_shuffle_pd(a,a,0x1); } template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) { return _mm_shuffle_epi32(a,0x1B); } template<> EIGEN_STRONG_INLINE Packet16b preverse(const Packet16b& a) { #ifdef EIGEN_VECTORIZE_SSSE3 __m128i mask = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); return _mm_shuffle_epi8(a, mask); #else Packet16b tmp = _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 1, 2, 3)); tmp = _mm_shufflehi_epi16(_mm_shufflelo_epi16(tmp, _MM_SHUFFLE(2, 3, 0, 1)), _MM_SHUFFLE(2, 3, 0, 1)); return _mm_or_si128(_mm_slli_epi16(tmp, 8), _mm_srli_epi16(tmp, 8)); #endif } template<> EIGEN_STRONG_INLINE Packet4f pfrexp(const Packet4f& a, Packet4f& exponent) { return pfrexp_generic(a,exponent); } // Extract exponent without existence of Packet2l. template<> EIGEN_STRONG_INLINE Packet2d pfrexp_generic_get_biased_exponent(const Packet2d& a) { const Packet2d cst_exp_mask = pset1frombits(static_cast(0x7ff0000000000000ull)); __m128i a_expo = _mm_srli_epi64(_mm_castpd_si128(pand(a, cst_exp_mask)), 52); return _mm_cvtepi32_pd(vec4i_swizzle1(a_expo, 0, 2, 1, 3)); } template<> EIGEN_STRONG_INLINE Packet2d pfrexp(const Packet2d& a, Packet2d& exponent) { return pfrexp_generic(a, exponent); } template<> EIGEN_STRONG_INLINE Packet4f pldexp(const Packet4f& a, const Packet4f& exponent) { return pldexp_generic(a,exponent); } // We specialize pldexp here, since the generic implementation uses Packet2l, which is not well // supported by SSE, and has more range than is needed for exponents. template<> EIGEN_STRONG_INLINE Packet2d pldexp(const Packet2d& a, const Packet2d& exponent) { // Clamp exponent to [-2099, 2099] const Packet2d max_exponent = pset1(2099.0); const Packet2d e = pmin(pmax(exponent, pnegate(max_exponent)), max_exponent); // Convert e to integer and swizzle to low-order bits. const Packet4i ei = vec4i_swizzle1(_mm_cvtpd_epi32(e), 0, 3, 1, 3); // Split 2^e into four factors and multiply: const Packet4i bias = _mm_set_epi32(0, 1023, 0, 1023); Packet4i b = parithmetic_shift_right<2>(ei); // floor(e/4) Packet2d c = _mm_castsi128_pd(_mm_slli_epi64(padd(b, bias), 52)); // 2^b Packet2d out = pmul(pmul(pmul(a, c), c), c); // a * 2^(3b) b = psub(psub(psub(ei, b), b), b); // e - 3b c = _mm_castsi128_pd(_mm_slli_epi64(padd(b, bias), 52)); // 2^(e - 3b) out = pmul(out, c); // a * 2^e return out; } // with AVX, the default implementations based on pload1 are faster #ifndef __AVX__ template<> EIGEN_STRONG_INLINE void pbroadcast4(const float *a, Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) { a3 = pload(a); a0 = vec4f_swizzle1(a3, 0,0,0,0); a1 = vec4f_swizzle1(a3, 1,1,1,1); a2 = vec4f_swizzle1(a3, 2,2,2,2); a3 = vec4f_swizzle1(a3, 3,3,3,3); } template<> EIGEN_STRONG_INLINE void pbroadcast4(const double *a, Packet2d& a0, Packet2d& a1, Packet2d& a2, Packet2d& a3) { #ifdef EIGEN_VECTORIZE_SSE3 a0 = _mm_loaddup_pd(a+0); a1 = _mm_loaddup_pd(a+1); a2 = _mm_loaddup_pd(a+2); a3 = _mm_loaddup_pd(a+3); #else a1 = pload(a); a0 = vec2d_swizzle1(a1, 0,0); a1 = vec2d_swizzle1(a1, 1,1); a3 = pload(a+2); a2 = vec2d_swizzle1(a3, 0,0); a3 = vec2d_swizzle1(a3, 1,1); #endif } #endif EIGEN_STRONG_INLINE void punpackp(Packet4f* vecs) { vecs[1] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x55)); vecs[2] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xAA)); vecs[3] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xFF)); vecs[0] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x00)); } template<> EIGEN_STRONG_INLINE float predux(const Packet4f& a) { // Disable SSE3 _mm_hadd_pd that is extremely slow on all existing Intel's architectures // (from Nehalem to Haswell) // #ifdef EIGEN_VECTORIZE_SSE3 // Packet4f tmp = _mm_add_ps(a, vec4f_swizzle1(a,2,3,2,3)); // return pfirst(_mm_hadd_ps(tmp, tmp)); // #else Packet4f tmp = _mm_add_ps(a, _mm_movehl_ps(a,a)); return pfirst(_mm_add_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1))); // #endif } template<> EIGEN_STRONG_INLINE double predux(const Packet2d& a) { // Disable SSE3 _mm_hadd_pd that is extremely slow on all existing Intel's architectures // (from Nehalem to Haswell) // #ifdef EIGEN_VECTORIZE_SSE3 // return pfirst(_mm_hadd_pd(a, a)); // #else return pfirst(_mm_add_sd(a, _mm_unpackhi_pd(a,a))); // #endif } #ifdef EIGEN_VECTORIZE_SSSE3 template<> EIGEN_STRONG_INLINE int predux(const Packet4i& a) { Packet4i tmp0 = _mm_hadd_epi32(a,a); return pfirst(_mm_hadd_epi32(tmp0,tmp0)); } #else template<> EIGEN_STRONG_INLINE int predux(const Packet4i& a) { Packet4i tmp = _mm_add_epi32(a, _mm_unpackhi_epi64(a,a)); return pfirst(tmp) + pfirst(_mm_shuffle_epi32(tmp, 1)); } #endif template<> EIGEN_STRONG_INLINE bool predux(const Packet16b& a) { Packet4i tmp = _mm_or_si128(a, _mm_unpackhi_epi64(a,a)); return (pfirst(tmp) != 0) || (pfirst(_mm_shuffle_epi32(tmp, 1)) != 0); } // Other reduction functions: // mul template<> EIGEN_STRONG_INLINE float predux_mul(const Packet4f& a) { Packet4f tmp = _mm_mul_ps(a, _mm_movehl_ps(a,a)); return pfirst(_mm_mul_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1))); } template<> EIGEN_STRONG_INLINE double predux_mul(const Packet2d& a) { return pfirst(_mm_mul_sd(a, _mm_unpackhi_pd(a,a))); } template<> EIGEN_STRONG_INLINE int predux_mul(const Packet4i& a) { // after some experiments, it is seems this is the fastest way to implement it // for GCC (eg., reusing pmul is very slow !) // TODO try to call _mm_mul_epu32 directly EIGEN_ALIGN16 int aux[4]; pstore(aux, a); return (aux[0] * aux[1]) * (aux[2] * aux[3]); } template<> EIGEN_STRONG_INLINE bool predux_mul(const Packet16b& a) { Packet4i tmp = _mm_and_si128(a, _mm_unpackhi_epi64(a,a)); return ((pfirst(tmp) == 0x01010101) && (pfirst(_mm_shuffle_epi32(tmp, 1)) == 0x01010101)); } // min template<> EIGEN_STRONG_INLINE float predux_min(const Packet4f& a) { Packet4f tmp = _mm_min_ps(a, _mm_movehl_ps(a,a)); return pfirst(_mm_min_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1))); } template<> EIGEN_STRONG_INLINE double predux_min(const Packet2d& a) { return pfirst(_mm_min_sd(a, _mm_unpackhi_pd(a,a))); } template<> EIGEN_STRONG_INLINE int predux_min(const Packet4i& a) { #ifdef EIGEN_VECTORIZE_SSE4_1 Packet4i tmp = _mm_min_epi32(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0,0,3,2))); return pfirst(_mm_min_epi32(tmp,_mm_shuffle_epi32(tmp, 1))); #else // after some experiments, it is seems this is the fastest way to implement it // for GCC (eg., it does not like using std::min after the pstore !!) EIGEN_ALIGN16 int aux[4]; pstore(aux, a); int aux0 = aux[0] EIGEN_STRONG_INLINE float predux_max(const Packet4f& a) { Packet4f tmp = _mm_max_ps(a, _mm_movehl_ps(a,a)); return pfirst(_mm_max_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1))); } template<> EIGEN_STRONG_INLINE double predux_max(const Packet2d& a) { return pfirst(_mm_max_sd(a, _mm_unpackhi_pd(a,a))); } template<> EIGEN_STRONG_INLINE int predux_max(const Packet4i& a) { #ifdef EIGEN_VECTORIZE_SSE4_1 Packet4i tmp = _mm_max_epi32(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0,0,3,2))); return pfirst(_mm_max_epi32(tmp,_mm_shuffle_epi32(tmp, 1))); #else // after some experiments, it is seems this is the fastest way to implement it // for GCC (eg., it does not like using std::min after the pstore !!) EIGEN_ALIGN16 int aux[4]; pstore(aux, a); int aux0 = aux[0]>aux[1] ? aux[0] : aux[1]; int aux2 = aux[2]>aux[3] ? aux[2] : aux[3]; return aux0>aux2 ? aux0 : aux2; #endif // EIGEN_VECTORIZE_SSE4_1 } // not needed yet // template<> EIGEN_STRONG_INLINE bool predux_all(const Packet4f& x) // { // return _mm_movemask_ps(x) == 0xF; // } template<> EIGEN_STRONG_INLINE bool predux_any(const Packet4f& x) { return _mm_movemask_ps(x) != 0x0; } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { _MM_TRANSPOSE4_PS(kernel.packet[0], kernel.packet[1], kernel.packet[2], kernel.packet[3]); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { __m128d tmp = _mm_unpackhi_pd(kernel.packet[0], kernel.packet[1]); kernel.packet[0] = _mm_unpacklo_pd(kernel.packet[0], kernel.packet[1]); kernel.packet[1] = tmp; } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { __m128i T0 = _mm_unpacklo_epi32(kernel.packet[0], kernel.packet[1]); __m128i T1 = _mm_unpacklo_epi32(kernel.packet[2], kernel.packet[3]); __m128i T2 = _mm_unpackhi_epi32(kernel.packet[0], kernel.packet[1]); __m128i T3 = _mm_unpackhi_epi32(kernel.packet[2], kernel.packet[3]); kernel.packet[0] = _mm_unpacklo_epi64(T0, T1); kernel.packet[1] = _mm_unpackhi_epi64(T0, T1); kernel.packet[2] = _mm_unpacklo_epi64(T2, T3); kernel.packet[3] = _mm_unpackhi_epi64(T2, T3); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { __m128i T0 = _mm_unpacklo_epi8(kernel.packet[0], kernel.packet[1]); __m128i T1 = _mm_unpackhi_epi8(kernel.packet[0], kernel.packet[1]); __m128i T2 = _mm_unpacklo_epi8(kernel.packet[2], kernel.packet[3]); __m128i T3 = _mm_unpackhi_epi8(kernel.packet[2], kernel.packet[3]); kernel.packet[0] = _mm_unpacklo_epi16(T0, T2); kernel.packet[1] = _mm_unpackhi_epi16(T0, T2); kernel.packet[2] = _mm_unpacklo_epi16(T1, T3); kernel.packet[3] = _mm_unpackhi_epi16(T1, T3); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { // If we number the elements in the input thus: // kernel.packet[ 0] = {00, 01, 02, 03, 04, 05, 06, 07, 08, 09, 0a, 0b, 0c, 0d, 0e, 0f} // kernel.packet[ 1] = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 1a, 1b, 1c, 1d, 1e, 1f} // ... // kernel.packet[15] = {f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, fa, fb, fc, fd, fe, ff}, // // the desired output is: // kernel.packet[ 0] = {00, 10, 20, 30, 40, 50, 60, 70, 80, 90, a0, b0, c0, d0, e0, f0} // kernel.packet[ 1] = {01, 11, 21, 31, 41, 51, 61, 71, 81, 91, a1, b1, c1, d1, e1, f1} // ... // kernel.packet[15] = {0f, 1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f, 9f, af, bf, cf, df, ef, ff}, __m128i t0 = _mm_unpacklo_epi8(kernel.packet[0], kernel.packet[1]); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17 __m128i t1 = _mm_unpackhi_epi8(kernel.packet[0], kernel.packet[1]); // 08 18 09 19 0a 1a 0b 1b 0c 1c 0d 1d 0e 1e 0f 1f __m128i t2 = _mm_unpacklo_epi8(kernel.packet[2], kernel.packet[3]); // 20 30 21 31 22 32 ... 27 37 __m128i t3 = _mm_unpackhi_epi8(kernel.packet[2], kernel.packet[3]); // 28 38 29 39 2a 3a ... 2f 3f __m128i t4 = _mm_unpacklo_epi8(kernel.packet[4], kernel.packet[5]); // 40 50 41 51 42 52 47 57 __m128i t5 = _mm_unpackhi_epi8(kernel.packet[4], kernel.packet[5]); // 48 58 49 59 4a 5a __m128i t6 = _mm_unpacklo_epi8(kernel.packet[6], kernel.packet[7]); __m128i t7 = _mm_unpackhi_epi8(kernel.packet[6], kernel.packet[7]); __m128i t8 = _mm_unpacklo_epi8(kernel.packet[8], kernel.packet[9]); __m128i t9 = _mm_unpackhi_epi8(kernel.packet[8], kernel.packet[9]); __m128i ta = _mm_unpacklo_epi8(kernel.packet[10], kernel.packet[11]); __m128i tb = _mm_unpackhi_epi8(kernel.packet[10], kernel.packet[11]); __m128i tc = _mm_unpacklo_epi8(kernel.packet[12], kernel.packet[13]); __m128i td = _mm_unpackhi_epi8(kernel.packet[12], kernel.packet[13]); __m128i te = _mm_unpacklo_epi8(kernel.packet[14], kernel.packet[15]); __m128i tf = _mm_unpackhi_epi8(kernel.packet[14], kernel.packet[15]); __m128i s0 = _mm_unpacklo_epi16(t0, t2); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 __m128i s1 = _mm_unpackhi_epi16(t0, t2); // 04 14 24 34 __m128i s2 = _mm_unpacklo_epi16(t1, t3); // 08 18 28 38 ... __m128i s3 = _mm_unpackhi_epi16(t1, t3); // 0c 1c 2c 3c ... __m128i s4 = _mm_unpacklo_epi16(t4, t6); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73 __m128i s5 = _mm_unpackhi_epi16(t4, t6); // 44 54 64 74 ... __m128i s6 = _mm_unpacklo_epi16(t5, t7); __m128i s7 = _mm_unpackhi_epi16(t5, t7); __m128i s8 = _mm_unpacklo_epi16(t8, ta); __m128i s9 = _mm_unpackhi_epi16(t8, ta); __m128i sa = _mm_unpacklo_epi16(t9, tb); __m128i sb = _mm_unpackhi_epi16(t9, tb); __m128i sc = _mm_unpacklo_epi16(tc, te); __m128i sd = _mm_unpackhi_epi16(tc, te); __m128i se = _mm_unpacklo_epi16(td, tf); __m128i sf = _mm_unpackhi_epi16(td, tf); __m128i u0 = _mm_unpacklo_epi32(s0, s4); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71 __m128i u1 = _mm_unpackhi_epi32(s0, s4); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73 __m128i u2 = _mm_unpacklo_epi32(s1, s5); __m128i u3 = _mm_unpackhi_epi32(s1, s5); __m128i u4 = _mm_unpacklo_epi32(s2, s6); __m128i u5 = _mm_unpackhi_epi32(s2, s6); __m128i u6 = _mm_unpacklo_epi32(s3, s7); __m128i u7 = _mm_unpackhi_epi32(s3, s7); __m128i u8 = _mm_unpacklo_epi32(s8, sc); __m128i u9 = _mm_unpackhi_epi32(s8, sc); __m128i ua = _mm_unpacklo_epi32(s9, sd); __m128i ub = _mm_unpackhi_epi32(s9, sd); __m128i uc = _mm_unpacklo_epi32(sa, se); __m128i ud = _mm_unpackhi_epi32(sa, se); __m128i ue = _mm_unpacklo_epi32(sb, sf); __m128i uf = _mm_unpackhi_epi32(sb, sf); kernel.packet[0] = _mm_unpacklo_epi64(u0, u8); kernel.packet[1] = _mm_unpackhi_epi64(u0, u8); kernel.packet[2] = _mm_unpacklo_epi64(u1, u9); kernel.packet[3] = _mm_unpackhi_epi64(u1, u9); kernel.packet[4] = _mm_unpacklo_epi64(u2, ua); kernel.packet[5] = _mm_unpackhi_epi64(u2, ua); kernel.packet[6] = _mm_unpacklo_epi64(u3, ub); kernel.packet[7] = _mm_unpackhi_epi64(u3, ub); kernel.packet[8] = _mm_unpacklo_epi64(u4, uc); kernel.packet[9] = _mm_unpackhi_epi64(u4, uc); kernel.packet[10] = _mm_unpacklo_epi64(u5, ud); kernel.packet[11] = _mm_unpackhi_epi64(u5, ud); kernel.packet[12] = _mm_unpacklo_epi64(u6, ue); kernel.packet[13] = _mm_unpackhi_epi64(u6, ue); kernel.packet[14] = _mm_unpacklo_epi64(u7, uf); kernel.packet[15] = _mm_unpackhi_epi64(u7, uf); } template<> EIGEN_STRONG_INLINE Packet4i pblend(const Selector<4>& ifPacket, const Packet4i& thenPacket, const Packet4i& elsePacket) { const __m128i zero = _mm_setzero_si128(); const __m128i select = _mm_set_epi32(ifPacket.select[3], ifPacket.select[2], ifPacket.select[1], ifPacket.select[0]); __m128i false_mask = _mm_cmpeq_epi32(select, zero); #ifdef EIGEN_VECTORIZE_SSE4_1 return _mm_blendv_epi8(thenPacket, elsePacket, false_mask); #else return _mm_or_si128(_mm_andnot_si128(false_mask, thenPacket), _mm_and_si128(false_mask, elsePacket)); #endif } template<> EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket, const Packet4f& elsePacket) { const __m128 zero = _mm_setzero_ps(); const __m128 select = _mm_set_ps(ifPacket.select[3], ifPacket.select[2], ifPacket.select[1], ifPacket.select[0]); __m128 false_mask = _mm_cmpeq_ps(select, zero); #ifdef EIGEN_VECTORIZE_SSE4_1 return _mm_blendv_ps(thenPacket, elsePacket, false_mask); #else return _mm_or_ps(_mm_andnot_ps(false_mask, thenPacket), _mm_and_ps(false_mask, elsePacket)); #endif } template<> EIGEN_STRONG_INLINE Packet2d pblend(const Selector<2>& ifPacket, const Packet2d& thenPacket, const Packet2d& elsePacket) { const __m128d zero = _mm_setzero_pd(); const __m128d select = _mm_set_pd(ifPacket.select[1], ifPacket.select[0]); __m128d false_mask = _mm_cmpeq_pd(select, zero); #ifdef EIGEN_VECTORIZE_SSE4_1 return _mm_blendv_pd(thenPacket, elsePacket, false_mask); #else return _mm_or_pd(_mm_andnot_pd(false_mask, thenPacket), _mm_and_pd(false_mask, elsePacket)); #endif } // Scalar path for pmadd with FMA to ensure consistency with vectorized path. #ifdef EIGEN_VECTORIZE_FMA template<> EIGEN_STRONG_INLINE float pmadd(const float& a, const float& b, const float& c) { return ::fmaf(a,b,c); } template<> EIGEN_STRONG_INLINE double pmadd(const double& a, const double& b, const double& c) { return ::fma(a,b,c); } #endif // Packet math for Eigen::half // Disable the following code since it's broken on too many platforms / compilers. //#elif defined(EIGEN_VECTORIZE_SSE) && (!EIGEN_ARCH_x86_64) && (!EIGEN_COMP_MSVC) #if 0 typedef struct { __m64 x; } Packet4h; template<> struct is_arithmetic { enum { value = true }; }; template <> struct packet_traits : default_packet_traits { typedef Packet4h type; // There is no half-size packet for Packet4h. typedef Packet4h half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 4, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 0, HasAbs = 0, HasAbs2 = 0, HasMin = 0, HasMax = 0, HasConj = 0, HasSetLinear = 0, HasSqrt = 0, HasRsqrt = 0, HasExp = 0, HasLog = 0, HasBlend = 0 }; }; template<> struct unpacket_traits { typedef Eigen::half type; enum {size=4, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet4h half; }; template<> EIGEN_STRONG_INLINE Packet4h pset1(const Eigen::half& from) { Packet4h result; result.x = _mm_set1_pi16(from.x); return result; } template<> EIGEN_STRONG_INLINE Eigen::half pfirst(const Packet4h& from) { return half_impl::raw_uint16_to_half(static_cast(_mm_cvtsi64_si32(from.x))); } template<> EIGEN_STRONG_INLINE Packet4h pconj(const Packet4h& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4h padd(const Packet4h& a, const Packet4h& b) { __int64_t a64 = _mm_cvtm64_si64(a.x); __int64_t b64 = _mm_cvtm64_si64(b.x); Eigen::half h[4]; Eigen::half ha = half_impl::raw_uint16_to_half(static_cast(a64)); Eigen::half hb = half_impl::raw_uint16_to_half(static_cast(b64)); h[0] = ha + hb; ha = half_impl::raw_uint16_to_half(static_cast(a64 >> 16)); hb = half_impl::raw_uint16_to_half(static_cast(b64 >> 16)); h[1] = ha + hb; ha = half_impl::raw_uint16_to_half(static_cast(a64 >> 32)); hb = half_impl::raw_uint16_to_half(static_cast(b64 >> 32)); h[2] = ha + hb; ha = half_impl::raw_uint16_to_half(static_cast(a64 >> 48)); hb = half_impl::raw_uint16_to_half(static_cast(b64 >> 48)); h[3] = ha + hb; Packet4h result; result.x = _mm_set_pi16(h[3].x, h[2].x, h[1].x, h[0].x); return result; } template<> EIGEN_STRONG_INLINE Packet4h psub(const Packet4h& a, const Packet4h& b) { __int64_t a64 = _mm_cvtm64_si64(a.x); __int64_t b64 = _mm_cvtm64_si64(b.x); Eigen::half h[4]; Eigen::half ha = half_impl::raw_uint16_to_half(static_cast(a64)); Eigen::half hb = half_impl::raw_uint16_to_half(static_cast(b64)); h[0] = ha - hb; ha = half_impl::raw_uint16_to_half(static_cast(a64 >> 16)); hb = half_impl::raw_uint16_to_half(static_cast(b64 >> 16)); h[1] = ha - hb; ha = half_impl::raw_uint16_to_half(static_cast(a64 >> 32)); hb = half_impl::raw_uint16_to_half(static_cast(b64 >> 32)); h[2] = ha - hb; ha = half_impl::raw_uint16_to_half(static_cast(a64 >> 48)); hb = half_impl::raw_uint16_to_half(static_cast(b64 >> 48)); h[3] = ha - hb; Packet4h result; result.x = _mm_set_pi16(h[3].x, h[2].x, h[1].x, h[0].x); return result; } template<> EIGEN_STRONG_INLINE Packet4h pmul(const Packet4h& a, const Packet4h& b) { __int64_t a64 = _mm_cvtm64_si64(a.x); __int64_t b64 = _mm_cvtm64_si64(b.x); Eigen::half h[4]; Eigen::half ha = half_impl::raw_uint16_to_half(static_cast(a64)); Eigen::half hb = half_impl::raw_uint16_to_half(static_cast(b64)); h[0] = ha * hb; ha = half_impl::raw_uint16_to_half(static_cast(a64 >> 16)); hb = half_impl::raw_uint16_to_half(static_cast(b64 >> 16)); h[1] = ha * hb; ha = half_impl::raw_uint16_to_half(static_cast(a64 >> 32)); hb = half_impl::raw_uint16_to_half(static_cast(b64 >> 32)); h[2] = ha * hb; ha = half_impl::raw_uint16_to_half(static_cast(a64 >> 48)); hb = half_impl::raw_uint16_to_half(static_cast(b64 >> 48)); h[3] = ha * hb; Packet4h result; result.x = _mm_set_pi16(h[3].x, h[2].x, h[1].x, h[0].x); return result; } template<> EIGEN_STRONG_INLINE Packet4h pdiv(const Packet4h& a, const Packet4h& b) { __int64_t a64 = _mm_cvtm64_si64(a.x); __int64_t b64 = _mm_cvtm64_si64(b.x); Eigen::half h[4]; Eigen::half ha = half_impl::raw_uint16_to_half(static_cast(a64)); Eigen::half hb = half_impl::raw_uint16_to_half(static_cast(b64)); h[0] = ha / hb; ha = half_impl::raw_uint16_to_half(static_cast(a64 >> 16)); hb = half_impl::raw_uint16_to_half(static_cast(b64 >> 16)); h[1] = ha / hb; ha = half_impl::raw_uint16_to_half(static_cast(a64 >> 32)); hb = half_impl::raw_uint16_to_half(static_cast(b64 >> 32)); h[2] = ha / hb; ha = half_impl::raw_uint16_to_half(static_cast(a64 >> 48)); hb = half_impl::raw_uint16_to_half(static_cast(b64 >> 48)); h[3] = ha / hb; Packet4h result; result.x = _mm_set_pi16(h[3].x, h[2].x, h[1].x, h[0].x); return result; } template<> EIGEN_STRONG_INLINE Packet4h pload(const Eigen::half* from) { Packet4h result; result.x = _mm_cvtsi64_m64(*reinterpret_cast(from)); return result; } template<> EIGEN_STRONG_INLINE Packet4h ploadu(const Eigen::half* from) { Packet4h result; result.x = _mm_cvtsi64_m64(*reinterpret_cast(from)); return result; } template<> EIGEN_STRONG_INLINE void pstore(Eigen::half* to, const Packet4h& from) { __int64_t r = _mm_cvtm64_si64(from.x); *(reinterpret_cast<__int64_t*>(to)) = r; } template<> EIGEN_STRONG_INLINE void pstoreu(Eigen::half* to, const Packet4h& from) { __int64_t r = _mm_cvtm64_si64(from.x); *(reinterpret_cast<__int64_t*>(to)) = r; } template<> EIGEN_STRONG_INLINE Packet4h ploadquad(const Eigen::half* from) { return pset1(*from); } template<> EIGEN_STRONG_INLINE Packet4h pgather(const Eigen::half* from, Index stride) { Packet4h result; result.x = _mm_set_pi16(from[3*stride].x, from[2*stride].x, from[1*stride].x, from[0*stride].x); return result; } template<> EIGEN_STRONG_INLINE void pscatter(Eigen::half* to, const Packet4h& from, Index stride) { __int64_t a = _mm_cvtm64_si64(from.x); to[stride*0].x = static_cast(a); to[stride*1].x = static_cast(a >> 16); to[stride*2].x = static_cast(a >> 32); to[stride*3].x = static_cast(a >> 48); } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { __m64 T0 = _mm_unpacklo_pi16(kernel.packet[0].x, kernel.packet[1].x); __m64 T1 = _mm_unpacklo_pi16(kernel.packet[2].x, kernel.packet[3].x); __m64 T2 = _mm_unpackhi_pi16(kernel.packet[0].x, kernel.packet[1].x); __m64 T3 = _mm_unpackhi_pi16(kernel.packet[2].x, kernel.packet[3].x); kernel.packet[0].x = _mm_unpacklo_pi32(T0, T1); kernel.packet[1].x = _mm_unpackhi_pi32(T0, T1); kernel.packet[2].x = _mm_unpacklo_pi32(T2, T3); kernel.packet[3].x = _mm_unpackhi_pi32(T2, T3); } #endif } // end namespace internal } // end namespace Eigen #if EIGEN_COMP_PGI && EIGEN_COMP_PGI < 1900 // PGI++ does not define the following intrinsics in C++ mode. static inline __m128 _mm_castpd_ps (__m128d x) { return reinterpret_cast<__m128&>(x); } static inline __m128i _mm_castpd_si128(__m128d x) { return reinterpret_cast<__m128i&>(x); } static inline __m128d _mm_castps_pd (__m128 x) { return reinterpret_cast<__m128d&>(x); } static inline __m128i _mm_castps_si128(__m128 x) { return reinterpret_cast<__m128i&>(x); } static inline __m128 _mm_castsi128_ps(__m128i x) { return reinterpret_cast<__m128&>(x); } static inline __m128d _mm_castsi128_pd(__m128i x) { return reinterpret_cast<__m128d&>(x); } #endif #endif // EIGEN_PACKET_MATH_SSE_H RcppEigen/inst/include/Eigen/src/Core/arch/SSE/MathFunctions.h0000644000176200001440000001515514567757725023576 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2007 Julien Pommier // Copyright (C) 2009 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. /* The sin and cos and functions of this file come from * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/ */ #ifndef EIGEN_MATH_FUNCTIONS_SSE_H #define EIGEN_MATH_FUNCTIONS_SSE_H namespace Eigen { namespace internal { template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f plog(const Packet4f& _x) { return plog_float(_x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d plog(const Packet2d& _x) { return plog_double(_x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f plog2(const Packet4f& _x) { return plog2_float(_x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d plog2(const Packet2d& _x) { return plog2_double(_x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f plog1p(const Packet4f& _x) { return generic_plog1p(_x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pexpm1(const Packet4f& _x) { return generic_expm1(_x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pexp(const Packet4f& _x) { return pexp_float(_x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d pexp(const Packet2d& x) { return pexp_double(x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f psin(const Packet4f& _x) { return psin_float(_x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pcos(const Packet4f& _x) { return pcos_float(_x); } #if EIGEN_FAST_MATH // Functions for sqrt. // The EIGEN_FAST_MATH version uses the _mm_rsqrt_ps approximation and one step // of Newton's method, at a cost of 1-2 bits of precision as opposed to the // exact solution. It does not handle +inf, or denormalized numbers correctly. // The main advantage of this approach is not just speed, but also the fact that // it can be inlined and pipelined with other computations, further reducing its // effective latency. This is similar to Quake3's fast inverse square root. // For detail see here: http://www.beyond3d.com/content/articles/8/ template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f psqrt(const Packet4f& _x) { Packet4f minus_half_x = pmul(_x, pset1(-0.5f)); Packet4f denormal_mask = pandnot( pcmp_lt(_x, pset1((std::numeric_limits::min)())), pcmp_lt(_x, pzero(_x))); // Compute approximate reciprocal sqrt. Packet4f x = _mm_rsqrt_ps(_x); // Do a single step of Newton's iteration. x = pmul(x, pmadd(minus_half_x, pmul(x,x), pset1(1.5f))); // Flush results for denormals to zero. return pandnot(pmul(_x,x), denormal_mask); } #else template<>EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f psqrt(const Packet4f& x) { return _mm_sqrt_ps(x); } #endif template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d psqrt(const Packet2d& x) { return _mm_sqrt_pd(x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16b psqrt(const Packet16b& x) { return x; } #if EIGEN_FAST_MATH template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f prsqrt(const Packet4f& _x) { _EIGEN_DECLARE_CONST_Packet4f(one_point_five, 1.5f); _EIGEN_DECLARE_CONST_Packet4f(minus_half, -0.5f); _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(inf, 0x7f800000u); _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(flt_min, 0x00800000u); Packet4f neg_half = pmul(_x, p4f_minus_half); // Identity infinite, zero, negative and denormal arguments. Packet4f lt_min_mask = _mm_cmplt_ps(_x, p4f_flt_min); Packet4f inf_mask = _mm_cmpeq_ps(_x, p4f_inf); Packet4f not_normal_finite_mask = _mm_or_ps(lt_min_mask, inf_mask); // Compute an approximate result using the rsqrt intrinsic. Packet4f y_approx = _mm_rsqrt_ps(_x); // Do a single step of Newton-Raphson iteration to improve the approximation. // This uses the formula y_{n+1} = y_n * (1.5 - y_n * (0.5 * x) * y_n). // It is essential to evaluate the inner term like this because forming // y_n^2 may over- or underflow. Packet4f y_newton = pmul( y_approx, pmadd(y_approx, pmul(neg_half, y_approx), p4f_one_point_five)); // Select the result of the Newton-Raphson step for positive normal arguments. // For other arguments, choose the output of the intrinsic. This will // return rsqrt(+inf) = 0, rsqrt(x) = NaN if x < 0, and rsqrt(x) = +inf if // x is zero or a positive denormalized float (equivalent to flushing positive // denormalized inputs to zero). return pselect(not_normal_finite_mask, y_approx, y_newton); } #else template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f prsqrt(const Packet4f& x) { // Unfortunately we can't use the much faster mm_rsqrt_ps since it only provides an approximation. return _mm_div_ps(pset1(1.0f), _mm_sqrt_ps(x)); } #endif template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d prsqrt(const Packet2d& x) { return _mm_div_pd(pset1(1.0), _mm_sqrt_pd(x)); } // Hyperbolic Tangent function. template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f ptanh(const Packet4f& x) { return internal::generic_fast_tanh_float(x); } } // end namespace internal namespace numext { template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float sqrt(const float &x) { return internal::pfirst(internal::Packet4f(_mm_sqrt_ss(_mm_set_ss(x)))); } template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double sqrt(const double &x) { #if EIGEN_COMP_GNUC_STRICT // This works around a GCC bug generating poor code for _mm_sqrt_pd // See https://gitlab.com/libeigen/eigen/commit/8dca9f97e38970 return internal::pfirst(internal::Packet2d(__builtin_ia32_sqrtsd(_mm_set_sd(x)))); #else return internal::pfirst(internal::Packet2d(_mm_sqrt_pd(_mm_set_sd(x)))); #endif } } // end namespace numex } // end namespace Eigen #endif // EIGEN_MATH_FUNCTIONS_SSE_H RcppEigen/inst/include/Eigen/src/Core/arch/Default/0000755000176200001440000000000014567757725021566 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/arch/Default/Settings.h0000644000176200001440000000332214567757725023537 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2010 Gael Guennebaud // Copyright (C) 2006-2008 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. /* All the parameters defined in this file can be specialized in the * architecture specific files, and/or by the user. * More to come... */ #ifndef EIGEN_DEFAULT_SETTINGS_H #define EIGEN_DEFAULT_SETTINGS_H /** Defines the maximal loop size to enable meta unrolling of loops. * Note that the value here is expressed in Eigen's own notion of "number of FLOPS", * it does not correspond to the number of iterations or the number of instructions */ #ifndef EIGEN_UNROLLING_LIMIT #define EIGEN_UNROLLING_LIMIT 110 #endif /** Defines the threshold between a "small" and a "large" matrix. * This threshold is mainly used to select the proper product implementation. */ #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 #endif /** Defines the maximal width of the blocks used in the triangular product and solver * for vectors (level 2 blas xTRMV and xTRSV). The default is 8. */ #ifndef EIGEN_TUNE_TRIANGULAR_PANEL_WIDTH #define EIGEN_TUNE_TRIANGULAR_PANEL_WIDTH 8 #endif /** Defines the default number of registers available for that architecture. * Currently it must be 8 or 16. Other values will fail. */ #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 8 #endif #endif // EIGEN_DEFAULT_SETTINGS_H RcppEigen/inst/include/Eigen/src/Core/arch/Default/TypeCasting.h0000644000176200001440000000724214567757725024176 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2016 Benoit Steiner // Copyright (C) 2019 Rasmus Munk Larsen // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_GENERIC_TYPE_CASTING_H #define EIGEN_GENERIC_TYPE_CASTING_H namespace Eigen { namespace internal { template<> struct scalar_cast_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) typedef Eigen::half result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half operator() (const float& a) const { #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) return __float2half(a); #else return Eigen::half(a); #endif } }; template<> struct functor_traits > { enum { Cost = NumTraits::AddCost, PacketAccess = false }; }; template<> struct scalar_cast_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) typedef Eigen::half result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half operator() (const int& a) const { #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) return __float2half(static_cast(a)); #else return Eigen::half(static_cast(a)); #endif } }; template<> struct functor_traits > { enum { Cost = NumTraits::AddCost, PacketAccess = false }; }; template<> struct scalar_cast_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) typedef float result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float operator() (const Eigen::half& a) const { #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) return __half2float(a); #else return static_cast(a); #endif } }; template<> struct functor_traits > { enum { Cost = NumTraits::AddCost, PacketAccess = false }; }; template<> struct scalar_cast_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) typedef Eigen::bfloat16 result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::bfloat16 operator() (const float& a) const { return Eigen::bfloat16(a); } }; template<> struct functor_traits > { enum { Cost = NumTraits::AddCost, PacketAccess = false }; }; template<> struct scalar_cast_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) typedef Eigen::bfloat16 result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::bfloat16 operator() (const int& a) const { return Eigen::bfloat16(static_cast(a)); } }; template<> struct functor_traits > { enum { Cost = NumTraits::AddCost, PacketAccess = false }; }; template<> struct scalar_cast_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) typedef float result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float operator() (const Eigen::bfloat16& a) const { return static_cast(a); } }; template<> struct functor_traits > { enum { Cost = NumTraits::AddCost, PacketAccess = false }; }; } } #endif // EIGEN_GENERIC_TYPE_CASTING_H RcppEigen/inst/include/Eigen/src/Core/arch/Default/ConjHelper.h0000644000176200001440000001220314567757725023766 0ustar liggesusers // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2017 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_ARCH_CONJ_HELPER_H #define EIGEN_ARCH_CONJ_HELPER_H #define EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(PACKET_CPLX, PACKET_REAL) \ template <> \ struct conj_helper { \ EIGEN_STRONG_INLINE PACKET_CPLX pmadd(const PACKET_REAL& x, \ const PACKET_CPLX& y, \ const PACKET_CPLX& c) const { \ return padd(c, this->pmul(x, y)); \ } \ EIGEN_STRONG_INLINE PACKET_CPLX pmul(const PACKET_REAL& x, \ const PACKET_CPLX& y) const { \ return PACKET_CPLX(Eigen::internal::pmul(x, y.v)); \ } \ }; \ \ template <> \ struct conj_helper { \ EIGEN_STRONG_INLINE PACKET_CPLX pmadd(const PACKET_CPLX& x, \ const PACKET_REAL& y, \ const PACKET_CPLX& c) const { \ return padd(c, this->pmul(x, y)); \ } \ EIGEN_STRONG_INLINE PACKET_CPLX pmul(const PACKET_CPLX& x, \ const PACKET_REAL& y) const { \ return PACKET_CPLX(Eigen::internal::pmul(x.v, y)); \ } \ }; namespace Eigen { namespace internal { template struct conj_if; template<> struct conj_if { template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T operator()(const T& x) const { return numext::conj(x); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T pconj(const T& x) const { return internal::pconj(x); } }; template<> struct conj_if { template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& operator()(const T& x) const { return x; } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& pconj(const T& x) const { return x; } }; // Generic Implementation, assume scalars since the packet-version is // specialized below. template struct conj_helper { typedef typename ScalarBinaryOpTraits::ReturnType ResultType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ResultType pmadd(const LhsType& x, const RhsType& y, const ResultType& c) const { return this->pmul(x, y) + c; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ResultType pmul(const LhsType& x, const RhsType& y) const { return conj_if()(x) * conj_if()(y); } }; template struct conj_helper { typedef typename ScalarBinaryOpTraits::ReturnType ResultType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ResultType pmadd(const LhsScalar& x, const RhsScalar& y, const ResultType& c) const { return this->pmul(x, y) + c; } // We save a conjuation by using the identity conj(a)*conj(b) = conj(a*b). EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ResultType pmul(const LhsScalar& x, const RhsScalar& y) const { return numext::conj(x * y); } }; // Implementation with equal type, use packet operations. template struct conj_helper { typedef Packet ResultType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pmadd(const Packet& x, const Packet& y, const Packet& c) const { return Eigen::internal::pmadd(conj_if().pconj(x), conj_if().pconj(y), c); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pmul(const Packet& x, const Packet& y) const { return Eigen::internal::pmul(conj_if().pconj(x), conj_if().pconj(y)); } }; template struct conj_helper { typedef Packet ResultType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pmadd(const Packet& x, const Packet& y, const Packet& c) const { return Eigen::internal::pmadd(pconj(x), pconj(y), c); } // We save a conjuation by using the identity conj(a)*conj(b) = conj(a*b). EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pmul(const Packet& x, const Packet& y) const { return pconj(Eigen::internal::pmul(x, y)); } }; } // namespace internal } // namespace Eigen #endif // EIGEN_ARCH_CONJ_HELPER_H RcppEigen/inst/include/Eigen/src/Core/arch/Default/Half.h0000644000176200001440000010531614567757725022617 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. // // The conversion routines are Copyright (c) Fabian Giesen, 2016. // The original license follows: // // Copyright (c) Fabian Giesen, 2016 // All rights reserved. // Redistribution and use in source and binary forms, with or without // modification, are permitted. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Standard 16-bit float type, mostly useful for GPUs. Defines a new // type Eigen::half (inheriting either from CUDA's or HIP's __half struct) with // operator overloads such that it behaves basically as an arithmetic // type. It will be quite slow on CPUs (so it is recommended to stay // in fp32 for CPUs, except for simple parameter conversions, I/O // to disk and the likes), but fast on GPUs. #ifndef EIGEN_HALF_H #define EIGEN_HALF_H #include #if defined(EIGEN_HAS_GPU_FP16) || defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) // When compiling with GPU support, the "__half_raw" base class as well as // some other routines are defined in the GPU compiler header files // (cuda_fp16.h, hip_fp16.h), and they are not tagged constexpr // As a consequence, we get compile failures when compiling Eigen with // GPU support. Hence the need to disable EIGEN_CONSTEXPR when building // Eigen with GPU support #pragma push_macro("EIGEN_CONSTEXPR") #undef EIGEN_CONSTEXPR #define EIGEN_CONSTEXPR #endif #define F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, METHOD) \ template <> \ EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_UNUSED \ PACKET_F16 METHOD(const PACKET_F16& _x) { \ return float2half(METHOD(half2float(_x))); \ } namespace Eigen { struct half; namespace half_impl { // We want to use the __half_raw struct from the HIP header file only during the device compile phase. // This is required because of a quirk in the way TensorFlow GPU builds are done. // When compiling TensorFlow source code with GPU support, files that // * contain GPU kernels (i.e. *.cu.cc files) are compiled via hipcc // * do not contain GPU kernels ( i.e. *.cc files) are compiled via gcc (typically) // // Tensorflow uses the Eigen::half type as its FP16 type, and there are functions that // * are defined in a file that gets compiled via hipcc AND // * have Eigen::half as a pass-by-value argument AND // * are called in a file that gets compiled via gcc // // In the scenario described above the caller and callee will see different versions // of the Eigen::half base class __half_raw, and they will be compiled by different compilers // // There appears to be an ABI mismatch between gcc and clang (which is called by hipcc) that results in // the callee getting corrupted values for the Eigen::half argument. // // Making the host side compile phase of hipcc use the same Eigen::half impl, as the gcc compile, resolves // this error, and hence the following convoluted #if condition #if !defined(EIGEN_HAS_GPU_FP16) || !defined(EIGEN_GPU_COMPILE_PHASE) // Make our own __half_raw definition that is similar to CUDA's. struct __half_raw { #if (defined(EIGEN_HAS_GPU_FP16) && !defined(EIGEN_GPU_COMPILE_PHASE)) // Eigen::half can be used as the datatype for shared memory declarations (in Eigen and TF) // The element type for shared memory cannot have non-trivial constructors // and hence the following special casing (which skips the zero-initilization). // Note that this check gets done even in the host compilation phase, and // hence the need for this EIGEN_DEVICE_FUNC __half_raw() {} #else EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __half_raw() : x(0) {} #endif #if defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __half_raw(numext::uint16_t raw) : x(numext::bit_cast<__fp16>(raw)) { } __fp16 x; #else explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __half_raw(numext::uint16_t raw) : x(raw) {} numext::uint16_t x; #endif }; #elif defined(EIGEN_HAS_HIP_FP16) // Nothing to do here // HIP fp16 header file has a definition for __half_raw #elif defined(EIGEN_HAS_CUDA_FP16) #if EIGEN_CUDA_SDK_VER < 90000 // In CUDA < 9.0, __half is the equivalent of CUDA 9's __half_raw typedef __half __half_raw; #endif // defined(EIGEN_HAS_CUDA_FP16) #elif defined(SYCL_DEVICE_ONLY) typedef cl::sycl::half __half_raw; #endif EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __half_raw raw_uint16_to_half(numext::uint16_t x); EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half_raw float_to_half_rtne(float ff); EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC float half_to_float(__half_raw h); struct half_base : public __half_raw { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR half_base() {} EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR half_base(const __half_raw& h) : __half_raw(h) {} #if defined(EIGEN_HAS_GPU_FP16) #if defined(EIGEN_HAS_HIP_FP16) EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR half_base(const __half& h) { x = __half_as_ushort(h); } #elif defined(EIGEN_HAS_CUDA_FP16) #if EIGEN_CUDA_SDK_VER >= 90000 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR half_base(const __half& h) : __half_raw(*(__half_raw*)&h) {} #endif #endif #endif }; } // namespace half_impl // Class definition. struct half : public half_impl::half_base { // Writing this out as separate #if-else blocks to make the code easier to follow // The same applies to most #if-else blocks in this file #if !defined(EIGEN_HAS_GPU_FP16) || !defined(EIGEN_GPU_COMPILE_PHASE) // Use the same base class for the following two scenarios // * when compiling without GPU support enabled // * during host compile phase when compiling with GPU support enabled typedef half_impl::__half_raw __half_raw; #elif defined(EIGEN_HAS_HIP_FP16) // Nothing to do here // HIP fp16 header file has a definition for __half_raw #elif defined(EIGEN_HAS_CUDA_FP16) // Note that EIGEN_CUDA_SDK_VER is set to 0 even when compiling with HIP, so // (EIGEN_CUDA_SDK_VER < 90000) is true even for HIP! So keeping this within // #if defined(EIGEN_HAS_CUDA_FP16) is needed #if defined(EIGEN_CUDA_SDK_VER) && EIGEN_CUDA_SDK_VER < 90000 typedef half_impl::__half_raw __half_raw; #endif #endif EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR half() {} EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR half(const __half_raw& h) : half_impl::half_base(h) {} #if defined(EIGEN_HAS_GPU_FP16) #if defined(EIGEN_HAS_HIP_FP16) EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR half(const __half& h) : half_impl::half_base(h) {} #elif defined(EIGEN_HAS_CUDA_FP16) #if defined(EIGEN_CUDA_SDK_VER) && EIGEN_CUDA_SDK_VER >= 90000 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR half(const __half& h) : half_impl::half_base(h) {} #endif #endif #endif explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR half(bool b) : half_impl::half_base(half_impl::raw_uint16_to_half(b ? 0x3c00 : 0)) {} template explicit EIGEN_DEVICE_FUNC half(T val) : half_impl::half_base(half_impl::float_to_half_rtne(static_cast(val))) {} explicit EIGEN_DEVICE_FUNC half(float f) : half_impl::half_base(half_impl::float_to_half_rtne(f)) {} // Following the convention of numpy, converting between complex and // float will lead to loss of imag value. template explicit EIGEN_DEVICE_FUNC half(std::complex c) : half_impl::half_base(half_impl::float_to_half_rtne(static_cast(c.real()))) {} EIGEN_DEVICE_FUNC operator float() const { // NOLINT: Allow implicit conversion to float, because it is lossless. return half_impl::half_to_float(*this); } #if defined(EIGEN_HAS_GPU_FP16) && !defined(EIGEN_GPU_COMPILE_PHASE) EIGEN_DEVICE_FUNC operator __half() const { ::__half_raw hr; hr.x = x; return __half(hr); } #endif }; } // end namespace Eigen namespace std { template<> struct numeric_limits { static const bool is_specialized = true; static const bool is_signed = true; static const bool is_integer = false; static const bool is_exact = false; static const bool has_infinity = true; static const bool has_quiet_NaN = true; static const bool has_signaling_NaN = true; static const float_denorm_style has_denorm = denorm_present; static const bool has_denorm_loss = false; static const std::float_round_style round_style = std::round_to_nearest; static const bool is_iec559 = false; static const bool is_bounded = false; static const bool is_modulo = false; static const int digits = 11; static const int digits10 = 3; // according to http://half.sourceforge.net/structstd_1_1numeric__limits_3_01half__float_1_1half_01_4.html static const int max_digits10 = 5; // according to http://half.sourceforge.net/structstd_1_1numeric__limits_3_01half__float_1_1half_01_4.html static const int radix = 2; static const int min_exponent = -13; static const int min_exponent10 = -4; static const int max_exponent = 16; static const int max_exponent10 = 4; static const bool traps = true; static const bool tinyness_before = false; static Eigen::half (min)() { return Eigen::half_impl::raw_uint16_to_half(0x400); } static Eigen::half lowest() { return Eigen::half_impl::raw_uint16_to_half(0xfbff); } static Eigen::half (max)() { return Eigen::half_impl::raw_uint16_to_half(0x7bff); } static Eigen::half epsilon() { return Eigen::half_impl::raw_uint16_to_half(0x0800); } static Eigen::half round_error() { return Eigen::half(0.5); } static Eigen::half infinity() { return Eigen::half_impl::raw_uint16_to_half(0x7c00); } static Eigen::half quiet_NaN() { return Eigen::half_impl::raw_uint16_to_half(0x7e00); } static Eigen::half signaling_NaN() { return Eigen::half_impl::raw_uint16_to_half(0x7d00); } static Eigen::half denorm_min() { return Eigen::half_impl::raw_uint16_to_half(0x1); } }; // If std::numeric_limits is specialized, should also specialize // std::numeric_limits, std::numeric_limits, and // std::numeric_limits // https://stackoverflow.com/a/16519653/ template<> struct numeric_limits : numeric_limits {}; template<> struct numeric_limits : numeric_limits {}; template<> struct numeric_limits : numeric_limits {}; } // end namespace std namespace Eigen { namespace half_impl { #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && \ EIGEN_CUDA_ARCH >= 530) || \ (defined(EIGEN_HAS_HIP_FP16) && defined(HIP_DEVICE_COMPILE)) // Note: We deliberatly do *not* define this to 1 even if we have Arm's native // fp16 type since GPU halfs are rather different from native CPU halfs. // TODO: Rename to something like EIGEN_HAS_NATIVE_GPU_FP16 #define EIGEN_HAS_NATIVE_FP16 #endif // Intrinsics for native fp16 support. Note that on current hardware, // these are no faster than fp32 arithmetic (you need to use the half2 // versions to get the ALU speed increased), but you do save the // conversion steps back and forth. #if defined(EIGEN_HAS_NATIVE_FP16) EIGEN_STRONG_INLINE __device__ half operator + (const half& a, const half& b) { #if defined(EIGEN_CUDA_SDK_VER) && EIGEN_CUDA_SDK_VER >= 90000 return __hadd(::__half(a), ::__half(b)); #else return __hadd(a, b); #endif } EIGEN_STRONG_INLINE __device__ half operator * (const half& a, const half& b) { return __hmul(a, b); } EIGEN_STRONG_INLINE __device__ half operator - (const half& a, const half& b) { return __hsub(a, b); } EIGEN_STRONG_INLINE __device__ half operator / (const half& a, const half& b) { #if defined(EIGEN_CUDA_SDK_VER) && EIGEN_CUDA_SDK_VER >= 90000 return __hdiv(a, b); #else float num = __half2float(a); float denom = __half2float(b); return __float2half(num / denom); #endif } EIGEN_STRONG_INLINE __device__ half operator - (const half& a) { return __hneg(a); } EIGEN_STRONG_INLINE __device__ half& operator += (half& a, const half& b) { a = a + b; return a; } EIGEN_STRONG_INLINE __device__ half& operator *= (half& a, const half& b) { a = a * b; return a; } EIGEN_STRONG_INLINE __device__ half& operator -= (half& a, const half& b) { a = a - b; return a; } EIGEN_STRONG_INLINE __device__ half& operator /= (half& a, const half& b) { a = a / b; return a; } EIGEN_STRONG_INLINE __device__ bool operator == (const half& a, const half& b) { return __heq(a, b); } EIGEN_STRONG_INLINE __device__ bool operator != (const half& a, const half& b) { return __hne(a, b); } EIGEN_STRONG_INLINE __device__ bool operator < (const half& a, const half& b) { return __hlt(a, b); } EIGEN_STRONG_INLINE __device__ bool operator <= (const half& a, const half& b) { return __hle(a, b); } EIGEN_STRONG_INLINE __device__ bool operator > (const half& a, const half& b) { return __hgt(a, b); } EIGEN_STRONG_INLINE __device__ bool operator >= (const half& a, const half& b) { return __hge(a, b); } #endif #if defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator + (const half& a, const half& b) { return half(vaddh_f16(a.x, b.x)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator * (const half& a, const half& b) { return half(vmulh_f16(a.x, b.x)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator - (const half& a, const half& b) { return half(vsubh_f16(a.x, b.x)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator / (const half& a, const half& b) { return half(vdivh_f16(a.x, b.x)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator - (const half& a) { return half(vnegh_f16(a.x)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator += (half& a, const half& b) { a = half(vaddh_f16(a.x, b.x)); return a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator *= (half& a, const half& b) { a = half(vmulh_f16(a.x, b.x)); return a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator -= (half& a, const half& b) { a = half(vsubh_f16(a.x, b.x)); return a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator /= (half& a, const half& b) { a = half(vdivh_f16(a.x, b.x)); return a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator == (const half& a, const half& b) { return vceqh_f16(a.x, b.x); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator != (const half& a, const half& b) { return !vceqh_f16(a.x, b.x); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator < (const half& a, const half& b) { return vclth_f16(a.x, b.x); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator <= (const half& a, const half& b) { return vcleh_f16(a.x, b.x); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator > (const half& a, const half& b) { return vcgth_f16(a.x, b.x); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator >= (const half& a, const half& b) { return vcgeh_f16(a.x, b.x); } // We need to distinguish ‘clang as the CUDA compiler’ from ‘clang as the host compiler, // invoked by NVCC’ (e.g. on MacOS). The former needs to see both host and device implementation // of the functions, while the latter can only deal with one of them. #elif !defined(EIGEN_HAS_NATIVE_FP16) || (EIGEN_COMP_CLANG && !EIGEN_COMP_NVCC) // Emulate support for half floats #if EIGEN_COMP_CLANG && defined(EIGEN_CUDACC) // We need to provide emulated *host-side* FP16 operators for clang. #pragma push_macro("EIGEN_DEVICE_FUNC") #undef EIGEN_DEVICE_FUNC #if defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_HAS_NATIVE_FP16) #define EIGEN_DEVICE_FUNC __host__ #else // both host and device need emulated ops. #define EIGEN_DEVICE_FUNC __host__ __device__ #endif #endif // Definitions for CPUs and older HIP+CUDA, mostly working through conversion // to/from fp32. EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator + (const half& a, const half& b) { return half(float(a) + float(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator * (const half& a, const half& b) { return half(float(a) * float(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator - (const half& a, const half& b) { return half(float(a) - float(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator / (const half& a, const half& b) { return half(float(a) / float(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator - (const half& a) { half result; result.x = a.x ^ 0x8000; return result; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator += (half& a, const half& b) { a = half(float(a) + float(b)); return a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator *= (half& a, const half& b) { a = half(float(a) * float(b)); return a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator -= (half& a, const half& b) { a = half(float(a) - float(b)); return a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator /= (half& a, const half& b) { a = half(float(a) / float(b)); return a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator == (const half& a, const half& b) { return numext::equal_strict(float(a),float(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator != (const half& a, const half& b) { return numext::not_equal_strict(float(a), float(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator < (const half& a, const half& b) { return float(a) < float(b); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator <= (const half& a, const half& b) { return float(a) <= float(b); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator > (const half& a, const half& b) { return float(a) > float(b); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator >= (const half& a, const half& b) { return float(a) >= float(b); } #if defined(__clang__) && defined(__CUDA__) #pragma pop_macro("EIGEN_DEVICE_FUNC") #endif #endif // Emulate support for half floats // Division by an index. Do it in full float precision to avoid accuracy // issues in converting the denominator to half. EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator / (const half& a, Index b) { return half(static_cast(a) / static_cast(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator++(half& a) { a += half(1); return a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator--(half& a) { a -= half(1); return a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator++(half& a, int) { half original_value = a; ++a; return original_value; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator--(half& a, int) { half original_value = a; --a; return original_value; } // Conversion routines, including fallbacks for the host or older CUDA. // Note that newer Intel CPUs (Haswell or newer) have vectorized versions of // these in hardware. If we need more performance on older/other CPUs, they are // also possible to vectorize directly. EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __half_raw raw_uint16_to_half(numext::uint16_t x) { // We cannot simply do a "return __half_raw(x)" here, because __half_raw is union type // in the hip_fp16 header file, and that will trigger a compile error // On the other hand, having anything but a return statement also triggers a compile error // because this is constexpr function. // Fortunately, since we need to disable EIGEN_CONSTEXPR for GPU anyway, we can get out // of this catch22 by having separate bodies for GPU / non GPU #if defined(EIGEN_HAS_GPU_FP16) __half_raw h; h.x = x; return h; #else return __half_raw(x); #endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC numext::uint16_t raw_half_as_uint16(const __half_raw& h) { // HIP/CUDA/Default have a member 'x' of type uint16_t. // For ARM64 native half, the member 'x' is of type __fp16, so we need to bit-cast. // For SYCL, cl::sycl::half is _Float16, so cast directly. #if defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) return numext::bit_cast(h.x); #elif defined(SYCL_DEVICE_ONLY) return numext::bit_cast(h); #else return h.x; #endif } union float32_bits { unsigned int u; float f; }; EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half_raw float_to_half_rtne(float ff) { #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) __half tmp_ff = __float2half(ff); return *(__half_raw*)&tmp_ff; #elif defined(EIGEN_HAS_FP16_C) __half_raw h; h.x = _cvtss_sh(ff, 0); return h; #elif defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) __half_raw h; h.x = static_cast<__fp16>(ff); return h; #else float32_bits f; f.f = ff; const float32_bits f32infty = { 255 << 23 }; const float32_bits f16max = { (127 + 16) << 23 }; const float32_bits denorm_magic = { ((127 - 15) + (23 - 10) + 1) << 23 }; unsigned int sign_mask = 0x80000000u; __half_raw o; o.x = static_cast(0x0u); unsigned int sign = f.u & sign_mask; f.u ^= sign; // NOTE all the integer compares in this function can be safely // compiled into signed compares since all operands are below // 0x80000000. Important if you want fast straight SSE2 code // (since there's no unsigned PCMPGTD). if (f.u >= f16max.u) { // result is Inf or NaN (all exponent bits set) o.x = (f.u > f32infty.u) ? 0x7e00 : 0x7c00; // NaN->qNaN and Inf->Inf } else { // (De)normalized number or zero if (f.u < (113 << 23)) { // resulting FP16 is subnormal or zero // use a magic value to align our 10 mantissa bits at the bottom of // the float. as long as FP addition is round-to-nearest-even this // just works. f.f += denorm_magic.f; // and one integer subtract of the bias later, we have our final float! o.x = static_cast(f.u - denorm_magic.u); } else { unsigned int mant_odd = (f.u >> 13) & 1; // resulting mantissa is odd // update exponent, rounding bias part 1 // Equivalent to `f.u += ((unsigned int)(15 - 127) << 23) + 0xfff`, but // without arithmetic overflow. f.u += 0xc8000fffU; // rounding bias part 2 f.u += mant_odd; // take the bits! o.x = static_cast(f.u >> 13); } } o.x |= static_cast(sign >> 16); return o; #endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC float half_to_float(__half_raw h) { #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) return __half2float(h); #elif defined(EIGEN_HAS_FP16_C) return _cvtsh_ss(h.x); #elif defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) return static_cast(h.x); #else const float32_bits magic = { 113 << 23 }; const unsigned int shifted_exp = 0x7c00 << 13; // exponent mask after shift float32_bits o; o.u = (h.x & 0x7fff) << 13; // exponent/mantissa bits unsigned int exp = shifted_exp & o.u; // just the exponent o.u += (127 - 15) << 23; // exponent adjust // handle exponent special cases if (exp == shifted_exp) { // Inf/NaN? o.u += (128 - 16) << 23; // extra exp adjust } else if (exp == 0) { // Zero/Denormal? o.u += 1 << 23; // extra exp adjust o.f -= magic.f; // renormalize } o.u |= (h.x & 0x8000) << 16; // sign bit return o.f; #endif } // --- standard functions --- EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isinf)(const half& a) { #ifdef EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC return (numext::bit_cast(a.x) & 0x7fff) == 0x7c00; #else return (a.x & 0x7fff) == 0x7c00; #endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isnan)(const half& a) { #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 530) || \ (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) return __hisnan(a); #elif defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) return (numext::bit_cast(a.x) & 0x7fff) > 0x7c00; #else return (a.x & 0x7fff) > 0x7c00; #endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isfinite)(const half& a) { return !(isinf EIGEN_NOT_A_MACRO (a)) && !(isnan EIGEN_NOT_A_MACRO (a)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half abs(const half& a) { #if defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) return half(vabsh_f16(a.x)); #else half result; result.x = a.x & 0x7FFF; return result; #endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half exp(const half& a) { #if (EIGEN_CUDA_SDK_VER >= 80000 && defined EIGEN_CUDA_ARCH && EIGEN_CUDA_ARCH >= 530) || \ defined(EIGEN_HIP_DEVICE_COMPILE) return half(hexp(a)); #else return half(::expf(float(a))); #endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half expm1(const half& a) { return half(numext::expm1(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half log(const half& a) { #if (defined(EIGEN_HAS_CUDA_FP16) && EIGEN_CUDA_SDK_VER >= 80000 && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 530) || \ (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) return half(::hlog(a)); #else return half(::logf(float(a))); #endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half log1p(const half& a) { return half(numext::log1p(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half log10(const half& a) { return half(::log10f(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half log2(const half& a) { return half(static_cast(EIGEN_LOG2E) * ::logf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half sqrt(const half& a) { #if (EIGEN_CUDA_SDK_VER >= 80000 && defined EIGEN_CUDA_ARCH && EIGEN_CUDA_ARCH >= 530) || \ defined(EIGEN_HIP_DEVICE_COMPILE) return half(hsqrt(a)); #else return half(::sqrtf(float(a))); #endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half pow(const half& a, const half& b) { return half(::powf(float(a), float(b))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half sin(const half& a) { return half(::sinf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half cos(const half& a) { return half(::cosf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half tan(const half& a) { return half(::tanf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half tanh(const half& a) { return half(::tanhf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half asin(const half& a) { return half(::asinf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half acos(const half& a) { return half(::acosf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half floor(const half& a) { #if (EIGEN_CUDA_SDK_VER >= 80000 && defined EIGEN_CUDA_ARCH && EIGEN_CUDA_ARCH >= 300) || \ defined(EIGEN_HIP_DEVICE_COMPILE) return half(hfloor(a)); #else return half(::floorf(float(a))); #endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half ceil(const half& a) { #if (EIGEN_CUDA_SDK_VER >= 80000 && defined EIGEN_CUDA_ARCH && EIGEN_CUDA_ARCH >= 300) || \ defined(EIGEN_HIP_DEVICE_COMPILE) return half(hceil(a)); #else return half(::ceilf(float(a))); #endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half rint(const half& a) { return half(::rintf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half round(const half& a) { return half(::roundf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half fmod(const half& a, const half& b) { return half(::fmodf(float(a), float(b))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half (min)(const half& a, const half& b) { #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 530) || \ (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) return __hlt(b, a) ? b : a; #else const float f1 = static_cast(a); const float f2 = static_cast(b); return f2 < f1 ? b : a; #endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half (max)(const half& a, const half& b) { #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 530) || \ (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) return __hlt(a, b) ? b : a; #else const float f1 = static_cast(a); const float f2 = static_cast(b); return f1 < f2 ? b : a; #endif } #ifndef EIGEN_NO_IO EIGEN_ALWAYS_INLINE std::ostream& operator << (std::ostream& os, const half& v) { os << static_cast(v); return os; } #endif } // end namespace half_impl // import Eigen::half_impl::half into Eigen namespace // using half_impl::half; namespace internal { template<> struct random_default_impl { static inline half run(const half& x, const half& y) { return x + (y-x) * half(float(std::rand()) / float(RAND_MAX)); } static inline half run() { return run(half(-1.f), half(1.f)); } }; template<> struct is_arithmetic { enum { value = true }; }; } // end namespace internal template<> struct NumTraits : GenericNumTraits { enum { IsSigned = true, IsInteger = false, IsComplex = false, RequireInitialization = false }; EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::half epsilon() { return half_impl::raw_uint16_to_half(0x0800); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::half dummy_precision() { return half_impl::raw_uint16_to_half(0x211f); // Eigen::half(1e-2f); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::half highest() { return half_impl::raw_uint16_to_half(0x7bff); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::half lowest() { return half_impl::raw_uint16_to_half(0xfbff); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::half infinity() { return half_impl::raw_uint16_to_half(0x7c00); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::half quiet_NaN() { return half_impl::raw_uint16_to_half(0x7e00); } }; } // end namespace Eigen #if defined(EIGEN_HAS_GPU_FP16) || defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) #pragma pop_macro("EIGEN_CONSTEXPR") #endif namespace Eigen { namespace numext { #if defined(EIGEN_GPU_COMPILE_PHASE) template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isnan)(const Eigen::half& h) { return (half_impl::isnan)(h); } template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isinf)(const Eigen::half& h) { return (half_impl::isinf)(h); } template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isfinite)(const Eigen::half& h) { return (half_impl::isfinite)(h); } #endif template <> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half bit_cast(const uint16_t& src) { return Eigen::half(Eigen::half_impl::raw_uint16_to_half(src)); } template <> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC uint16_t bit_cast(const Eigen::half& src) { return Eigen::half_impl::raw_half_as_uint16(src); } } // namespace numext } // namespace Eigen // Add the missing shfl* intrinsics. // The __shfl* functions are only valid on HIP or _CUDA_ARCH_ >= 300. // CUDA defines them for (__CUDA_ARCH__ >= 300 || !defined(__CUDA_ARCH__)) // // HIP and CUDA prior to SDK 9.0 define // __shfl, __shfl_up, __shfl_down, __shfl_xor for int and float // CUDA since 9.0 deprecates those and instead defines // __shfl_sync, __shfl_up_sync, __shfl_down_sync, __shfl_xor_sync, // with native support for __half and __nv_bfloat16 // // Note that the following are __device__ - only functions. #if (defined(EIGEN_CUDACC) && (!defined(EIGEN_CUDA_ARCH) || EIGEN_CUDA_ARCH >= 300)) \ || defined(EIGEN_HIPCC) #if defined(EIGEN_HAS_CUDA_FP16) && EIGEN_CUDA_SDK_VER >= 90000 __device__ EIGEN_STRONG_INLINE Eigen::half __shfl_sync(unsigned mask, Eigen::half var, int srcLane, int width=warpSize) { const __half h = var; return static_cast(__shfl_sync(mask, h, srcLane, width)); } __device__ EIGEN_STRONG_INLINE Eigen::half __shfl_up_sync(unsigned mask, Eigen::half var, unsigned int delta, int width=warpSize) { const __half h = var; return static_cast(__shfl_up_sync(mask, h, delta, width)); } __device__ EIGEN_STRONG_INLINE Eigen::half __shfl_down_sync(unsigned mask, Eigen::half var, unsigned int delta, int width=warpSize) { const __half h = var; return static_cast(__shfl_down_sync(mask, h, delta, width)); } __device__ EIGEN_STRONG_INLINE Eigen::half __shfl_xor_sync(unsigned mask, Eigen::half var, int laneMask, int width=warpSize) { const __half h = var; return static_cast(__shfl_xor_sync(mask, h, laneMask, width)); } #else // HIP or CUDA SDK < 9.0 __device__ EIGEN_STRONG_INLINE Eigen::half __shfl(Eigen::half var, int srcLane, int width=warpSize) { const int ivar = static_cast(Eigen::numext::bit_cast(var)); return Eigen::numext::bit_cast(static_cast(__shfl(ivar, srcLane, width))); } __device__ EIGEN_STRONG_INLINE Eigen::half __shfl_up(Eigen::half var, unsigned int delta, int width=warpSize) { const int ivar = static_cast(Eigen::numext::bit_cast(var)); return Eigen::numext::bit_cast(static_cast(__shfl_up(ivar, delta, width))); } __device__ EIGEN_STRONG_INLINE Eigen::half __shfl_down(Eigen::half var, unsigned int delta, int width=warpSize) { const int ivar = static_cast(Eigen::numext::bit_cast(var)); return Eigen::numext::bit_cast(static_cast(__shfl_down(ivar, delta, width))); } __device__ EIGEN_STRONG_INLINE Eigen::half __shfl_xor(Eigen::half var, int laneMask, int width=warpSize) { const int ivar = static_cast(Eigen::numext::bit_cast(var)); return Eigen::numext::bit_cast(static_cast(__shfl_xor(ivar, laneMask, width))); } #endif // HIP vs CUDA #endif // __shfl* // ldg() has an overload for __half_raw, but we also need one for Eigen::half. #if (defined(EIGEN_CUDACC) && (!defined(EIGEN_CUDA_ARCH) || EIGEN_CUDA_ARCH >= 350)) \ || defined(EIGEN_HIPCC) EIGEN_STRONG_INLINE __device__ Eigen::half __ldg(const Eigen::half* ptr) { return Eigen::half_impl::raw_uint16_to_half(__ldg(reinterpret_cast(ptr))); } #endif // __ldg #if EIGEN_HAS_STD_HASH namespace std { template <> struct hash { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::size_t operator()(const Eigen::half& a) const { return static_cast(Eigen::numext::bit_cast(a)); } }; } // end namespace std #endif #endif // EIGEN_HALF_H RcppEigen/inst/include/Eigen/src/Core/arch/Default/GenericPacketMathFunctionsFwd.h0000644000176200001440000000727214567757725027617 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2019 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_ARCH_GENERIC_PACKET_MATH_FUNCTIONS_FWD_H #define EIGEN_ARCH_GENERIC_PACKET_MATH_FUNCTIONS_FWD_H namespace Eigen { namespace internal { // Forward declarations of the generic math functions // implemented in GenericPacketMathFunctions.h // This is needed to workaround a circular dependency. /*************************************************************************** * Some generic implementations to be used by implementors ***************************************************************************/ /** Default implementation of pfrexp. * It is expected to be called by implementers of template<> pfrexp. */ template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet pfrexp_generic(const Packet& a, Packet& exponent); // Extracts the biased exponent value from Packet p, and casts the results to // a floating-point Packet type. Used by pfrexp_generic. Override this if // there is no unpacket_traits::integer_packet. template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet pfrexp_generic_get_biased_exponent(const Packet& p); /** Default implementation of pldexp. * It is expected to be called by implementers of template<> pldexp. */ template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet pldexp_generic(const Packet& a, const Packet& exponent); /** \internal \returns log(x) for single precision float */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet plog_float(const Packet _x); /** \internal \returns log2(x) for single precision float */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet plog2_float(const Packet _x); /** \internal \returns log(x) for single precision float */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet plog_double(const Packet _x); /** \internal \returns log2(x) for single precision float */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet plog2_double(const Packet _x); /** \internal \returns log(1 + x) */ template Packet generic_plog1p(const Packet& x); /** \internal \returns exp(x)-1 */ template Packet generic_expm1(const Packet& x); /** \internal \returns exp(x) for single precision float */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet pexp_float(const Packet _x); /** \internal \returns exp(x) for double precision real numbers */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet pexp_double(const Packet _x); /** \internal \returns sin(x) for single precision float */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet psin_float(const Packet& x); /** \internal \returns cos(x) for single precision float */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet pcos_float(const Packet& x); /** \internal \returns sqrt(x) for complex types */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet psqrt_complex(const Packet& a); template struct ppolevl; } // end namespace internal } // end namespace Eigen #endif // EIGEN_ARCH_GENERIC_PACKET_MATH_FUNCTIONS_FWD_H RcppEigen/inst/include/Eigen/src/Core/arch/Default/BFloat16.h0000644000176200001440000006442714567757725023272 0ustar liggesusers/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ #ifndef EIGEN_BFLOAT16_H #define EIGEN_BFLOAT16_H #define BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, METHOD) \ template <> \ EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED \ PACKET_BF16 METHOD(const PACKET_BF16& _x) { \ return F32ToBf16(METHOD(Bf16ToF32(_x))); \ } namespace Eigen { struct bfloat16; namespace bfloat16_impl { // Make our own __bfloat16_raw definition. struct __bfloat16_raw { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw() : value(0) {} explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw(unsigned short raw) : value(raw) {} unsigned short value; }; EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw raw_uint16_to_bfloat16(unsigned short value); template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne(float ff); // Forward declarations of template specializations, to avoid Visual C++ 2019 errors, saying: // > error C2908: explicit specialization; 'float_to_bfloat16_rtne' has already been instantiated template <> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne(float ff); template <> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne(float ff); EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC float bfloat16_to_float(__bfloat16_raw h); struct bfloat16_base : public __bfloat16_raw { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16_base() {} EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16_base(const __bfloat16_raw& h) : __bfloat16_raw(h) {} }; } // namespace bfloat16_impl // Class definition. struct bfloat16 : public bfloat16_impl::bfloat16_base { typedef bfloat16_impl::__bfloat16_raw __bfloat16_raw; EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16() {} EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16(const __bfloat16_raw& h) : bfloat16_impl::bfloat16_base(h) {} explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16(bool b) : bfloat16_impl::bfloat16_base(bfloat16_impl::raw_uint16_to_bfloat16(b ? 0x3f80 : 0)) {} template explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16(T val) : bfloat16_impl::bfloat16_base(bfloat16_impl::float_to_bfloat16_rtne::value>(static_cast(val))) {} explicit EIGEN_DEVICE_FUNC bfloat16(float f) : bfloat16_impl::bfloat16_base(bfloat16_impl::float_to_bfloat16_rtne(f)) {} // Following the convention of numpy, converting between complex and // float will lead to loss of imag value. template explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16(const std::complex& val) : bfloat16_impl::bfloat16_base(bfloat16_impl::float_to_bfloat16_rtne(static_cast(val.real()))) {} EIGEN_DEVICE_FUNC operator float() const { // NOLINT: Allow implicit conversion to float, because it is lossless. return bfloat16_impl::bfloat16_to_float(*this); } }; } // namespace Eigen namespace std { template<> struct numeric_limits { static const bool is_specialized = true; static const bool is_signed = true; static const bool is_integer = false; static const bool is_exact = false; static const bool has_infinity = true; static const bool has_quiet_NaN = true; static const bool has_signaling_NaN = true; static const float_denorm_style has_denorm = std::denorm_absent; static const bool has_denorm_loss = false; static const std::float_round_style round_style = numeric_limits::round_style; static const bool is_iec559 = false; static const bool is_bounded = true; static const bool is_modulo = false; static const int digits = 8; static const int digits10 = 2; static const int max_digits10 = 4; static const int radix = 2; static const int min_exponent = numeric_limits::min_exponent; static const int min_exponent10 = numeric_limits::min_exponent10; static const int max_exponent = numeric_limits::max_exponent; static const int max_exponent10 = numeric_limits::max_exponent10; static const bool traps = numeric_limits::traps; static const bool tinyness_before = numeric_limits::tinyness_before; static Eigen::bfloat16 (min)() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x0080); } static Eigen::bfloat16 lowest() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0xff7f); } static Eigen::bfloat16 (max)() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7f7f); } static Eigen::bfloat16 epsilon() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x3c00); } static Eigen::bfloat16 round_error() { return Eigen::bfloat16(0x3f00); } static Eigen::bfloat16 infinity() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7f80); } static Eigen::bfloat16 quiet_NaN() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7fc0); } static Eigen::bfloat16 signaling_NaN() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7f81); } static Eigen::bfloat16 denorm_min() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x0001); } }; // If std::numeric_limits is specialized, should also specialize // std::numeric_limits, std::numeric_limits, and // std::numeric_limits // https://stackoverflow.com/a/16519653/ template<> struct numeric_limits : numeric_limits {}; template<> struct numeric_limits : numeric_limits {}; template<> struct numeric_limits : numeric_limits {}; } // namespace std namespace Eigen { namespace bfloat16_impl { // We need to distinguish ‘clang as the CUDA compiler’ from ‘clang as the host compiler, // invoked by NVCC’ (e.g. on MacOS). The former needs to see both host and device implementation // of the functions, while the latter can only deal with one of them. #if !defined(EIGEN_HAS_NATIVE_BF16) || (EIGEN_COMP_CLANG && !EIGEN_COMP_NVCC) // Emulate support for bfloat16 floats #if EIGEN_COMP_CLANG && defined(EIGEN_CUDACC) // We need to provide emulated *host-side* BF16 operators for clang. #pragma push_macro("EIGEN_DEVICE_FUNC") #undef EIGEN_DEVICE_FUNC #if defined(EIGEN_HAS_CUDA_BF16) && defined(EIGEN_HAS_NATIVE_BF16) #define EIGEN_DEVICE_FUNC __host__ #else // both host and device need emulated ops. #define EIGEN_DEVICE_FUNC __host__ __device__ #endif #endif // Definitions for CPUs, mostly working through conversion // to/from fp32. EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator + (const bfloat16& a, const bfloat16& b) { return bfloat16(float(a) + float(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator + (const bfloat16& a, const int& b) { return bfloat16(float(a) + static_cast(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator + (const int& a, const bfloat16& b) { return bfloat16(static_cast(a) + float(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator * (const bfloat16& a, const bfloat16& b) { return bfloat16(float(a) * float(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator - (const bfloat16& a, const bfloat16& b) { return bfloat16(float(a) - float(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator / (const bfloat16& a, const bfloat16& b) { return bfloat16(float(a) / float(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator - (const bfloat16& a) { bfloat16 result; result.value = a.value ^ 0x8000; return result; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator += (bfloat16& a, const bfloat16& b) { a = bfloat16(float(a) + float(b)); return a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator *= (bfloat16& a, const bfloat16& b) { a = bfloat16(float(a) * float(b)); return a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator -= (bfloat16& a, const bfloat16& b) { a = bfloat16(float(a) - float(b)); return a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator /= (bfloat16& a, const bfloat16& b) { a = bfloat16(float(a) / float(b)); return a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator++(bfloat16& a) { a += bfloat16(1); return a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator--(bfloat16& a) { a -= bfloat16(1); return a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator++(bfloat16& a, int) { bfloat16 original_value = a; ++a; return original_value; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator--(bfloat16& a, int) { bfloat16 original_value = a; --a; return original_value; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator == (const bfloat16& a, const bfloat16& b) { return numext::equal_strict(float(a),float(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator != (const bfloat16& a, const bfloat16& b) { return numext::not_equal_strict(float(a), float(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator < (const bfloat16& a, const bfloat16& b) { return float(a) < float(b); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator <= (const bfloat16& a, const bfloat16& b) { return float(a) <= float(b); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator > (const bfloat16& a, const bfloat16& b) { return float(a) > float(b); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator >= (const bfloat16& a, const bfloat16& b) { return float(a) >= float(b); } #if EIGEN_COMP_CLANG && defined(EIGEN_CUDACC) #pragma pop_macro("EIGEN_DEVICE_FUNC") #endif #endif // Emulate support for bfloat16 floats // Division by an index. Do it in full float precision to avoid accuracy // issues in converting the denominator to bfloat16. EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator / (const bfloat16& a, Index b) { return bfloat16(static_cast(a) / static_cast(b)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw truncate_to_bfloat16(const float v) { __bfloat16_raw output; if (Eigen::numext::isnan EIGEN_NOT_A_MACRO(v)) { output.value = std::signbit(v) ? 0xFFC0: 0x7FC0; return output; } const uint16_t* p = reinterpret_cast(&v); #if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ output.value = p[0]; #else output.value = p[1]; #endif return output; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw raw_uint16_to_bfloat16(numext::uint16_t value) { return __bfloat16_raw(value); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR numext::uint16_t raw_bfloat16_as_uint16(const __bfloat16_raw& bf) { return bf.value; } // float_to_bfloat16_rtne template specialization that does not make any // assumption about the value of its function argument (ff). template <> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne(float ff) { #if (defined(EIGEN_HAS_CUDA_BF16) && defined(EIGEN_HAS_HIP_BF16)) // Nothing to do here #else __bfloat16_raw output; if (Eigen::numext::isnan EIGEN_NOT_A_MACRO(ff)) { // If the value is a NaN, squash it to a qNaN with msb of fraction set, // this makes sure after truncation we don't end up with an inf. // // qNaN magic: All exponent bits set + most significant bit of fraction // set. output.value = std::signbit(ff) ? 0xFFC0: 0x7FC0; } else { // Fast rounding algorithm that rounds a half value to nearest even. This // reduces expected error when we convert a large number of floats. Here // is how it works: // // Definitions: // To convert a float 32 to bfloat16, a float 32 can be viewed as 32 bits // with the following tags: // // Sign | Exp (8 bits) | Frac (23 bits) // S EEEEEEEE FFFFFFLRTTTTTTTTTTTTTTT // // S: Sign bit. // E: Exponent bits. // F: First 6 bits of fraction. // L: Least significant bit of resulting bfloat16 if we truncate away the // rest of the float32. This is also the 7th bit of fraction // R: Rounding bit, 8th bit of fraction. // T: Sticky bits, rest of fraction, 15 bits. // // To round half to nearest even, there are 3 cases where we want to round // down (simply truncate the result of the bits away, which consists of // rounding bit and sticky bits) and two cases where we want to round up // (truncate then add one to the result). // // The fast converting algorithm simply adds lsb (L) to 0x7fff (15 bits of // 1s) as the rounding bias, adds the rounding bias to the input, then // truncates the last 16 bits away. // // To understand how it works, we can analyze this algorithm case by case: // // 1. L = 0, R = 0: // Expect: round down, this is less than half value. // // Algorithm: // - Rounding bias: 0x7fff + 0 = 0x7fff // - Adding rounding bias to input may create any carry, depending on // whether there is any value set to 1 in T bits. // - R may be set to 1 if there is a carry. // - L remains 0. // - Note that this case also handles Inf and -Inf, where all fraction // bits, including L, R and Ts are all 0. The output remains Inf after // this algorithm. // // 2. L = 1, R = 0: // Expect: round down, this is less than half value. // // Algorithm: // - Rounding bias: 0x7fff + 1 = 0x8000 // - Adding rounding bias to input doesn't change sticky bits but // adds 1 to rounding bit. // - L remains 1. // // 3. L = 0, R = 1, all of T are 0: // Expect: round down, this is exactly at half, the result is already // even (L=0). // // Algorithm: // - Rounding bias: 0x7fff + 0 = 0x7fff // - Adding rounding bias to input sets all sticky bits to 1, but // doesn't create a carry. // - R remains 1. // - L remains 0. // // 4. L = 1, R = 1: // Expect: round up, this is exactly at half, the result needs to be // round to the next even number. // // Algorithm: // - Rounding bias: 0x7fff + 1 = 0x8000 // - Adding rounding bias to input doesn't change sticky bits, but // creates a carry from rounding bit. // - The carry sets L to 0, creates another carry bit and propagate // forward to F bits. // - If all the F bits are 1, a carry then propagates to the exponent // bits, which then creates the minimum value with the next exponent // value. Note that we won't have the case where exponents are all 1, // since that's either a NaN (handled in the other if condition) or inf // (handled in case 1). // // 5. L = 0, R = 1, any of T is 1: // Expect: round up, this is greater than half. // // Algorithm: // - Rounding bias: 0x7fff + 0 = 0x7fff // - Adding rounding bias to input creates a carry from sticky bits, // sets rounding bit to 0, then create another carry. // - The second carry sets L to 1. // // Examples: // // Exact half value that is already even: // Input: // Sign | Exp (8 bit) | Frac (first 7 bit) | Frac (last 16 bit) // S E E E E E E E E F F F F F F L RTTTTTTTTTTTTTTT // 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1000000000000000 // // This falls into case 3. We truncate the rest of 16 bits and no // carry is created into F and L: // // Output: // Sign | Exp (8 bit) | Frac (first 7 bit) // S E E E E E E E E F F F F F F L // 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 // // Exact half value, round to next even number: // Input: // Sign | Exp (8 bit) | Frac (first 7 bit) | Frac (last 16 bit) // S E E E E E E E E F F F F F F L RTTTTTTTTTTTTTTT // 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1000000000000000 // // This falls into case 4. We create a carry from R and T, // which then propagates into L and F: // // Output: // Sign | Exp (8 bit) | Frac (first 7 bit) // S E E E E E E E E F F F F F F L // 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 // // // Max denormal value round to min normal value: // Input: // Sign | Exp (8 bit) | Frac (first 7 bit) | Frac (last 16 bit) // S E E E E E E E E F F F F F F L RTTTTTTTTTTTTTTT // 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1111111111111111 // // This falls into case 4. We create a carry from R and T, // propagate into L and F, which then propagates into exponent // bits: // // Output: // Sign | Exp (8 bit) | Frac (first 7 bit) // S E E E E E E E E F F F F F F L // 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 // // Max normal value round to Inf: // Input: // Sign | Exp (8 bit) | Frac (first 7 bit) | Frac (last 16 bit) // S E E E E E E E E F F F F F F L RTTTTTTTTTTTTTTT // 0 1 1 1 1 1 1 1 0 1 1 1 1 1 1 1 1111111111111111 // // This falls into case 4. We create a carry from R and T, // propagate into L and F, which then propagates into exponent // bits: // // Sign | Exp (8 bit) | Frac (first 7 bit) // S E E E E E E E E F F F F F F L // 0 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 // At this point, ff must be either a normal float, or +/-infinity. output = float_to_bfloat16_rtne(ff); } return output; #endif } // float_to_bfloat16_rtne template specialization that assumes that its function // argument (ff) is either a normal floating point number, or +/-infinity, or // zero. Used to improve the runtime performance of conversion from an integer // type to bfloat16. template <> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne(float ff) { #if (defined(EIGEN_HAS_CUDA_BF16) && defined(EIGEN_HAS_HIP_BF16)) // Nothing to do here #else numext::uint32_t input = numext::bit_cast(ff); __bfloat16_raw output; // Least significant bit of resulting bfloat. numext::uint32_t lsb = (input >> 16) & 1; numext::uint32_t rounding_bias = 0x7fff + lsb; input += rounding_bias; output.value = static_cast(input >> 16); return output; #endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC float bfloat16_to_float(__bfloat16_raw h) { float result = 0; unsigned short* q = reinterpret_cast(&result); #if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ q[0] = h.value; #else q[1] = h.value; #endif return result; } // --- standard functions --- EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isinf)(const bfloat16& a) { EIGEN_USING_STD(isinf); return (isinf)(float(a)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isnan)(const bfloat16& a) { EIGEN_USING_STD(isnan); return (isnan)(float(a)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isfinite)(const bfloat16& a) { return !(isinf EIGEN_NOT_A_MACRO (a)) && !(isnan EIGEN_NOT_A_MACRO (a)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 abs(const bfloat16& a) { bfloat16 result; result.value = a.value & 0x7FFF; return result; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 exp(const bfloat16& a) { return bfloat16(::expf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 expm1(const bfloat16& a) { return bfloat16(numext::expm1(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log(const bfloat16& a) { return bfloat16(::logf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log1p(const bfloat16& a) { return bfloat16(numext::log1p(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log10(const bfloat16& a) { return bfloat16(::log10f(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log2(const bfloat16& a) { return bfloat16(static_cast(EIGEN_LOG2E) * ::logf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 sqrt(const bfloat16& a) { return bfloat16(::sqrtf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 pow(const bfloat16& a, const bfloat16& b) { return bfloat16(::powf(float(a), float(b))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 sin(const bfloat16& a) { return bfloat16(::sinf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 cos(const bfloat16& a) { return bfloat16(::cosf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 tan(const bfloat16& a) { return bfloat16(::tanf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 asin(const bfloat16& a) { return bfloat16(::asinf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 acos(const bfloat16& a) { return bfloat16(::acosf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 atan(const bfloat16& a) { return bfloat16(::atanf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 sinh(const bfloat16& a) { return bfloat16(::sinhf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 cosh(const bfloat16& a) { return bfloat16(::coshf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 tanh(const bfloat16& a) { return bfloat16(::tanhf(float(a))); } #if EIGEN_HAS_CXX11_MATH EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 asinh(const bfloat16& a) { return bfloat16(::asinhf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 acosh(const bfloat16& a) { return bfloat16(::acoshf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 atanh(const bfloat16& a) { return bfloat16(::atanhf(float(a))); } #endif EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 floor(const bfloat16& a) { return bfloat16(::floorf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 ceil(const bfloat16& a) { return bfloat16(::ceilf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 rint(const bfloat16& a) { return bfloat16(::rintf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 round(const bfloat16& a) { return bfloat16(::roundf(float(a))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 fmod(const bfloat16& a, const bfloat16& b) { return bfloat16(::fmodf(float(a), float(b))); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 (min)(const bfloat16& a, const bfloat16& b) { const float f1 = static_cast(a); const float f2 = static_cast(b); return f2 < f1 ? b : a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 (max)(const bfloat16& a, const bfloat16& b) { const float f1 = static_cast(a); const float f2 = static_cast(b); return f1 < f2 ? b : a; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 fmin(const bfloat16& a, const bfloat16& b) { const float f1 = static_cast(a); const float f2 = static_cast(b); return bfloat16(::fminf(f1, f2)); } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 fmax(const bfloat16& a, const bfloat16& b) { const float f1 = static_cast(a); const float f2 = static_cast(b); return bfloat16(::fmaxf(f1, f2)); } #ifndef EIGEN_NO_IO EIGEN_ALWAYS_INLINE std::ostream& operator << (std::ostream& os, const bfloat16& v) { os << static_cast(v); return os; } #endif } // namespace bfloat16_impl namespace internal { template<> struct random_default_impl { static inline bfloat16 run(const bfloat16& x, const bfloat16& y) { return x + (y-x) * bfloat16(float(std::rand()) / float(RAND_MAX)); } static inline bfloat16 run() { return run(bfloat16(-1.f), bfloat16(1.f)); } }; template<> struct is_arithmetic { enum { value = true }; }; } // namespace internal template<> struct NumTraits : GenericNumTraits { enum { IsSigned = true, IsInteger = false, IsComplex = false, RequireInitialization = false }; EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 epsilon() { return bfloat16_impl::raw_uint16_to_bfloat16(0x3c00); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 dummy_precision() { return bfloat16_impl::raw_uint16_to_bfloat16(0x3D4D); // bfloat16(5e-2f); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 highest() { return bfloat16_impl::raw_uint16_to_bfloat16(0x7F7F); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 lowest() { return bfloat16_impl::raw_uint16_to_bfloat16(0xFF7F); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 infinity() { return bfloat16_impl::raw_uint16_to_bfloat16(0x7f80); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 quiet_NaN() { return bfloat16_impl::raw_uint16_to_bfloat16(0x7fc0); } }; } // namespace Eigen namespace Eigen { namespace numext { template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool (isnan)(const Eigen::bfloat16& h) { return (bfloat16_impl::isnan)(h); } template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool (isinf)(const Eigen::bfloat16& h) { return (bfloat16_impl::isinf)(h); } template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool (isfinite)(const Eigen::bfloat16& h) { return (bfloat16_impl::isfinite)(h); } template <> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::bfloat16 bit_cast(const uint16_t& src) { return Eigen::bfloat16(Eigen::bfloat16_impl::raw_uint16_to_bfloat16(src)); } template <> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC uint16_t bit_cast(const Eigen::bfloat16& src) { return Eigen::bfloat16_impl::raw_bfloat16_as_uint16(src); } } // namespace numext } // namespace Eigen #if EIGEN_HAS_STD_HASH namespace std { template <> struct hash { EIGEN_STRONG_INLINE std::size_t operator()(const Eigen::bfloat16& a) const { return static_cast(Eigen::numext::bit_cast(a)); } }; } // namespace std #endif #endif // EIGEN_BFLOAT16_H RcppEigen/inst/include/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h0000644000176200001440000020416014567757725027151 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2007 Julien Pommier // Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com) // Copyright (C) 2009-2019 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. /* The exp and log functions of this file initially come from * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/ */ #ifndef EIGEN_ARCH_GENERIC_PACKET_MATH_FUNCTIONS_H #define EIGEN_ARCH_GENERIC_PACKET_MATH_FUNCTIONS_H namespace Eigen { namespace internal { // Creates a Scalar integer type with same bit-width. template struct make_integer; template<> struct make_integer { typedef numext::int32_t type; }; template<> struct make_integer { typedef numext::int64_t type; }; template<> struct make_integer { typedef numext::int16_t type; }; template<> struct make_integer { typedef numext::int16_t type; }; template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet pfrexp_generic_get_biased_exponent(const Packet& a) { typedef typename unpacket_traits::type Scalar; typedef typename unpacket_traits::integer_packet PacketI; enum { mantissa_bits = numext::numeric_limits::digits - 1}; return pcast(plogical_shift_right(preinterpret(pabs(a)))); } // Safely applies frexp, correctly handles denormals. // Assumes IEEE floating point format. template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet pfrexp_generic(const Packet& a, Packet& exponent) { typedef typename unpacket_traits::type Scalar; typedef typename make_unsigned::type>::type ScalarUI; enum { TotalBits = sizeof(Scalar) * CHAR_BIT, MantissaBits = numext::numeric_limits::digits - 1, ExponentBits = int(TotalBits) - int(MantissaBits) - 1 }; EIGEN_CONSTEXPR ScalarUI scalar_sign_mantissa_mask = ~(((ScalarUI(1) << int(ExponentBits)) - ScalarUI(1)) << int(MantissaBits)); // ~0x7f800000 const Packet sign_mantissa_mask = pset1frombits(static_cast(scalar_sign_mantissa_mask)); const Packet half = pset1(Scalar(0.5)); const Packet zero = pzero(a); const Packet normal_min = pset1((numext::numeric_limits::min)()); // Minimum normal value, 2^-126 // To handle denormals, normalize by multiplying by 2^(int(MantissaBits)+1). const Packet is_denormal = pcmp_lt(pabs(a), normal_min); EIGEN_CONSTEXPR ScalarUI scalar_normalization_offset = ScalarUI(int(MantissaBits) + 1); // 24 // The following cannot be constexpr because bfloat16(uint16_t) is not constexpr. const Scalar scalar_normalization_factor = Scalar(ScalarUI(1) << int(scalar_normalization_offset)); // 2^24 const Packet normalization_factor = pset1(scalar_normalization_factor); const Packet normalized_a = pselect(is_denormal, pmul(a, normalization_factor), a); // Determine exponent offset: -126 if normal, -126-24 if denormal const Scalar scalar_exponent_offset = -Scalar((ScalarUI(1)<<(int(ExponentBits)-1)) - ScalarUI(2)); // -126 Packet exponent_offset = pset1(scalar_exponent_offset); const Packet normalization_offset = pset1(-Scalar(scalar_normalization_offset)); // -24 exponent_offset = pselect(is_denormal, padd(exponent_offset, normalization_offset), exponent_offset); // Determine exponent and mantissa from normalized_a. exponent = pfrexp_generic_get_biased_exponent(normalized_a); // Zero, Inf and NaN return 'a' unmodified, exponent is zero // (technically the exponent is unspecified for inf/NaN, but GCC/Clang set it to zero) const Scalar scalar_non_finite_exponent = Scalar((ScalarUI(1) << int(ExponentBits)) - ScalarUI(1)); // 255 const Packet non_finite_exponent = pset1(scalar_non_finite_exponent); const Packet is_zero_or_not_finite = por(pcmp_eq(a, zero), pcmp_eq(exponent, non_finite_exponent)); const Packet m = pselect(is_zero_or_not_finite, a, por(pand(normalized_a, sign_mantissa_mask), half)); exponent = pselect(is_zero_or_not_finite, zero, padd(exponent, exponent_offset)); return m; } // Safely applies ldexp, correctly handles overflows, underflows and denormals. // Assumes IEEE floating point format. template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet pldexp_generic(const Packet& a, const Packet& exponent) { // We want to return a * 2^exponent, allowing for all possible integer // exponents without overflowing or underflowing in intermediate // computations. // // Since 'a' and the output can be denormal, the maximum range of 'exponent' // to consider for a float is: // -255-23 -> 255+23 // Below -278 any finite float 'a' will become zero, and above +278 any // finite float will become inf, including when 'a' is the smallest possible // denormal. // // Unfortunately, 2^(278) cannot be represented using either one or two // finite normal floats, so we must split the scale factor into at least // three parts. It turns out to be faster to split 'exponent' into four // factors, since [exponent>>2] is much faster to compute that [exponent/3]. // // Set e = min(max(exponent, -278), 278); // b = floor(e/4); // out = ((((a * 2^(b)) * 2^(b)) * 2^(b)) * 2^(e-3*b)) // // This will avoid any intermediate overflows and correctly handle 0, inf, // NaN cases. typedef typename unpacket_traits::integer_packet PacketI; typedef typename unpacket_traits::type Scalar; typedef typename unpacket_traits::type ScalarI; enum { TotalBits = sizeof(Scalar) * CHAR_BIT, MantissaBits = numext::numeric_limits::digits - 1, ExponentBits = int(TotalBits) - int(MantissaBits) - 1 }; const Packet max_exponent = pset1(Scalar((ScalarI(1)<((ScalarI(1)<<(int(ExponentBits)-1)) - ScalarI(1)); // 127 const PacketI e = pcast(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent)); PacketI b = parithmetic_shift_right<2>(e); // floor(e/4); Packet c = preinterpret(plogical_shift_left(padd(b, bias))); // 2^b Packet out = pmul(pmul(pmul(a, c), c), c); // a * 2^(3b) b = psub(psub(psub(e, b), b), b); // e - 3b c = preinterpret(plogical_shift_left(padd(b, bias))); // 2^(e-3*b) out = pmul(out, c); return out; } // Explicitly multiplies // a * (2^e) // clamping e to the range // [NumTraits::min_exponent()-2, NumTraits::max_exponent()] // // This is approx 7x faster than pldexp_impl, but will prematurely over/underflow // if 2^e doesn't fit into a normal floating-point Scalar. // // Assumes IEEE floating point format template struct pldexp_fast_impl { typedef typename unpacket_traits::integer_packet PacketI; typedef typename unpacket_traits::type Scalar; typedef typename unpacket_traits::type ScalarI; enum { TotalBits = sizeof(Scalar) * CHAR_BIT, MantissaBits = numext::numeric_limits::digits - 1, ExponentBits = int(TotalBits) - int(MantissaBits) - 1 }; static EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet run(const Packet& a, const Packet& exponent) { const Packet bias = pset1(Scalar((ScalarI(1)<<(int(ExponentBits)-1)) - ScalarI(1))); // 127 const Packet limit = pset1(Scalar((ScalarI(1)<(pmin(pmax(padd(exponent, bias), pzero(limit)), limit)); // exponent + 127 // return a * (2^e) return pmul(a, preinterpret(plogical_shift_left(e))); } }; // Natural or base 2 logarithm. // Computes log(x) as log(2^e * m) = C*e + log(m), where the constant C =log(2) // and m is in the range [sqrt(1/2),sqrt(2)). In this range, the logarithm can // be easily approximated by a polynomial centered on m=1 for stability. // TODO(gonnet): Further reduce the interval allowing for lower-degree // polynomial interpolants -> ... -> profit! template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet plog_impl_float(const Packet _x) { Packet x = _x; const Packet cst_1 = pset1(1.0f); const Packet cst_neg_half = pset1(-0.5f); // The smallest non denormalized float number. const Packet cst_min_norm_pos = pset1frombits( 0x00800000u); const Packet cst_minus_inf = pset1frombits( 0xff800000u); const Packet cst_pos_inf = pset1frombits( 0x7f800000u); // Polynomial coefficients. const Packet cst_cephes_SQRTHF = pset1(0.707106781186547524f); const Packet cst_cephes_log_p0 = pset1(7.0376836292E-2f); const Packet cst_cephes_log_p1 = pset1(-1.1514610310E-1f); const Packet cst_cephes_log_p2 = pset1(1.1676998740E-1f); const Packet cst_cephes_log_p3 = pset1(-1.2420140846E-1f); const Packet cst_cephes_log_p4 = pset1(+1.4249322787E-1f); const Packet cst_cephes_log_p5 = pset1(-1.6668057665E-1f); const Packet cst_cephes_log_p6 = pset1(+2.0000714765E-1f); const Packet cst_cephes_log_p7 = pset1(-2.4999993993E-1f); const Packet cst_cephes_log_p8 = pset1(+3.3333331174E-1f); // Truncate input values to the minimum positive normal. x = pmax(x, cst_min_norm_pos); Packet e; // extract significant in the range [0.5,1) and exponent x = pfrexp(x,e); // part2: Shift the inputs from the range [0.5,1) to [sqrt(1/2),sqrt(2)) // and shift by -1. The values are then centered around 0, which improves // the stability of the polynomial evaluation. // if( x < SQRTHF ) { // e -= 1; // x = x + x - 1.0; // } else { x = x - 1.0; } Packet mask = pcmp_lt(x, cst_cephes_SQRTHF); Packet tmp = pand(x, mask); x = psub(x, cst_1); e = psub(e, pand(cst_1, mask)); x = padd(x, tmp); Packet x2 = pmul(x, x); Packet x3 = pmul(x2, x); // Evaluate the polynomial approximant of degree 8 in three parts, probably // to improve instruction-level parallelism. Packet y, y1, y2; y = pmadd(cst_cephes_log_p0, x, cst_cephes_log_p1); y1 = pmadd(cst_cephes_log_p3, x, cst_cephes_log_p4); y2 = pmadd(cst_cephes_log_p6, x, cst_cephes_log_p7); y = pmadd(y, x, cst_cephes_log_p2); y1 = pmadd(y1, x, cst_cephes_log_p5); y2 = pmadd(y2, x, cst_cephes_log_p8); y = pmadd(y, x3, y1); y = pmadd(y, x3, y2); y = pmul(y, x3); y = pmadd(cst_neg_half, x2, y); x = padd(x, y); // Add the logarithm of the exponent back to the result of the interpolation. if (base2) { const Packet cst_log2e = pset1(static_cast(EIGEN_LOG2E)); x = pmadd(x, cst_log2e, e); } else { const Packet cst_ln2 = pset1(static_cast(EIGEN_LN2)); x = pmadd(e, cst_ln2, x); } Packet invalid_mask = pcmp_lt_or_nan(_x, pzero(_x)); Packet iszero_mask = pcmp_eq(_x,pzero(_x)); Packet pos_inf_mask = pcmp_eq(_x,cst_pos_inf); // Filter out invalid inputs, i.e.: // - negative arg will be NAN // - 0 will be -INF // - +INF will be +INF return pselect(iszero_mask, cst_minus_inf, por(pselect(pos_inf_mask,cst_pos_inf,x), invalid_mask)); } template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet plog_float(const Packet _x) { return plog_impl_float(_x); } template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet plog2_float(const Packet _x) { return plog_impl_float(_x); } /* Returns the base e (2.718...) or base 2 logarithm of x. * The argument is separated into its exponent and fractional parts. * The logarithm of the fraction in the interval [sqrt(1/2), sqrt(2)], * is approximated by * * log(1+x) = x - 0.5 x**2 + x**3 P(x)/Q(x). * * for more detail see: http://www.netlib.org/cephes/ */ template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet plog_impl_double(const Packet _x) { Packet x = _x; const Packet cst_1 = pset1(1.0); const Packet cst_neg_half = pset1(-0.5); // The smallest non denormalized double. const Packet cst_min_norm_pos = pset1frombits( static_cast(0x0010000000000000ull)); const Packet cst_minus_inf = pset1frombits( static_cast(0xfff0000000000000ull)); const Packet cst_pos_inf = pset1frombits( static_cast(0x7ff0000000000000ull)); // Polynomial Coefficients for log(1+x) = x - x**2/2 + x**3 P(x)/Q(x) // 1/sqrt(2) <= x < sqrt(2) const Packet cst_cephes_SQRTHF = pset1(0.70710678118654752440E0); const Packet cst_cephes_log_p0 = pset1(1.01875663804580931796E-4); const Packet cst_cephes_log_p1 = pset1(4.97494994976747001425E-1); const Packet cst_cephes_log_p2 = pset1(4.70579119878881725854E0); const Packet cst_cephes_log_p3 = pset1(1.44989225341610930846E1); const Packet cst_cephes_log_p4 = pset1(1.79368678507819816313E1); const Packet cst_cephes_log_p5 = pset1(7.70838733755885391666E0); const Packet cst_cephes_log_q0 = pset1(1.0); const Packet cst_cephes_log_q1 = pset1(1.12873587189167450590E1); const Packet cst_cephes_log_q2 = pset1(4.52279145837532221105E1); const Packet cst_cephes_log_q3 = pset1(8.29875266912776603211E1); const Packet cst_cephes_log_q4 = pset1(7.11544750618563894466E1); const Packet cst_cephes_log_q5 = pset1(2.31251620126765340583E1); // Truncate input values to the minimum positive normal. x = pmax(x, cst_min_norm_pos); Packet e; // extract significant in the range [0.5,1) and exponent x = pfrexp(x,e); // Shift the inputs from the range [0.5,1) to [sqrt(1/2),sqrt(2)) // and shift by -1. The values are then centered around 0, which improves // the stability of the polynomial evaluation. // if( x < SQRTHF ) { // e -= 1; // x = x + x - 1.0; // } else { x = x - 1.0; } Packet mask = pcmp_lt(x, cst_cephes_SQRTHF); Packet tmp = pand(x, mask); x = psub(x, cst_1); e = psub(e, pand(cst_1, mask)); x = padd(x, tmp); Packet x2 = pmul(x, x); Packet x3 = pmul(x2, x); // Evaluate the polynomial approximant , probably to improve instruction-level parallelism. // y = x - 0.5*x^2 + x^3 * polevl( x, P, 5 ) / p1evl( x, Q, 5 ) ); Packet y, y1, y_; y = pmadd(cst_cephes_log_p0, x, cst_cephes_log_p1); y1 = pmadd(cst_cephes_log_p3, x, cst_cephes_log_p4); y = pmadd(y, x, cst_cephes_log_p2); y1 = pmadd(y1, x, cst_cephes_log_p5); y_ = pmadd(y, x3, y1); y = pmadd(cst_cephes_log_q0, x, cst_cephes_log_q1); y1 = pmadd(cst_cephes_log_q3, x, cst_cephes_log_q4); y = pmadd(y, x, cst_cephes_log_q2); y1 = pmadd(y1, x, cst_cephes_log_q5); y = pmadd(y, x3, y1); y_ = pmul(y_, x3); y = pdiv(y_, y); y = pmadd(cst_neg_half, x2, y); x = padd(x, y); // Add the logarithm of the exponent back to the result of the interpolation. if (base2) { const Packet cst_log2e = pset1(static_cast(EIGEN_LOG2E)); x = pmadd(x, cst_log2e, e); } else { const Packet cst_ln2 = pset1(static_cast(EIGEN_LN2)); x = pmadd(e, cst_ln2, x); } Packet invalid_mask = pcmp_lt_or_nan(_x, pzero(_x)); Packet iszero_mask = pcmp_eq(_x,pzero(_x)); Packet pos_inf_mask = pcmp_eq(_x,cst_pos_inf); // Filter out invalid inputs, i.e.: // - negative arg will be NAN // - 0 will be -INF // - +INF will be +INF return pselect(iszero_mask, cst_minus_inf, por(pselect(pos_inf_mask,cst_pos_inf,x), invalid_mask)); } template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet plog_double(const Packet _x) { return plog_impl_double(_x); } template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet plog2_double(const Packet _x) { return plog_impl_double(_x); } /** \internal \returns log(1 + x) computed using W. Kahan's formula. See: http://www.plunk.org/~hatch/rightway.php */ template Packet generic_plog1p(const Packet& x) { typedef typename unpacket_traits::type ScalarType; const Packet one = pset1(ScalarType(1)); Packet xp1 = padd(x, one); Packet small_mask = pcmp_eq(xp1, one); Packet log1 = plog(xp1); Packet inf_mask = pcmp_eq(xp1, log1); Packet log_large = pmul(x, pdiv(log1, psub(xp1, one))); return pselect(por(small_mask, inf_mask), x, log_large); } /** \internal \returns exp(x)-1 computed using W. Kahan's formula. See: http://www.plunk.org/~hatch/rightway.php */ template Packet generic_expm1(const Packet& x) { typedef typename unpacket_traits::type ScalarType; const Packet one = pset1(ScalarType(1)); const Packet neg_one = pset1(ScalarType(-1)); Packet u = pexp(x); Packet one_mask = pcmp_eq(u, one); Packet u_minus_one = psub(u, one); Packet neg_one_mask = pcmp_eq(u_minus_one, neg_one); Packet logu = plog(u); // The following comparison is to catch the case where // exp(x) = +inf. It is written in this way to avoid having // to form the constant +inf, which depends on the packet // type. Packet pos_inf_mask = pcmp_eq(logu, u); Packet expm1 = pmul(u_minus_one, pdiv(x, logu)); expm1 = pselect(pos_inf_mask, u, expm1); return pselect(one_mask, x, pselect(neg_one_mask, neg_one, expm1)); } // Exponential function. Works by writing "x = m*log(2) + r" where // "m = floor(x/log(2)+1/2)" and "r" is the remainder. The result is then // "exp(x) = 2^m*exp(r)" where exp(r) is in the range [-1,1). template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet pexp_float(const Packet _x) { const Packet cst_1 = pset1(1.0f); const Packet cst_half = pset1(0.5f); const Packet cst_exp_hi = pset1( 88.723f); const Packet cst_exp_lo = pset1(-88.723f); const Packet cst_cephes_LOG2EF = pset1(1.44269504088896341f); const Packet cst_cephes_exp_p0 = pset1(1.9875691500E-4f); const Packet cst_cephes_exp_p1 = pset1(1.3981999507E-3f); const Packet cst_cephes_exp_p2 = pset1(8.3334519073E-3f); const Packet cst_cephes_exp_p3 = pset1(4.1665795894E-2f); const Packet cst_cephes_exp_p4 = pset1(1.6666665459E-1f); const Packet cst_cephes_exp_p5 = pset1(5.0000001201E-1f); // Clamp x. Packet x = pmax(pmin(_x, cst_exp_hi), cst_exp_lo); // Express exp(x) as exp(m*ln(2) + r), start by extracting // m = floor(x/ln(2) + 0.5). Packet m = pfloor(pmadd(x, cst_cephes_LOG2EF, cst_half)); // Get r = x - m*ln(2). If no FMA instructions are available, m*ln(2) is // subtracted out in two parts, m*C1+m*C2 = m*ln(2), to avoid accumulating // truncation errors. const Packet cst_cephes_exp_C1 = pset1(-0.693359375f); const Packet cst_cephes_exp_C2 = pset1(2.12194440e-4f); Packet r = pmadd(m, cst_cephes_exp_C1, x); r = pmadd(m, cst_cephes_exp_C2, r); Packet r2 = pmul(r, r); Packet r3 = pmul(r2, r); // Evaluate the polynomial approximant,improved by instruction-level parallelism. Packet y, y1, y2; y = pmadd(cst_cephes_exp_p0, r, cst_cephes_exp_p1); y1 = pmadd(cst_cephes_exp_p3, r, cst_cephes_exp_p4); y2 = padd(r, cst_1); y = pmadd(y, r, cst_cephes_exp_p2); y1 = pmadd(y1, r, cst_cephes_exp_p5); y = pmadd(y, r3, y1); y = pmadd(y, r2, y2); // Return 2^m * exp(r). // TODO: replace pldexp with faster implementation since y in [-1, 1). return pmax(pldexp(y,m), _x); } template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet pexp_double(const Packet _x) { Packet x = _x; const Packet cst_1 = pset1(1.0); const Packet cst_2 = pset1(2.0); const Packet cst_half = pset1(0.5); const Packet cst_exp_hi = pset1(709.784); const Packet cst_exp_lo = pset1(-709.784); const Packet cst_cephes_LOG2EF = pset1(1.4426950408889634073599); const Packet cst_cephes_exp_p0 = pset1(1.26177193074810590878e-4); const Packet cst_cephes_exp_p1 = pset1(3.02994407707441961300e-2); const Packet cst_cephes_exp_p2 = pset1(9.99999999999999999910e-1); const Packet cst_cephes_exp_q0 = pset1(3.00198505138664455042e-6); const Packet cst_cephes_exp_q1 = pset1(2.52448340349684104192e-3); const Packet cst_cephes_exp_q2 = pset1(2.27265548208155028766e-1); const Packet cst_cephes_exp_q3 = pset1(2.00000000000000000009e0); const Packet cst_cephes_exp_C1 = pset1(0.693145751953125); const Packet cst_cephes_exp_C2 = pset1(1.42860682030941723212e-6); Packet tmp, fx; // clamp x x = pmax(pmin(x, cst_exp_hi), cst_exp_lo); // Express exp(x) as exp(g + n*log(2)). fx = pmadd(cst_cephes_LOG2EF, x, cst_half); // Get the integer modulus of log(2), i.e. the "n" described above. fx = pfloor(fx); // Get the remainder modulo log(2), i.e. the "g" described above. Subtract // n*log(2) out in two steps, i.e. n*C1 + n*C2, C1+C2=log2 to get the last // digits right. tmp = pmul(fx, cst_cephes_exp_C1); Packet z = pmul(fx, cst_cephes_exp_C2); x = psub(x, tmp); x = psub(x, z); Packet x2 = pmul(x, x); // Evaluate the numerator polynomial of the rational interpolant. Packet px = cst_cephes_exp_p0; px = pmadd(px, x2, cst_cephes_exp_p1); px = pmadd(px, x2, cst_cephes_exp_p2); px = pmul(px, x); // Evaluate the denominator polynomial of the rational interpolant. Packet qx = cst_cephes_exp_q0; qx = pmadd(qx, x2, cst_cephes_exp_q1); qx = pmadd(qx, x2, cst_cephes_exp_q2); qx = pmadd(qx, x2, cst_cephes_exp_q3); // I don't really get this bit, copied from the SSE2 routines, so... // TODO(gonnet): Figure out what is going on here, perhaps find a better // rational interpolant? x = pdiv(px, psub(qx, px)); x = pmadd(cst_2, x, cst_1); // Construct the result 2^n * exp(g) = e * x. The max is used to catch // non-finite values in the input. // TODO: replace pldexp with faster implementation since x in [-1, 1). return pmax(pldexp(x,fx), _x); } // The following code is inspired by the following stack-overflow answer: // https://stackoverflow.com/questions/30463616/payne-hanek-algorithm-implementation-in-c/30465751#30465751 // It has been largely optimized: // - By-pass calls to frexp. // - Aligned loads of required 96 bits of 2/pi. This is accomplished by // (1) balancing the mantissa and exponent to the required bits of 2/pi are // aligned on 8-bits, and (2) replicating the storage of the bits of 2/pi. // - Avoid a branch in rounding and extraction of the remaining fractional part. // Overall, I measured a speed up higher than x2 on x86-64. inline float trig_reduce_huge (float xf, int *quadrant) { using Eigen::numext::int32_t; using Eigen::numext::uint32_t; using Eigen::numext::int64_t; using Eigen::numext::uint64_t; const double pio2_62 = 3.4061215800865545e-19; // pi/2 * 2^-62 const uint64_t zero_dot_five = uint64_t(1) << 61; // 0.5 in 2.62-bit fixed-point foramt // 192 bits of 2/pi for Payne-Hanek reduction // Bits are introduced by packet of 8 to enable aligned reads. static const uint32_t two_over_pi [] = { 0x00000028, 0x000028be, 0x0028be60, 0x28be60db, 0xbe60db93, 0x60db9391, 0xdb939105, 0x9391054a, 0x91054a7f, 0x054a7f09, 0x4a7f09d5, 0x7f09d5f4, 0x09d5f47d, 0xd5f47d4d, 0xf47d4d37, 0x7d4d3770, 0x4d377036, 0x377036d8, 0x7036d8a5, 0x36d8a566, 0xd8a5664f, 0xa5664f10, 0x664f10e4, 0x4f10e410, 0x10e41000, 0xe4100000 }; uint32_t xi = numext::bit_cast(xf); // Below, -118 = -126 + 8. // -126 is to get the exponent, // +8 is to enable alignment of 2/pi's bits on 8 bits. // This is possible because the fractional part of x as only 24 meaningful bits. uint32_t e = (xi >> 23) - 118; // Extract the mantissa and shift it to align it wrt the exponent xi = ((xi & 0x007fffffu)| 0x00800000u) << (e & 0x7); uint32_t i = e >> 3; uint32_t twoopi_1 = two_over_pi[i-1]; uint32_t twoopi_2 = two_over_pi[i+3]; uint32_t twoopi_3 = two_over_pi[i+7]; // Compute x * 2/pi in 2.62-bit fixed-point format. uint64_t p; p = uint64_t(xi) * twoopi_3; p = uint64_t(xi) * twoopi_2 + (p >> 32); p = (uint64_t(xi * twoopi_1) << 32) + p; // Round to nearest: add 0.5 and extract integral part. uint64_t q = (p + zero_dot_five) >> 62; *quadrant = int(q); // Now it remains to compute "r = x - q*pi/2" with high accuracy, // since we have p=x/(pi/2) with high accuracy, we can more efficiently compute r as: // r = (p-q)*pi/2, // where the product can be be carried out with sufficient accuracy using double precision. p -= q<<62; return float(double(int64_t(p)) * pio2_62); } template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED #if EIGEN_GNUC_AT_LEAST(4,4) && EIGEN_COMP_GNUC_STRICT __attribute__((optimize("-fno-unsafe-math-optimizations"))) #endif Packet psincos_float(const Packet& _x) { typedef typename unpacket_traits::integer_packet PacketI; const Packet cst_2oPI = pset1(0.636619746685028076171875f); // 2/PI const Packet cst_rounding_magic = pset1(12582912); // 2^23 for rounding const PacketI csti_1 = pset1(1); const Packet cst_sign_mask = pset1frombits(0x80000000u); Packet x = pabs(_x); // Scale x by 2/Pi to find x's octant. Packet y = pmul(x, cst_2oPI); // Rounding trick: Packet y_round = padd(y, cst_rounding_magic); EIGEN_OPTIMIZATION_BARRIER(y_round) PacketI y_int = preinterpret(y_round); // last 23 digits represent integer (if abs(x)<2^24) y = psub(y_round, cst_rounding_magic); // nearest integer to x*4/pi // Reduce x by y octants to get: -Pi/4 <= x <= +Pi/4 // using "Extended precision modular arithmetic" #if defined(EIGEN_HAS_SINGLE_INSTRUCTION_MADD) // This version requires true FMA for high accuracy // It provides a max error of 1ULP up to (with absolute_error < 5.9605e-08): const float huge_th = ComputeSine ? 117435.992f : 71476.0625f; x = pmadd(y, pset1(-1.57079601287841796875f), x); x = pmadd(y, pset1(-3.1391647326017846353352069854736328125e-07f), x); x = pmadd(y, pset1(-5.390302529957764765544681040410068817436695098876953125e-15f), x); #else // Without true FMA, the previous set of coefficients maintain 1ULP accuracy // up to x<15.7 (for sin), but accuracy is immediately lost for x>15.7. // We thus use one more iteration to maintain 2ULPs up to reasonably large inputs. // The following set of coefficients maintain 1ULP up to 9.43 and 14.16 for sin and cos respectively. // and 2 ULP up to: const float huge_th = ComputeSine ? 25966.f : 18838.f; x = pmadd(y, pset1(-1.5703125), x); // = 0xbfc90000 EIGEN_OPTIMIZATION_BARRIER(x) x = pmadd(y, pset1(-0.000483989715576171875), x); // = 0xb9fdc000 EIGEN_OPTIMIZATION_BARRIER(x) x = pmadd(y, pset1(1.62865035235881805419921875e-07), x); // = 0x342ee000 x = pmadd(y, pset1(5.5644315544167710640977020375430583953857421875e-11), x); // = 0x2e74b9ee // For the record, the following set of coefficients maintain 2ULP up // to a slightly larger range: // const float huge_th = ComputeSine ? 51981.f : 39086.125f; // but it slightly fails to maintain 1ULP for two values of sin below pi. // x = pmadd(y, pset1(-3.140625/2.), x); // x = pmadd(y, pset1(-0.00048351287841796875), x); // x = pmadd(y, pset1(-3.13855707645416259765625e-07), x); // x = pmadd(y, pset1(-6.0771006282767103812147979624569416046142578125e-11), x); // For the record, with only 3 iterations it is possible to maintain // 1 ULP up to 3PI (maybe more) and 2ULP up to 255. // The coefficients are: 0xbfc90f80, 0xb7354480, 0x2e74b9ee #endif if(predux_any(pcmp_le(pset1(huge_th),pabs(_x)))) { const int PacketSize = unpacket_traits::size; EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) float vals[PacketSize]; EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) float x_cpy[PacketSize]; EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) int y_int2[PacketSize]; pstoreu(vals, pabs(_x)); pstoreu(x_cpy, x); pstoreu(y_int2, y_int); for(int k=0; k=huge_th && (numext::isfinite)(val)) x_cpy[k] = trig_reduce_huge(val,&y_int2[k]); } x = ploadu(x_cpy); y_int = ploadu(y_int2); } // Compute the sign to apply to the polynomial. // sin: sign = second_bit(y_int) xor signbit(_x) // cos: sign = second_bit(y_int+1) Packet sign_bit = ComputeSine ? pxor(_x, preinterpret(plogical_shift_left<30>(y_int))) : preinterpret(plogical_shift_left<30>(padd(y_int,csti_1))); sign_bit = pand(sign_bit, cst_sign_mask); // clear all but left most bit // Get the polynomial selection mask from the second bit of y_int // We'll calculate both (sin and cos) polynomials and then select from the two. Packet poly_mask = preinterpret(pcmp_eq(pand(y_int, csti_1), pzero(y_int))); Packet x2 = pmul(x,x); // Evaluate the cos(x) polynomial. (-Pi/4 <= x <= Pi/4) Packet y1 = pset1(2.4372266125283204019069671630859375e-05f); y1 = pmadd(y1, x2, pset1(-0.00138865201734006404876708984375f )); y1 = pmadd(y1, x2, pset1(0.041666619479656219482421875f )); y1 = pmadd(y1, x2, pset1(-0.5f)); y1 = pmadd(y1, x2, pset1(1.f)); // Evaluate the sin(x) polynomial. (Pi/4 <= x <= Pi/4) // octave/matlab code to compute those coefficients: // x = (0:0.0001:pi/4)'; // A = [x.^3 x.^5 x.^7]; // w = ((1.-(x/(pi/4)).^2).^5)*2000+1; # weights trading relative accuracy // c = (A'*diag(w)*A)\(A'*diag(w)*(sin(x)-x)); # weighted LS, linear coeff forced to 1 // printf('%.64f\n %.64f\n%.64f\n', c(3), c(2), c(1)) // Packet y2 = pset1(-0.0001959234114083702898469196984621021329076029360294342041015625f); y2 = pmadd(y2, x2, pset1( 0.0083326873655616851693794799871284340042620897293090820312500000f)); y2 = pmadd(y2, x2, pset1(-0.1666666203982298255503735617821803316473960876464843750000000000f)); y2 = pmul(y2, x2); y2 = pmadd(y2, x, x); // Select the correct result from the two polynomials. y = ComputeSine ? pselect(poly_mask,y2,y1) : pselect(poly_mask,y1,y2); // Update the sign and filter huge inputs return pxor(y, sign_bit); } template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet psin_float(const Packet& x) { return psincos_float(x); } template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet pcos_float(const Packet& x) { return psincos_float(x); } template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet psqrt_complex(const Packet& a) { typedef typename unpacket_traits::type Scalar; typedef typename Scalar::value_type RealScalar; typedef typename unpacket_traits::as_real RealPacket; // Computes the principal sqrt of the complex numbers in the input. // // For example, for packets containing 2 complex numbers stored in interleaved format // a = [a0, a1] = [x0, y0, x1, y1], // where x0 = real(a0), y0 = imag(a0) etc., this function returns // b = [b0, b1] = [u0, v0, u1, v1], // such that b0^2 = a0, b1^2 = a1. // // To derive the formula for the complex square roots, let's consider the equation for // a single complex square root of the number x + i*y. We want to find real numbers // u and v such that // (u + i*v)^2 = x + i*y <=> // u^2 - v^2 + i*2*u*v = x + i*v. // By equating the real and imaginary parts we get: // u^2 - v^2 = x // 2*u*v = y. // // For x >= 0, this has the numerically stable solution // u = sqrt(0.5 * (x + sqrt(x^2 + y^2))) // v = 0.5 * (y / u) // and for x < 0, // v = sign(y) * sqrt(0.5 * (-x + sqrt(x^2 + y^2))) // u = 0.5 * (y / v) // // To avoid unnecessary over- and underflow, we compute sqrt(x^2 + y^2) as // l = max(|x|, |y|) * sqrt(1 + (min(|x|, |y|) / max(|x|, |y|))^2) , // In the following, without lack of generality, we have annotated the code, assuming // that the input is a packet of 2 complex numbers. // // Step 1. Compute l = [l0, l0, l1, l1], where // l0 = sqrt(x0^2 + y0^2), l1 = sqrt(x1^2 + y1^2) // To avoid over- and underflow, we use the stable formula for each hypotenuse // l0 = (min0 == 0 ? max0 : max0 * sqrt(1 + (min0/max0)**2)), // where max0 = max(|x0|, |y0|), min0 = min(|x0|, |y0|), and similarly for l1. RealPacket a_abs = pabs(a.v); // [|x0|, |y0|, |x1|, |y1|] RealPacket a_abs_flip = pcplxflip(Packet(a_abs)).v; // [|y0|, |x0|, |y1|, |x1|] RealPacket a_max = pmax(a_abs, a_abs_flip); RealPacket a_min = pmin(a_abs, a_abs_flip); RealPacket a_min_zero_mask = pcmp_eq(a_min, pzero(a_min)); RealPacket a_max_zero_mask = pcmp_eq(a_max, pzero(a_max)); RealPacket r = pdiv(a_min, a_max); const RealPacket cst_one = pset1(RealScalar(1)); RealPacket l = pmul(a_max, psqrt(padd(cst_one, pmul(r, r)))); // [l0, l0, l1, l1] // Set l to a_max if a_min is zero. l = pselect(a_min_zero_mask, a_max, l); // Step 2. Compute [rho0, *, rho1, *], where // rho0 = sqrt(0.5 * (l0 + |x0|)), rho1 = sqrt(0.5 * (l1 + |x1|)) // We don't care about the imaginary parts computed here. They will be overwritten later. const RealPacket cst_half = pset1(RealScalar(0.5)); Packet rho; rho.v = psqrt(pmul(cst_half, padd(a_abs, l))); // Step 3. Compute [rho0, eta0, rho1, eta1], where // eta0 = (y0 / l0) / 2, and eta1 = (y1 / l1) / 2. // set eta = 0 of input is 0 + i0. RealPacket eta = pandnot(pmul(cst_half, pdiv(a.v, pcplxflip(rho).v)), a_max_zero_mask); RealPacket real_mask = peven_mask(a.v); Packet positive_real_result; // Compute result for inputs with positive real part. positive_real_result.v = pselect(real_mask, rho.v, eta); // Step 4. Compute solution for inputs with negative real part: // [|eta0|, sign(y0)*rho0, |eta1|, sign(y1)*rho1] const RealScalar neg_zero = RealScalar(numext::bit_cast(0x80000000u)); const RealPacket cst_imag_sign_mask = pset1(Scalar(RealScalar(0.0), neg_zero)).v; RealPacket imag_signs = pand(a.v, cst_imag_sign_mask); Packet negative_real_result; // Notice that rho is positive, so taking it's absolute value is a noop. negative_real_result.v = por(pabs(pcplxflip(positive_real_result).v), imag_signs); // Step 5. Select solution branch based on the sign of the real parts. Packet negative_real_mask; negative_real_mask.v = pcmp_lt(pand(real_mask, a.v), pzero(a.v)); negative_real_mask.v = por(negative_real_mask.v, pcplxflip(negative_real_mask).v); Packet result = pselect(negative_real_mask, negative_real_result, positive_real_result); // Step 6. Handle special cases for infinities: // * If z is (x,+∞), the result is (+∞,+∞) even if x is NaN // * If z is (x,-∞), the result is (+∞,-∞) even if x is NaN // * If z is (-∞,y), the result is (0*|y|,+∞) for finite or NaN y // * If z is (+∞,y), the result is (+∞,0*|y|) for finite or NaN y const RealPacket cst_pos_inf = pset1(NumTraits::infinity()); Packet is_inf; is_inf.v = pcmp_eq(a_abs, cst_pos_inf); Packet is_real_inf; is_real_inf.v = pand(is_inf.v, real_mask); is_real_inf = por(is_real_inf, pcplxflip(is_real_inf)); // prepare packet of (+∞,0*|y|) or (0*|y|,+∞), depending on the sign of the infinite real part. Packet real_inf_result; real_inf_result.v = pmul(a_abs, pset1(Scalar(RealScalar(1.0), RealScalar(0.0))).v); real_inf_result.v = pselect(negative_real_mask.v, pcplxflip(real_inf_result).v, real_inf_result.v); // prepare packet of (+∞,+∞) or (+∞,-∞), depending on the sign of the infinite imaginary part. Packet is_imag_inf; is_imag_inf.v = pandnot(is_inf.v, real_mask); is_imag_inf = por(is_imag_inf, pcplxflip(is_imag_inf)); Packet imag_inf_result; imag_inf_result.v = por(pand(cst_pos_inf, real_mask), pandnot(a.v, real_mask)); return pselect(is_imag_inf, imag_inf_result, pselect(is_real_inf, real_inf_result,result)); } // TODO(rmlarsen): The following set of utilities for double word arithmetic // should perhaps be refactored as a separate file, since it would be generally // useful for special function implementation etc. Writing the algorithms in // terms if a double word type would also make the code more readable. // This function splits x into the nearest integer n and fractional part r, // such that x = n + r holds exactly. template EIGEN_STRONG_INLINE void absolute_split(const Packet& x, Packet& n, Packet& r) { n = pround(x); r = psub(x, n); } // This function computes the sum {s, r}, such that x + y = s_hi + s_lo // holds exactly, and s_hi = fl(x+y), if |x| >= |y|. template EIGEN_STRONG_INLINE void fast_twosum(const Packet& x, const Packet& y, Packet& s_hi, Packet& s_lo) { s_hi = padd(x, y); const Packet t = psub(s_hi, x); s_lo = psub(y, t); } #ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD // This function implements the extended precision product of // a pair of floating point numbers. Given {x, y}, it computes the pair // {p_hi, p_lo} such that x * y = p_hi + p_lo holds exactly and // p_hi = fl(x * y). template EIGEN_STRONG_INLINE void twoprod(const Packet& x, const Packet& y, Packet& p_hi, Packet& p_lo) { p_hi = pmul(x, y); p_lo = pmadd(x, y, pnegate(p_hi)); } #else // This function implements the Veltkamp splitting. Given a floating point // number x it returns the pair {x_hi, x_lo} such that x_hi + x_lo = x holds // exactly and that half of the significant of x fits in x_hi. // This is Algorithm 3 from Jean-Michel Muller, "Elementary Functions", // 3rd edition, Birkh\"auser, 2016. template EIGEN_STRONG_INLINE void veltkamp_splitting(const Packet& x, Packet& x_hi, Packet& x_lo) { typedef typename unpacket_traits::type Scalar; EIGEN_CONSTEXPR int shift = (NumTraits::digits() + 1) / 2; const Scalar shift_scale = Scalar(uint64_t(1) << shift); // Scalar constructor not necessarily constexpr. const Packet gamma = pmul(pset1(shift_scale + Scalar(1)), x); Packet rho = psub(x, gamma); x_hi = padd(rho, gamma); x_lo = psub(x, x_hi); } // This function implements Dekker's algorithm for products x * y. // Given floating point numbers {x, y} computes the pair // {p_hi, p_lo} such that x * y = p_hi + p_lo holds exactly and // p_hi = fl(x * y). template EIGEN_STRONG_INLINE void twoprod(const Packet& x, const Packet& y, Packet& p_hi, Packet& p_lo) { Packet x_hi, x_lo, y_hi, y_lo; veltkamp_splitting(x, x_hi, x_lo); veltkamp_splitting(y, y_hi, y_lo); p_hi = pmul(x, y); p_lo = pmadd(x_hi, y_hi, pnegate(p_hi)); p_lo = pmadd(x_hi, y_lo, p_lo); p_lo = pmadd(x_lo, y_hi, p_lo); p_lo = pmadd(x_lo, y_lo, p_lo); } #endif // EIGEN_HAS_SINGLE_INSTRUCTION_MADD // This function implements Dekker's algorithm for the addition // of two double word numbers represented by {x_hi, x_lo} and {y_hi, y_lo}. // It returns the result as a pair {s_hi, s_lo} such that // x_hi + x_lo + y_hi + y_lo = s_hi + s_lo holds exactly. // This is Algorithm 5 from Jean-Michel Muller, "Elementary Functions", // 3rd edition, Birkh\"auser, 2016. template EIGEN_STRONG_INLINE void twosum(const Packet& x_hi, const Packet& x_lo, const Packet& y_hi, const Packet& y_lo, Packet& s_hi, Packet& s_lo) { const Packet x_greater_mask = pcmp_lt(pabs(y_hi), pabs(x_hi)); Packet r_hi_1, r_lo_1; fast_twosum(x_hi, y_hi,r_hi_1, r_lo_1); Packet r_hi_2, r_lo_2; fast_twosum(y_hi, x_hi,r_hi_2, r_lo_2); const Packet r_hi = pselect(x_greater_mask, r_hi_1, r_hi_2); const Packet s1 = padd(padd(y_lo, r_lo_1), x_lo); const Packet s2 = padd(padd(x_lo, r_lo_2), y_lo); const Packet s = pselect(x_greater_mask, s1, s2); fast_twosum(r_hi, s, s_hi, s_lo); } // This is a version of twosum for double word numbers, // which assumes that |x_hi| >= |y_hi|. template EIGEN_STRONG_INLINE void fast_twosum(const Packet& x_hi, const Packet& x_lo, const Packet& y_hi, const Packet& y_lo, Packet& s_hi, Packet& s_lo) { Packet r_hi, r_lo; fast_twosum(x_hi, y_hi, r_hi, r_lo); const Packet s = padd(padd(y_lo, r_lo), x_lo); fast_twosum(r_hi, s, s_hi, s_lo); } // This is a version of twosum for adding a floating point number x to // double word number {y_hi, y_lo} number, with the assumption // that |x| >= |y_hi|. template EIGEN_STRONG_INLINE void fast_twosum(const Packet& x, const Packet& y_hi, const Packet& y_lo, Packet& s_hi, Packet& s_lo) { Packet r_hi, r_lo; fast_twosum(x, y_hi, r_hi, r_lo); const Packet s = padd(y_lo, r_lo); fast_twosum(r_hi, s, s_hi, s_lo); } // This function implements the multiplication of a double word // number represented by {x_hi, x_lo} by a floating point number y. // It returns the result as a pair {p_hi, p_lo} such that // (x_hi + x_lo) * y = p_hi + p_lo hold with a relative error // of less than 2*2^{-2p}, where p is the number of significand bit // in the floating point type. // This is Algorithm 7 from Jean-Michel Muller, "Elementary Functions", // 3rd edition, Birkh\"auser, 2016. template EIGEN_STRONG_INLINE void twoprod(const Packet& x_hi, const Packet& x_lo, const Packet& y, Packet& p_hi, Packet& p_lo) { Packet c_hi, c_lo1; twoprod(x_hi, y, c_hi, c_lo1); const Packet c_lo2 = pmul(x_lo, y); Packet t_hi, t_lo1; fast_twosum(c_hi, c_lo2, t_hi, t_lo1); const Packet t_lo2 = padd(t_lo1, c_lo1); fast_twosum(t_hi, t_lo2, p_hi, p_lo); } // This function implements the multiplication of two double word // numbers represented by {x_hi, x_lo} and {y_hi, y_lo}. // It returns the result as a pair {p_hi, p_lo} such that // (x_hi + x_lo) * (y_hi + y_lo) = p_hi + p_lo holds with a relative error // of less than 2*2^{-2p}, where p is the number of significand bit // in the floating point type. template EIGEN_STRONG_INLINE void twoprod(const Packet& x_hi, const Packet& x_lo, const Packet& y_hi, const Packet& y_lo, Packet& p_hi, Packet& p_lo) { Packet p_hi_hi, p_hi_lo; twoprod(x_hi, x_lo, y_hi, p_hi_hi, p_hi_lo); Packet p_lo_hi, p_lo_lo; twoprod(x_hi, x_lo, y_lo, p_lo_hi, p_lo_lo); fast_twosum(p_hi_hi, p_hi_lo, p_lo_hi, p_lo_lo, p_hi, p_lo); } // This function computes the reciprocal of a floating point number // with extra precision and returns the result as a double word. template void doubleword_reciprocal(const Packet& x, Packet& recip_hi, Packet& recip_lo) { typedef typename unpacket_traits::type Scalar; // 1. Approximate the reciprocal as the reciprocal of the high order element. Packet approx_recip = prsqrt(x); approx_recip = pmul(approx_recip, approx_recip); // 2. Run one step of Newton-Raphson iteration in double word arithmetic // to get the bottom half. The NR iteration for reciprocal of 'a' is // x_{i+1} = x_i * (2 - a * x_i) // -a*x_i Packet t1_hi, t1_lo; twoprod(pnegate(x), approx_recip, t1_hi, t1_lo); // 2 - a*x_i Packet t2_hi, t2_lo; fast_twosum(pset1(Scalar(2)), t1_hi, t2_hi, t2_lo); Packet t3_hi, t3_lo; fast_twosum(t2_hi, padd(t2_lo, t1_lo), t3_hi, t3_lo); // x_i * (2 - a * x_i) twoprod(t3_hi, t3_lo, approx_recip, recip_hi, recip_lo); } // This function computes log2(x) and returns the result as a double word. template struct accurate_log2 { template EIGEN_STRONG_INLINE void operator()(const Packet& x, Packet& log2_x_hi, Packet& log2_x_lo) { log2_x_hi = plog2(x); log2_x_lo = pzero(x); } }; // This specialization uses a more accurate algorithm to compute log2(x) for // floats in [1/sqrt(2);sqrt(2)] with a relative accuracy of ~6.42e-10. // This additional accuracy is needed to counter the error-magnification // inherent in multiplying by a potentially large exponent in pow(x,y). // The minimax polynomial used was calculated using the Sollya tool. // See sollya.org. template <> struct accurate_log2 { template EIGEN_STRONG_INLINE void operator()(const Packet& z, Packet& log2_x_hi, Packet& log2_x_lo) { // The function log(1+x)/x is approximated in the interval // [1/sqrt(2)-1;sqrt(2)-1] by a degree 10 polynomial of the form // Q(x) = (C0 + x * (C1 + x * (C2 + x * (C3 + x * P(x))))), // where the degree 6 polynomial P(x) is evaluated in single precision, // while the remaining 4 terms of Q(x), as well as the final multiplication by x // to reconstruct log(1+x) are evaluated in extra precision using // double word arithmetic. C0 through C3 are extra precise constants // stored as double words. // // The polynomial coefficients were calculated using Sollya commands: // > n = 10; // > f = log2(1+x)/x; // > interval = [sqrt(0.5)-1;sqrt(2)-1]; // > p = fpminimax(f,n,[|double,double,double,double,single...|],interval,relative,floating); const Packet p6 = pset1( 9.703654795885e-2f); const Packet p5 = pset1(-0.1690667718648f); const Packet p4 = pset1( 0.1720575392246f); const Packet p3 = pset1(-0.1789081543684f); const Packet p2 = pset1( 0.2050433009862f); const Packet p1 = pset1(-0.2404672354459f); const Packet p0 = pset1( 0.2885761857032f); const Packet C3_hi = pset1(-0.360674142838f); const Packet C3_lo = pset1(-6.13283912543e-09f); const Packet C2_hi = pset1(0.480897903442f); const Packet C2_lo = pset1(-1.44861207474e-08f); const Packet C1_hi = pset1(-0.721347510815f); const Packet C1_lo = pset1(-4.84483164698e-09f); const Packet C0_hi = pset1(1.44269502163f); const Packet C0_lo = pset1(2.01711713999e-08f); const Packet one = pset1(1.0f); const Packet x = psub(z, one); // Evaluate P(x) in working precision. // We evaluate it in multiple parts to improve instruction level // parallelism. Packet x2 = pmul(x,x); Packet p_even = pmadd(p6, x2, p4); p_even = pmadd(p_even, x2, p2); p_even = pmadd(p_even, x2, p0); Packet p_odd = pmadd(p5, x2, p3); p_odd = pmadd(p_odd, x2, p1); Packet p = pmadd(p_odd, x, p_even); // Now evaluate the low-order tems of Q(x) in double word precision. // In the following, due to the alternating signs and the fact that // |x| < sqrt(2)-1, we can assume that |C*_hi| >= q_i, and use // fast_twosum instead of the slower twosum. Packet q_hi, q_lo; Packet t_hi, t_lo; // C3 + x * p(x) twoprod(p, x, t_hi, t_lo); fast_twosum(C3_hi, C3_lo, t_hi, t_lo, q_hi, q_lo); // C2 + x * p(x) twoprod(q_hi, q_lo, x, t_hi, t_lo); fast_twosum(C2_hi, C2_lo, t_hi, t_lo, q_hi, q_lo); // C1 + x * p(x) twoprod(q_hi, q_lo, x, t_hi, t_lo); fast_twosum(C1_hi, C1_lo, t_hi, t_lo, q_hi, q_lo); // C0 + x * p(x) twoprod(q_hi, q_lo, x, t_hi, t_lo); fast_twosum(C0_hi, C0_lo, t_hi, t_lo, q_hi, q_lo); // log(z) ~= x * Q(x) twoprod(q_hi, q_lo, x, log2_x_hi, log2_x_lo); } }; // This specialization uses a more accurate algorithm to compute log2(x) for // floats in [1/sqrt(2);sqrt(2)] with a relative accuracy of ~1.27e-18. // This additional accuracy is needed to counter the error-magnification // inherent in multiplying by a potentially large exponent in pow(x,y). // The minimax polynomial used was calculated using the Sollya tool. // See sollya.org. template <> struct accurate_log2 { template EIGEN_STRONG_INLINE void operator()(const Packet& x, Packet& log2_x_hi, Packet& log2_x_lo) { // We use a transformation of variables: // r = c * (x-1) / (x+1), // such that // log2(x) = log2((1 + r/c) / (1 - r/c)) = f(r). // The function f(r) can be approximated well using an odd polynomial // of the form // P(r) = ((Q(r^2) * r^2 + C) * r^2 + 1) * r, // For the implementation of log2 here, Q is of degree 6 with // coefficient represented in working precision (double), while C is a // constant represented in extra precision as a double word to achieve // full accuracy. // // The polynomial coefficients were computed by the Sollya script: // // c = 2 / log(2); // trans = c * (x-1)/(x+1); // itrans = (1+x/c)/(1-x/c); // interval=[trans(sqrt(0.5)); trans(sqrt(2))]; // print(interval); // f = log2(itrans(x)); // p=fpminimax(f,[|1,3,5,7,9,11,13,15,17|],[|1,DD,double...|],interval,relative,floating); const Packet q12 = pset1(2.87074255468000586e-9); const Packet q10 = pset1(2.38957980901884082e-8); const Packet q8 = pset1(2.31032094540014656e-7); const Packet q6 = pset1(2.27279857398537278e-6); const Packet q4 = pset1(2.31271023278625638e-5); const Packet q2 = pset1(2.47556738444535513e-4); const Packet q0 = pset1(2.88543873228900172e-3); const Packet C_hi = pset1(0.0400377511598501157); const Packet C_lo = pset1(-4.77726582251425391e-19); const Packet one = pset1(1.0); const Packet cst_2_log2e_hi = pset1(2.88539008177792677); const Packet cst_2_log2e_lo = pset1(4.07660016854549667e-17); // c * (x - 1) Packet num_hi, num_lo; twoprod(cst_2_log2e_hi, cst_2_log2e_lo, psub(x, one), num_hi, num_lo); // TODO(rmlarsen): Investigate if using the division algorithm by // Muller et al. is faster/more accurate. // 1 / (x + 1) Packet denom_hi, denom_lo; doubleword_reciprocal(padd(x, one), denom_hi, denom_lo); // r = c * (x-1) / (x+1), Packet r_hi, r_lo; twoprod(num_hi, num_lo, denom_hi, denom_lo, r_hi, r_lo); // r2 = r * r Packet r2_hi, r2_lo; twoprod(r_hi, r_lo, r_hi, r_lo, r2_hi, r2_lo); // r4 = r2 * r2 Packet r4_hi, r4_lo; twoprod(r2_hi, r2_lo, r2_hi, r2_lo, r4_hi, r4_lo); // Evaluate Q(r^2) in working precision. We evaluate it in two parts // (even and odd in r^2) to improve instruction level parallelism. Packet q_even = pmadd(q12, r4_hi, q8); Packet q_odd = pmadd(q10, r4_hi, q6); q_even = pmadd(q_even, r4_hi, q4); q_odd = pmadd(q_odd, r4_hi, q2); q_even = pmadd(q_even, r4_hi, q0); Packet q = pmadd(q_odd, r2_hi, q_even); // Now evaluate the low order terms of P(x) in double word precision. // In the following, due to the increasing magnitude of the coefficients // and r being constrained to [-0.5, 0.5] we can use fast_twosum instead // of the slower twosum. // Q(r^2) * r^2 Packet p_hi, p_lo; twoprod(r2_hi, r2_lo, q, p_hi, p_lo); // Q(r^2) * r^2 + C Packet p1_hi, p1_lo; fast_twosum(C_hi, C_lo, p_hi, p_lo, p1_hi, p1_lo); // (Q(r^2) * r^2 + C) * r^2 Packet p2_hi, p2_lo; twoprod(r2_hi, r2_lo, p1_hi, p1_lo, p2_hi, p2_lo); // ((Q(r^2) * r^2 + C) * r^2 + 1) Packet p3_hi, p3_lo; fast_twosum(one, p2_hi, p2_lo, p3_hi, p3_lo); // log(z) ~= ((Q(r^2) * r^2 + C) * r^2 + 1) * r twoprod(p3_hi, p3_lo, r_hi, r_lo, log2_x_hi, log2_x_lo); } }; // This function computes exp2(x) (i.e. 2**x). template struct fast_accurate_exp2 { template EIGEN_STRONG_INLINE Packet operator()(const Packet& x) { // TODO(rmlarsen): Add a pexp2 packetop. return pexp(pmul(pset1(Scalar(EIGEN_LN2)), x)); } }; // This specialization uses a faster algorithm to compute exp2(x) for floats // in [-0.5;0.5] with a relative accuracy of 1 ulp. // The minimax polynomial used was calculated using the Sollya tool. // See sollya.org. template <> struct fast_accurate_exp2 { template EIGEN_STRONG_INLINE Packet operator()(const Packet& x) { // This function approximates exp2(x) by a degree 6 polynomial of the form // Q(x) = 1 + x * (C + x * P(x)), where the degree 4 polynomial P(x) is evaluated in // single precision, and the remaining steps are evaluated with extra precision using // double word arithmetic. C is an extra precise constant stored as a double word. // // The polynomial coefficients were calculated using Sollya commands: // > n = 6; // > f = 2^x; // > interval = [-0.5;0.5]; // > p = fpminimax(f,n,[|1,double,single...|],interval,relative,floating); const Packet p4 = pset1(1.539513905e-4f); const Packet p3 = pset1(1.340007293e-3f); const Packet p2 = pset1(9.618283249e-3f); const Packet p1 = pset1(5.550328270e-2f); const Packet p0 = pset1(0.2402264923f); const Packet C_hi = pset1(0.6931471825f); const Packet C_lo = pset1(2.36836577e-08f); const Packet one = pset1(1.0f); // Evaluate P(x) in working precision. // We evaluate even and odd parts of the polynomial separately // to gain some instruction level parallelism. Packet x2 = pmul(x,x); Packet p_even = pmadd(p4, x2, p2); Packet p_odd = pmadd(p3, x2, p1); p_even = pmadd(p_even, x2, p0); Packet p = pmadd(p_odd, x, p_even); // Evaluate the remaining terms of Q(x) with extra precision using // double word arithmetic. Packet p_hi, p_lo; // x * p(x) twoprod(p, x, p_hi, p_lo); // C + x * p(x) Packet q1_hi, q1_lo; twosum(p_hi, p_lo, C_hi, C_lo, q1_hi, q1_lo); // x * (C + x * p(x)) Packet q2_hi, q2_lo; twoprod(q1_hi, q1_lo, x, q2_hi, q2_lo); // 1 + x * (C + x * p(x)) Packet q3_hi, q3_lo; // Since |q2_hi| <= sqrt(2)-1 < 1, we can use fast_twosum // for adding it to unity here. fast_twosum(one, q2_hi, q3_hi, q3_lo); return padd(q3_hi, padd(q2_lo, q3_lo)); } }; // in [-0.5;0.5] with a relative accuracy of 1 ulp. // The minimax polynomial used was calculated using the Sollya tool. // See sollya.org. template <> struct fast_accurate_exp2 { template EIGEN_STRONG_INLINE Packet operator()(const Packet& x) { // This function approximates exp2(x) by a degree 10 polynomial of the form // Q(x) = 1 + x * (C + x * P(x)), where the degree 8 polynomial P(x) is evaluated in // single precision, and the remaining steps are evaluated with extra precision using // double word arithmetic. C is an extra precise constant stored as a double word. // // The polynomial coefficients were calculated using Sollya commands: // > n = 11; // > f = 2^x; // > interval = [-0.5;0.5]; // > p = fpminimax(f,n,[|1,DD,double...|],interval,relative,floating); const Packet p9 = pset1(4.431642109085495276e-10); const Packet p8 = pset1(7.073829923303358410e-9); const Packet p7 = pset1(1.017822306737031311e-7); const Packet p6 = pset1(1.321543498017646657e-6); const Packet p5 = pset1(1.525273342728892877e-5); const Packet p4 = pset1(1.540353045780084423e-4); const Packet p3 = pset1(1.333355814685869807e-3); const Packet p2 = pset1(9.618129107593478832e-3); const Packet p1 = pset1(5.550410866481961247e-2); const Packet p0 = pset1(0.240226506959101332); const Packet C_hi = pset1(0.693147180559945286); const Packet C_lo = pset1(4.81927865669806721e-17); const Packet one = pset1(1.0); // Evaluate P(x) in working precision. // We evaluate even and odd parts of the polynomial separately // to gain some instruction level parallelism. Packet x2 = pmul(x,x); Packet p_even = pmadd(p8, x2, p6); Packet p_odd = pmadd(p9, x2, p7); p_even = pmadd(p_even, x2, p4); p_odd = pmadd(p_odd, x2, p5); p_even = pmadd(p_even, x2, p2); p_odd = pmadd(p_odd, x2, p3); p_even = pmadd(p_even, x2, p0); p_odd = pmadd(p_odd, x2, p1); Packet p = pmadd(p_odd, x, p_even); // Evaluate the remaining terms of Q(x) with extra precision using // double word arithmetic. Packet p_hi, p_lo; // x * p(x) twoprod(p, x, p_hi, p_lo); // C + x * p(x) Packet q1_hi, q1_lo; twosum(p_hi, p_lo, C_hi, C_lo, q1_hi, q1_lo); // x * (C + x * p(x)) Packet q2_hi, q2_lo; twoprod(q1_hi, q1_lo, x, q2_hi, q2_lo); // 1 + x * (C + x * p(x)) Packet q3_hi, q3_lo; // Since |q2_hi| <= sqrt(2)-1 < 1, we can use fast_twosum // for adding it to unity here. fast_twosum(one, q2_hi, q3_hi, q3_lo); return padd(q3_hi, padd(q2_lo, q3_lo)); } }; // This function implements the non-trivial case of pow(x,y) where x is // positive and y is (possibly) non-integer. // Formally, pow(x,y) = exp2(y * log2(x)), where exp2(x) is shorthand for 2^x. // TODO(rmlarsen): We should probably add this as a packet up 'ppow', to make it // easier to specialize or turn off for specific types and/or backends.x template EIGEN_STRONG_INLINE Packet generic_pow_impl(const Packet& x, const Packet& y) { typedef typename unpacket_traits::type Scalar; // Split x into exponent e_x and mantissa m_x. Packet e_x; Packet m_x = pfrexp(x, e_x); // Adjust m_x to lie in [1/sqrt(2):sqrt(2)] to minimize absolute error in log2(m_x). EIGEN_CONSTEXPR Scalar sqrt_half = Scalar(0.70710678118654752440); const Packet m_x_scale_mask = pcmp_lt(m_x, pset1(sqrt_half)); m_x = pselect(m_x_scale_mask, pmul(pset1(Scalar(2)), m_x), m_x); e_x = pselect(m_x_scale_mask, psub(e_x, pset1(Scalar(1))), e_x); // Compute log2(m_x) with 6 extra bits of accuracy. Packet rx_hi, rx_lo; accurate_log2()(m_x, rx_hi, rx_lo); // Compute the two terms {y * e_x, y * r_x} in f = y * log2(x) with doubled // precision using double word arithmetic. Packet f1_hi, f1_lo, f2_hi, f2_lo; twoprod(e_x, y, f1_hi, f1_lo); twoprod(rx_hi, rx_lo, y, f2_hi, f2_lo); // Sum the two terms in f using double word arithmetic. We know // that |e_x| > |log2(m_x)|, except for the case where e_x==0. // This means that we can use fast_twosum(f1,f2). // In the case e_x == 0, e_x * y = f1 = 0, so we don't lose any // accuracy by violating the assumption of fast_twosum, because // it's a no-op. Packet f_hi, f_lo; fast_twosum(f1_hi, f1_lo, f2_hi, f2_lo, f_hi, f_lo); // Split f into integer and fractional parts. Packet n_z, r_z; absolute_split(f_hi, n_z, r_z); r_z = padd(r_z, f_lo); Packet n_r; absolute_split(r_z, n_r, r_z); n_z = padd(n_z, n_r); // We now have an accurate split of f = n_z + r_z and can compute // x^y = 2**{n_z + r_z) = exp2(r_z) * 2**{n_z}. // Since r_z is in [-0.5;0.5], we compute the first factor to high accuracy // using a specialized algorithm. Multiplication by the second factor can // be done exactly using pldexp(), since it is an integer power of 2. const Packet e_r = fast_accurate_exp2()(r_z); return pldexp(e_r, n_z); } // Generic implementation of pow(x,y). template EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet generic_pow(const Packet& x, const Packet& y) { typedef typename unpacket_traits::type Scalar; const Packet cst_pos_inf = pset1(NumTraits::infinity()); const Packet cst_zero = pset1(Scalar(0)); const Packet cst_one = pset1(Scalar(1)); const Packet cst_nan = pset1(NumTraits::quiet_NaN()); const Packet abs_x = pabs(x); // Predicates for sign and magnitude of x. const Packet x_is_zero = pcmp_eq(x, cst_zero); const Packet x_is_neg = pcmp_lt(x, cst_zero); const Packet abs_x_is_inf = pcmp_eq(abs_x, cst_pos_inf); const Packet abs_x_is_one = pcmp_eq(abs_x, cst_one); const Packet abs_x_is_gt_one = pcmp_lt(cst_one, abs_x); const Packet abs_x_is_lt_one = pcmp_lt(abs_x, cst_one); const Packet x_is_one = pandnot(abs_x_is_one, x_is_neg); const Packet x_is_neg_one = pand(abs_x_is_one, x_is_neg); const Packet x_is_nan = pandnot(ptrue(x), pcmp_eq(x, x)); // Predicates for sign and magnitude of y. const Packet y_is_one = pcmp_eq(y, cst_one); const Packet y_is_zero = pcmp_eq(y, cst_zero); const Packet y_is_neg = pcmp_lt(y, cst_zero); const Packet y_is_pos = pandnot(ptrue(y), por(y_is_zero, y_is_neg)); const Packet y_is_nan = pandnot(ptrue(y), pcmp_eq(y, y)); const Packet abs_y_is_inf = pcmp_eq(pabs(y), cst_pos_inf); EIGEN_CONSTEXPR Scalar huge_exponent = (NumTraits::max_exponent() * Scalar(EIGEN_LN2)) / NumTraits::epsilon(); const Packet abs_y_is_huge = pcmp_le(pset1(huge_exponent), pabs(y)); // Predicates for whether y is integer and/or even. const Packet y_is_int = pcmp_eq(pfloor(y), y); const Packet y_div_2 = pmul(y, pset1(Scalar(0.5))); const Packet y_is_even = pcmp_eq(pround(y_div_2), y_div_2); // Predicates encoding special cases for the value of pow(x,y) const Packet invalid_negative_x = pandnot(pandnot(pandnot(x_is_neg, abs_x_is_inf), y_is_int), abs_y_is_inf); const Packet pow_is_one = por(por(x_is_one, y_is_zero), pand(x_is_neg_one, por(abs_y_is_inf, pandnot(y_is_even, invalid_negative_x)))); const Packet pow_is_nan = por(invalid_negative_x, por(x_is_nan, y_is_nan)); const Packet pow_is_zero = por(por(por(pand(x_is_zero, y_is_pos), pand(abs_x_is_inf, y_is_neg)), pand(pand(abs_x_is_lt_one, abs_y_is_huge), y_is_pos)), pand(pand(abs_x_is_gt_one, abs_y_is_huge), y_is_neg)); const Packet pow_is_inf = por(por(por(pand(x_is_zero, y_is_neg), pand(abs_x_is_inf, y_is_pos)), pand(pand(abs_x_is_lt_one, abs_y_is_huge), y_is_neg)), pand(pand(abs_x_is_gt_one, abs_y_is_huge), y_is_pos)); // General computation of pow(x,y) for positive x or negative x and integer y. const Packet negate_pow_abs = pandnot(x_is_neg, y_is_even); const Packet pow_abs = generic_pow_impl(abs_x, y); return pselect(y_is_one, x, pselect(pow_is_one, cst_one, pselect(pow_is_nan, cst_nan, pselect(pow_is_inf, cst_pos_inf, pselect(pow_is_zero, cst_zero, pselect(negate_pow_abs, pnegate(pow_abs), pow_abs)))))); } /* polevl (modified for Eigen) * * Evaluate polynomial * * * * SYNOPSIS: * * int N; * Scalar x, y, coef[N+1]; * * y = polevl( x, coef); * * * * DESCRIPTION: * * Evaluates polynomial of degree N: * * 2 N * y = C + C x + C x +...+ C x * 0 1 2 N * * Coefficients are stored in reverse order: * * coef[0] = C , ..., coef[N] = C . * N 0 * * The function p1evl() assumes that coef[N] = 1.0 and is * omitted from the array. Its calling arguments are * otherwise the same as polevl(). * * * The Eigen implementation is templatized. For best speed, store * coef as a const array (constexpr), e.g. * * const double coef[] = {1.0, 2.0, 3.0, ...}; * */ template struct ppolevl { static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& x, const typename unpacket_traits::type coeff[]) { EIGEN_STATIC_ASSERT((N > 0), YOU_MADE_A_PROGRAMMING_MISTAKE); return pmadd(ppolevl::run(x, coeff), x, pset1(coeff[N])); } }; template struct ppolevl { static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& x, const typename unpacket_traits::type coeff[]) { EIGEN_UNUSED_VARIABLE(x); return pset1(coeff[0]); } }; /* chbevl (modified for Eigen) * * Evaluate Chebyshev series * * * * SYNOPSIS: * * int N; * Scalar x, y, coef[N], chebevl(); * * y = chbevl( x, coef, N ); * * * * DESCRIPTION: * * Evaluates the series * * N-1 * - ' * y = > coef[i] T (x/2) * - i * i=0 * * of Chebyshev polynomials Ti at argument x/2. * * Coefficients are stored in reverse order, i.e. the zero * order term is last in the array. Note N is the number of * coefficients, not the order. * * If coefficients are for the interval a to b, x must * have been transformed to x -> 2(2x - b - a)/(b-a) before * entering the routine. This maps x from (a, b) to (-1, 1), * over which the Chebyshev polynomials are defined. * * If the coefficients are for the inverted interval, in * which (a, b) is mapped to (1/b, 1/a), the transformation * required is x -> 2(2ab/x - b - a)/(b-a). If b is infinity, * this becomes x -> 4a/x - 1. * * * * SPEED: * * Taking advantage of the recurrence properties of the * Chebyshev polynomials, the routine requires one more * addition per loop than evaluating a nested polynomial of * the same degree. * */ template struct pchebevl { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Packet run(Packet x, const typename unpacket_traits::type coef[]) { typedef typename unpacket_traits::type Scalar; Packet b0 = pset1(coef[0]); Packet b1 = pset1(static_cast(0.f)); Packet b2; for (int i = 1; i < N; i++) { b2 = b1; b1 = b0; b0 = psub(pmadd(x, b1, pset1(coef[i])), b2); } return pmul(pset1(static_cast(0.5f)), psub(b0, b2)); } }; } // end namespace internal } // end namespace Eigen #endif // EIGEN_ARCH_GENERIC_PACKET_MATH_FUNCTIONS_H RcppEigen/inst/include/Eigen/src/Core/arch/HIP/0000755000176200001440000000000014567757725020622 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/arch/HIP/hcc/0000755000176200001440000000000014567757725021357 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/arch/HIP/hcc/math_constants.h0000644000176200001440000000126314567757725024557 0ustar liggesusers/* * math_constants.h - * HIP equivalent of the CUDA header of the same name */ #ifndef __MATH_CONSTANTS_H__ #define __MATH_CONSTANTS_H__ /* single precision constants */ #define HIPRT_INF_F __int_as_float(0x7f800000) #define HIPRT_NAN_F __int_as_float(0x7fffffff) #define HIPRT_MIN_DENORM_F __int_as_float(0x00000001) #define HIPRT_MAX_NORMAL_F __int_as_float(0x7f7fffff) #define HIPRT_NEG_ZERO_F __int_as_float(0x80000000) #define HIPRT_ZERO_F 0.0f #define HIPRT_ONE_F 1.0f /* double precision constants */ #define HIPRT_INF __hiloint2double(0x7ff00000, 0x00000000) #define HIPRT_NAN __hiloint2double(0xfff80000, 0x00000000) #endif RcppEigen/inst/include/Eigen/src/Core/arch/NEON/0000755000176200001440000000000014567757725020741 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/arch/NEON/TypeCasting.h0000644000176200001440000014412614567757725023354 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2018 Rasmus Munk Larsen // Copyright (C) 2020 Antonio Sanchez // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_TYPE_CASTING_NEON_H #define EIGEN_TYPE_CASTING_NEON_H namespace Eigen { namespace internal { //============================================================================== // pcast, SrcType = float //============================================================================== template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4f pcast(const Packet4f& a) { return a; } template <> EIGEN_STRONG_INLINE Packet2f pcast(const Packet2f& a) { return a; } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; // If float64 exists, first convert to that to keep as much precision as possible. #if EIGEN_ARCH_ARM64 template <> EIGEN_STRONG_INLINE Packet2l pcast(const Packet4f& a) { // Discard second half of input. return vcvtq_s64_f64(vcvt_f64_f32(vget_low_f32(a))); } template <> EIGEN_STRONG_INLINE Packet2ul pcast(const Packet4f& a) { // Discard second half of input. return vcvtq_u64_f64(vcvt_f64_f32(vget_low_f32(a))); } #else template <> EIGEN_STRONG_INLINE Packet2l pcast(const Packet4f& a) { // Discard second half of input. return vmovl_s32(vget_low_s32(vcvtq_s32_f32(a))); } template <> EIGEN_STRONG_INLINE Packet2ul pcast(const Packet4f& a) { // Discard second half of input. return vmovl_u32(vget_low_u32(vcvtq_u32_f32(a))); } #endif // EIGEN_ARCH_ARM64 template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4i pcast(const Packet4f& a) { return vcvtq_s32_f32(a); } template <> EIGEN_STRONG_INLINE Packet2i pcast(const Packet2f& a) { return vcvt_s32_f32(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet4f& a) { return vcvtq_u32_f32(a); } template <> EIGEN_STRONG_INLINE Packet2ui pcast(const Packet2f& a) { return vcvt_u32_f32(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet8s pcast(const Packet4f& a, const Packet4f& b) { return vcombine_s16(vmovn_s32(vcvtq_s32_f32(a)), vmovn_s32(vcvtq_s32_f32(b))); } template <> EIGEN_STRONG_INLINE Packet4s pcast(const Packet2f& a, const Packet2f& b) { return vmovn_s32(vcombine_s32(vcvt_s32_f32(a), vcvt_s32_f32(b))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet8us pcast(const Packet4f& a, const Packet4f& b) { return vcombine_u16(vmovn_u32(vcvtq_u32_f32(a)), vmovn_u32(vcvtq_u32_f32(b))); } template <> EIGEN_STRONG_INLINE Packet4us pcast(const Packet2f& a, const Packet2f& b) { return vmovn_u32(vcombine_u32(vcvt_u32_f32(a), vcvt_u32_f32(b))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16c pcast(const Packet4f& a, const Packet4f& b, const Packet4f& c, const Packet4f& d) { const int16x8_t ab_s16 = pcast(a, b); const int16x8_t cd_s16 = pcast(c, d); return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16)); } template <> EIGEN_STRONG_INLINE Packet8c pcast(const Packet2f& a, const Packet2f& b, const Packet2f& c, const Packet2f& d) { const int16x4_t ab_s16 = pcast(a, b); const int16x4_t cd_s16 = pcast(c, d); return vmovn_s16(vcombine_s16(ab_s16, cd_s16)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16uc pcast(const Packet4f& a, const Packet4f& b, const Packet4f& c, const Packet4f& d) { const uint16x8_t ab_u16 = pcast(a, b); const uint16x8_t cd_u16 = pcast(c, d); return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16)); } template <> EIGEN_STRONG_INLINE Packet8uc pcast(const Packet2f& a, const Packet2f& b, const Packet2f& c, const Packet2f& d) { const uint16x4_t ab_u16 = pcast(a, b); const uint16x4_t cd_u16 = pcast(c, d); return vmovn_u16(vcombine_u16(ab_u16, cd_u16)); } //============================================================================== // pcast, SrcType = int8_t //============================================================================== template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 }; }; template <> EIGEN_STRONG_INLINE Packet4f pcast(const Packet16c& a) { // Discard all but first 4 bytes. return vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a))))); } template <> EIGEN_STRONG_INLINE Packet2f pcast(const Packet8c& a) { // Discard all but first 2 bytes. return vcvt_f32_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a))))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 }; }; template <> EIGEN_STRONG_INLINE Packet2l pcast(const Packet16c& a) { // Discard all but first two bytes. return vmovl_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a)))))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 }; }; template <> EIGEN_STRONG_INLINE Packet2ul pcast(const Packet16c& a) { return vreinterpretq_u64_s64(pcast(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 }; }; template <> EIGEN_STRONG_INLINE Packet4i pcast(const Packet16c& a) { // Discard all but first 4 bytes. return vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a)))); } template <> EIGEN_STRONG_INLINE Packet2i pcast(const Packet8c& a) { // Discard all but first 2 bytes. return vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a)))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 }; }; template <> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet16c& a) { return vreinterpretq_u32_s32(pcast(a)); } template <> EIGEN_STRONG_INLINE Packet2ui pcast(const Packet8c& a) { return vreinterpret_u32_s32(pcast(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet8s pcast(const Packet16c& a) { // Discard second half of input. return vmovl_s8(vget_low_s8(a)); } template <> EIGEN_STRONG_INLINE Packet4s pcast(const Packet8c& a) { // Discard second half of input. return vget_low_s16(vmovl_s8(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet8us pcast(const Packet16c& a) { return vreinterpretq_u16_s16(pcast(a)); } template <> EIGEN_STRONG_INLINE Packet4us pcast(const Packet8c& a) { return vreinterpret_u16_s16(pcast(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16c pcast(const Packet16c& a) { return a; } template <> EIGEN_STRONG_INLINE Packet8c pcast(const Packet8c& a) { return a; } template <> EIGEN_STRONG_INLINE Packet4c pcast(const Packet4c& a) { return a; } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16uc pcast(const Packet16c& a) { return vreinterpretq_u8_s8(a); } template <> EIGEN_STRONG_INLINE Packet8uc pcast(const Packet8c& a) { return vreinterpret_u8_s8(a); } template <> EIGEN_STRONG_INLINE Packet4uc pcast(const Packet4c& a) { return static_cast(a); } //============================================================================== // pcast, SrcType = uint8_t //============================================================================== template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 }; }; template <> EIGEN_STRONG_INLINE Packet4f pcast(const Packet16uc& a) { // Discard all but first 4 bytes. return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))))); } template <> EIGEN_STRONG_INLINE Packet2f pcast(const Packet8uc& a) { // Discard all but first 2 bytes. return vcvt_f32_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a))))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 }; }; template <> EIGEN_STRONG_INLINE Packet2ul pcast(const Packet16uc& a) { // Discard all but first two bytes. return vmovl_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a)))))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 }; }; template <> EIGEN_STRONG_INLINE Packet2l pcast(const Packet16uc& a) { return vreinterpretq_s64_u64(pcast(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 }; }; template <> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet16uc& a) { // Discard all but first 4 bytes. return vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a)))); } template <> EIGEN_STRONG_INLINE Packet2ui pcast(const Packet8uc& a) { // Discard all but first 2 bytes. return vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a)))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 }; }; template <> EIGEN_STRONG_INLINE Packet4i pcast(const Packet16uc& a) { return vreinterpretq_s32_u32(pcast(a)); } template <> EIGEN_STRONG_INLINE Packet2i pcast(const Packet8uc& a) { return vreinterpret_s32_u32(pcast(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet8us pcast(const Packet16uc& a) { // Discard second half of input. return vmovl_u8(vget_low_u8(a)); } template <> EIGEN_STRONG_INLINE Packet4us pcast(const Packet8uc& a) { // Discard second half of input. return vget_low_u16(vmovl_u8(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet8s pcast(const Packet16uc& a) { return vreinterpretq_s16_u16(pcast(a)); } template <> EIGEN_STRONG_INLINE Packet4s pcast(const Packet8uc& a) { return vreinterpret_s16_u16(pcast(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16uc pcast(const Packet16uc& a) { return a; } template <> EIGEN_STRONG_INLINE Packet8uc pcast(const Packet8uc& a) { return a; } template <> EIGEN_STRONG_INLINE Packet4uc pcast(const Packet4uc& a) { return a; } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16c pcast(const Packet16uc& a) { return vreinterpretq_s8_u8(a); } template <> EIGEN_STRONG_INLINE Packet8c pcast(const Packet8uc& a) { return vreinterpret_s8_u8(a); } template <> EIGEN_STRONG_INLINE Packet4c pcast(const Packet4uc& a) { return static_cast(a); } //============================================================================== // pcast, SrcType = int16_t //============================================================================== template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet4f pcast(const Packet8s& a) { // Discard second half of input. return vcvtq_f32_s32(vmovl_s16(vget_low_s16(a))); } template <> EIGEN_STRONG_INLINE Packet2f pcast(const Packet4s& a) { // Discard second half of input. return vcvt_f32_s32(vget_low_s32(vmovl_s16(a))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 }; }; template <> EIGEN_STRONG_INLINE Packet2l pcast(const Packet8s& a) { // Discard all but first two values. return vmovl_s32(vget_low_s32(vmovl_s16(vget_low_s16(a)))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 }; }; template <> EIGEN_STRONG_INLINE Packet2ul pcast(const Packet8s& a) { return vreinterpretq_u64_s64(pcast(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet4i pcast(const Packet8s& a) { // Discard second half of input. return vmovl_s16(vget_low_s16(a)); } template <> EIGEN_STRONG_INLINE Packet2i pcast(const Packet4s& a) { // Discard second half of input. return vget_low_s32(vmovl_s16(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet8s& a) { return vreinterpretq_u32_s32(pcast(a)); } template <> EIGEN_STRONG_INLINE Packet2ui pcast(const Packet4s& a) { return vreinterpret_u32_s32(pcast(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet8s pcast(const Packet8s& a) { return a; } template <> EIGEN_STRONG_INLINE Packet4s pcast(const Packet4s& a) { return a; } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet8us pcast(const Packet8s& a) { return vreinterpretq_u16_s16(a); } template <> EIGEN_STRONG_INLINE Packet4us pcast(const Packet4s& a) { return vreinterpret_u16_s16(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16c pcast(const Packet8s& a, const Packet8s& b) { return vcombine_s8(vmovn_s16(a), vmovn_s16(b)); } template <> EIGEN_STRONG_INLINE Packet8c pcast(const Packet4s& a, const Packet4s& b) { return vmovn_s16(vcombine_s16(a, b)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16uc pcast(const Packet8s& a, const Packet8s& b) { return vcombine_u8(vmovn_u16(vreinterpretq_u16_s16(a)), vmovn_u16(vreinterpretq_u16_s16(b))); } template <> EIGEN_STRONG_INLINE Packet8uc pcast(const Packet4s& a, const Packet4s& b) { return vmovn_u16(vcombine_u16(vreinterpret_u16_s16(a), vreinterpret_u16_s16(b))); } //============================================================================== // pcast, SrcType = uint16_t //============================================================================== template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet4f pcast(const Packet8us& a) { // Discard second half of input. return vcvtq_f32_u32(vmovl_u16(vget_low_u16(a))); } template <> EIGEN_STRONG_INLINE Packet2f pcast(const Packet4us& a) { // Discard second half of input. return vcvt_f32_u32(vget_low_u32(vmovl_u16(a))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 }; }; template <> EIGEN_STRONG_INLINE Packet2ul pcast(const Packet8us& a) { // Discard all but first two values. return vmovl_u32(vget_low_u32(vmovl_u16(vget_low_u16(a)))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 }; }; template <> EIGEN_STRONG_INLINE Packet2l pcast(const Packet8us& a) { return vreinterpretq_s64_u64(pcast(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet8us& a) { // Discard second half of input. return vmovl_u16(vget_low_u16(a)); } template <> EIGEN_STRONG_INLINE Packet2ui pcast(const Packet4us& a) { // Discard second half of input. return vget_low_u32(vmovl_u16(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet4i pcast(const Packet8us& a) { return vreinterpretq_s32_u32(pcast(a)); } template <> EIGEN_STRONG_INLINE Packet2i pcast(const Packet4us& a) { return vreinterpret_s32_u32(pcast(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet8us pcast(const Packet8us& a) { return a; } template <> EIGEN_STRONG_INLINE Packet4us pcast(const Packet4us& a) { return a; } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet8s pcast(const Packet8us& a) { return vreinterpretq_s16_u16(a); } template <> EIGEN_STRONG_INLINE Packet4s pcast(const Packet4us& a) { return vreinterpret_s16_u16(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16uc pcast(const Packet8us& a, const Packet8us& b) { return vcombine_u8(vmovn_u16(a), vmovn_u16(b)); } template <> EIGEN_STRONG_INLINE Packet8uc pcast(const Packet4us& a, const Packet4us& b) { return vmovn_u16(vcombine_u16(a, b)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16c pcast(const Packet8us& a, const Packet8us& b) { return vreinterpretq_s8_u8(pcast(a, b)); } template <> EIGEN_STRONG_INLINE Packet8c pcast(const Packet4us& a, const Packet4us& b) { return vreinterpret_s8_u8(pcast(a, b)); } //============================================================================== // pcast, SrcType = int32_t //============================================================================== template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4f pcast(const Packet4i& a) { return vcvtq_f32_s32(a); } template <> EIGEN_STRONG_INLINE Packet2f pcast(const Packet2i& a) { return vcvt_f32_s32(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet2l pcast(const Packet4i& a) { // Discard second half of input. return vmovl_s32(vget_low_s32(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet2ul pcast(const Packet4i& a) { return vreinterpretq_u64_s64(pcast(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4i pcast(const Packet4i& a) { return a; } template <> EIGEN_STRONG_INLINE Packet2i pcast(const Packet2i& a) { return a; } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet4i& a) { return vreinterpretq_u32_s32(a); } template <> EIGEN_STRONG_INLINE Packet2ui pcast(const Packet2i& a) { return vreinterpret_u32_s32(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet8s pcast(const Packet4i& a, const Packet4i& b) { return vcombine_s16(vmovn_s32(a), vmovn_s32(b)); } template <> EIGEN_STRONG_INLINE Packet4s pcast(const Packet2i& a, const Packet2i& b) { return vmovn_s32(vcombine_s32(a, b)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet8us pcast(const Packet4i& a, const Packet4i& b) { return vcombine_u16(vmovn_u32(vreinterpretq_u32_s32(a)), vmovn_u32(vreinterpretq_u32_s32(b))); } template <> EIGEN_STRONG_INLINE Packet4us pcast(const Packet2i& a, const Packet2i& b) { return vmovn_u32(vreinterpretq_u32_s32(vcombine_s32(a, b))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16c pcast(const Packet4i& a, const Packet4i& b, const Packet4i& c, const Packet4i& d) { const int16x8_t ab_s16 = pcast(a, b); const int16x8_t cd_s16 = pcast(c, d); return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16)); } template <> EIGEN_STRONG_INLINE Packet8c pcast(const Packet2i& a, const Packet2i& b, const Packet2i& c, const Packet2i& d) { const int16x4_t ab_s16 = vmovn_s32(vcombine_s32(a, b)); const int16x4_t cd_s16 = vmovn_s32(vcombine_s32(c, d)); return vmovn_s16(vcombine_s16(ab_s16, cd_s16)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16uc pcast(const Packet4i& a, const Packet4i& b, const Packet4i& c, const Packet4i& d) { const uint16x8_t ab_u16 = pcast(a, b); const uint16x8_t cd_u16 = pcast(c, d); return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16)); } template <> EIGEN_STRONG_INLINE Packet8uc pcast(const Packet2i& a, const Packet2i& b, const Packet2i& c, const Packet2i& d) { const uint16x4_t ab_u16 = pcast(a, b); const uint16x4_t cd_u16 = pcast(c, d); return vmovn_u16(vcombine_u16(ab_u16, cd_u16)); } //============================================================================== // pcast, SrcType = uint32_t //============================================================================== template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4f pcast(const Packet4ui& a) { return vcvtq_f32_u32(a); } template <> EIGEN_STRONG_INLINE Packet2f pcast(const Packet2ui& a) { return vcvt_f32_u32(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet2ul pcast(const Packet4ui& a) { // Discard second half of input. return vmovl_u32(vget_low_u32(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet2l pcast(const Packet4ui& a) { return vreinterpretq_s64_u64(pcast(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet4ui& a) { return a; } template <> EIGEN_STRONG_INLINE Packet2ui pcast(const Packet2ui& a) { return a; } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4i pcast(const Packet4ui& a) { return vreinterpretq_s32_u32(a); } template <> EIGEN_STRONG_INLINE Packet2i pcast(const Packet2ui& a) { return vreinterpret_s32_u32(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet8us pcast(const Packet4ui& a, const Packet4ui& b) { return vcombine_u16(vmovn_u32(a), vmovn_u32(b)); } template <> EIGEN_STRONG_INLINE Packet4us pcast(const Packet2ui& a, const Packet2ui& b) { return vmovn_u32(vcombine_u32(a, b)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet8s pcast(const Packet4ui& a, const Packet4ui& b) { return vreinterpretq_s16_u16(pcast(a, b)); } template <> EIGEN_STRONG_INLINE Packet4s pcast(const Packet2ui& a, const Packet2ui& b) { return vreinterpret_s16_u16(pcast(a, b)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16uc pcast(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c, const Packet4ui& d) { const uint16x8_t ab_u16 = vcombine_u16(vmovn_u32(a), vmovn_u32(b)); const uint16x8_t cd_u16 = vcombine_u16(vmovn_u32(c), vmovn_u32(d)); return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16)); } template <> EIGEN_STRONG_INLINE Packet8uc pcast(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c, const Packet2ui& d) { const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(a, b)); const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(c, d)); return vmovn_u16(vcombine_u16(ab_u16, cd_u16)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16c pcast(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c, const Packet4ui& d) { return vreinterpretq_s8_u8(pcast(a, b, c, d)); } template <> EIGEN_STRONG_INLINE Packet8c pcast(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c, const Packet2ui& d) { return vreinterpret_s8_u8(pcast(a, b, c, d)); } //============================================================================== // pcast, SrcType = int64_t //============================================================================== template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4f pcast(const Packet2l& a, const Packet2l& b) { return vcvtq_f32_s32(vcombine_s32(vmovn_s64(a), vmovn_s64(b))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet2l pcast(const Packet2l& a) { return a; } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet2ul pcast(const Packet2l& a) { return vreinterpretq_u64_s64(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4i pcast(const Packet2l& a, const Packet2l& b) { return vcombine_s32(vmovn_s64(a), vmovn_s64(b)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet2l& a, const Packet2l& b) { return vcombine_u32(vmovn_u64(vreinterpretq_u64_s64(a)), vmovn_u64(vreinterpretq_u64_s64(b))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet8s pcast(const Packet2l& a, const Packet2l& b, const Packet2l& c, const Packet2l& d) { const int32x4_t ab_s32 = pcast(a, b); const int32x4_t cd_s32 = pcast(c, d); return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet8us pcast(const Packet2l& a, const Packet2l& b, const Packet2l& c, const Packet2l& d) { const uint32x4_t ab_u32 = pcast(a, b); const uint32x4_t cd_u32 = pcast(c, d); return vcombine_u16(vmovn_u32(ab_u32), vmovn_u32(cd_u32)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16c pcast(const Packet2l& a, const Packet2l& b, const Packet2l& c, const Packet2l& d, const Packet2l& e, const Packet2l& f, const Packet2l& g, const Packet2l& h) { const int16x8_t abcd_s16 = pcast(a, b, c, d); const int16x8_t efgh_s16 = pcast(e, f, g, h); return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16uc pcast(const Packet2l& a, const Packet2l& b, const Packet2l& c, const Packet2l& d, const Packet2l& e, const Packet2l& f, const Packet2l& g, const Packet2l& h) { const uint16x8_t abcd_u16 = pcast(a, b, c, d); const uint16x8_t efgh_u16 = pcast(e, f, g, h); return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16)); } //============================================================================== // pcast, SrcType = uint64_t //============================================================================== template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4f pcast(const Packet2ul& a, const Packet2ul& b) { return vcvtq_f32_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet2ul pcast(const Packet2ul& a) { return a; } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet2l pcast(const Packet2ul& a) { return vreinterpretq_s64_u64(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet2ul& a, const Packet2ul& b) { return vcombine_u32(vmovn_u64(a), vmovn_u64(b)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4i pcast(const Packet2ul& a, const Packet2ul& b) { return vreinterpretq_s32_u32(pcast(a, b)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet8us pcast(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c, const Packet2ul& d) { const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b))); const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(vmovn_u64(c), vmovn_u64(d))); return vcombine_u16(ab_u16, cd_u16); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet8s pcast(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c, const Packet2ul& d) { return vreinterpretq_s16_u16(pcast(a, b, c, d)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16uc pcast(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c, const Packet2ul& d, const Packet2ul& e, const Packet2ul& f, const Packet2ul& g, const Packet2ul& h) { const uint16x8_t abcd_u16 = pcast(a, b, c, d); const uint16x8_t efgh_u16 = pcast(e, f, g, h); return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16c pcast(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c, const Packet2ul& d, const Packet2ul& e, const Packet2ul& f, const Packet2ul& g, const Packet2ul& h) { return vreinterpretq_s8_u8(pcast(a, b, c, d, e, f, g, h)); } //============================================================================== // preinterpret //============================================================================== template <> EIGEN_STRONG_INLINE Packet2f preinterpret(const Packet2i& a) { return vreinterpret_f32_s32(a); } template <> EIGEN_STRONG_INLINE Packet2f preinterpret(const Packet2ui& a) { return vreinterpret_f32_u32(a); } template <> EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet4i& a) { return vreinterpretq_f32_s32(a); } template <> EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet4ui& a) { return vreinterpretq_f32_u32(a); } template <> EIGEN_STRONG_INLINE Packet4c preinterpret(const Packet4uc& a) { return static_cast(a); } template <> EIGEN_STRONG_INLINE Packet8c preinterpret(const Packet8uc& a) { return vreinterpret_s8_u8(a); } template <> EIGEN_STRONG_INLINE Packet16c preinterpret(const Packet16uc& a) { return vreinterpretq_s8_u8(a); } template <> EIGEN_STRONG_INLINE Packet4uc preinterpret(const Packet4c& a) { return static_cast(a); } template <> EIGEN_STRONG_INLINE Packet8uc preinterpret(const Packet8c& a) { return vreinterpret_u8_s8(a); } template <> EIGEN_STRONG_INLINE Packet16uc preinterpret(const Packet16c& a) { return vreinterpretq_u8_s8(a); } template <> EIGEN_STRONG_INLINE Packet4s preinterpret(const Packet4us& a) { return vreinterpret_s16_u16(a); } template <> EIGEN_STRONG_INLINE Packet8s preinterpret(const Packet8us& a) { return vreinterpretq_s16_u16(a); } template <> EIGEN_STRONG_INLINE Packet4us preinterpret(const Packet4s& a) { return vreinterpret_u16_s16(a); } template <> EIGEN_STRONG_INLINE Packet8us preinterpret(const Packet8s& a) { return vreinterpretq_u16_s16(a); } template <> EIGEN_STRONG_INLINE Packet2i preinterpret(const Packet2f& a) { return vreinterpret_s32_f32(a); } template <> EIGEN_STRONG_INLINE Packet2i preinterpret(const Packet2ui& a) { return vreinterpret_s32_u32(a); } template <> EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet4f& a) { return vreinterpretq_s32_f32(a); } template <> EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet4ui& a) { return vreinterpretq_s32_u32(a); } template <> EIGEN_STRONG_INLINE Packet2ui preinterpret(const Packet2f& a) { return vreinterpret_u32_f32(a); } template <> EIGEN_STRONG_INLINE Packet2ui preinterpret(const Packet2i& a) { return vreinterpret_u32_s32(a); } template <> EIGEN_STRONG_INLINE Packet4ui preinterpret(const Packet4f& a) { return vreinterpretq_u32_f32(a); } template <> EIGEN_STRONG_INLINE Packet4ui preinterpret(const Packet4i& a) { return vreinterpretq_u32_s32(a); } template <> EIGEN_STRONG_INLINE Packet2l preinterpret(const Packet2ul& a) { return vreinterpretq_s64_u64(a); } template <> EIGEN_STRONG_INLINE Packet2ul preinterpret(const Packet2l& a) { return vreinterpretq_u64_s64(a); } #if EIGEN_ARCH_ARM64 //============================================================================== // pcast/preinterpret, Double //============================================================================== template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet2d pcast(const Packet2d& a) { return a; } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4f pcast(const Packet2d& a, const Packet2d& b) { return vcombine_f32(vcvt_f32_f64(a), vcvt_f32_f64(b)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet2l pcast(const Packet2d& a) { return vcvtq_s64_f64(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet2ul pcast(const Packet2d& a) { return vcvtq_u64_f64(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4i pcast(const Packet2d& a, const Packet2d& b) { return vcombine_s32(vmovn_s64(vcvtq_s64_f64(a)), vmovn_s64(vcvtq_s64_f64(b))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet4ui pcast(const Packet2d& a, const Packet2d& b) { return vcombine_u32(vmovn_u64(vcvtq_u64_f64(a)), vmovn_u64(vcvtq_u64_f64(b))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet8s pcast(const Packet2d& a, const Packet2d& b, const Packet2d& c, const Packet2d& d) { const int32x4_t ab_s32 = pcast(a, b); const int32x4_t cd_s32 = pcast(c, d); return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet8us pcast(const Packet2d& a, const Packet2d& b, const Packet2d& c, const Packet2d& d) { const uint32x4_t ab_u32 = pcast(a, b); const uint32x4_t cd_u32 = pcast(c, d); return vcombine_u16(vmovn_u32(ab_u32), vmovn_u32(cd_u32)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16c pcast(const Packet2d& a, const Packet2d& b, const Packet2d& c, const Packet2d& d, const Packet2d& e, const Packet2d& f, const Packet2d& g, const Packet2d& h) { const int16x8_t abcd_s16 = pcast(a, b, c, d); const int16x8_t efgh_s16 = pcast(e, f, g, h); return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet16uc pcast(const Packet2d& a, const Packet2d& b, const Packet2d& c, const Packet2d& d, const Packet2d& e, const Packet2d& f, const Packet2d& g, const Packet2d& h) { const uint16x8_t abcd_u16 = pcast(a, b, c, d); const uint16x8_t efgh_u16 = pcast(e, f, g, h); return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet2d pcast(const Packet4f& a) { // Discard second-half of input. return vcvt_f64_f32(vget_low_f32(a)); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 }; }; template <> EIGEN_STRONG_INLINE Packet2d pcast(const Packet16c& a) { // Discard all but first two values. return vcvt_f64_f32(pcast(vget_low_s8(a))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 }; }; template <> EIGEN_STRONG_INLINE Packet2d pcast(const Packet16uc& a) { // Discard all but first two values. return vcvt_f64_f32(pcast(vget_low_u8(a))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 }; }; template <> EIGEN_STRONG_INLINE Packet2d pcast(const Packet8s& a) { // Discard all but first two values. return vcvt_f64_f32(pcast(vget_low_s16(a))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 }; }; template <> EIGEN_STRONG_INLINE Packet2d pcast(const Packet8us& a) { // Discard all but first two values. return vcvt_f64_f32(pcast(vget_low_u16(a))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet2d pcast(const Packet4i& a) { // Discard second half of input. return vcvtq_f64_s64(vmovl_s32(vget_low_s32(a))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template <> EIGEN_STRONG_INLINE Packet2d pcast(const Packet4ui& a) { // Discard second half of input. return vcvtq_f64_u64(vmovl_u32(vget_low_u32(a))); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet2d pcast(const Packet2l& a) { return vcvtq_f64_s64(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE Packet2d pcast(const Packet2ul& a) { return vcvtq_f64_u64(a); } template <> EIGEN_STRONG_INLINE Packet2d preinterpret(const Packet2l& a) { return vreinterpretq_f64_s64(a); } template <> EIGEN_STRONG_INLINE Packet2d preinterpret(const Packet2ul& a) { return vreinterpretq_f64_u64(a); } template <> EIGEN_STRONG_INLINE Packet2l preinterpret(const Packet2d& a) { return vreinterpretq_s64_f64(a); } template <> EIGEN_STRONG_INLINE Packet2ul preinterpret(const Packet2d& a) { return vreinterpretq_u64_f64(a); } template <> EIGEN_STRONG_INLINE Packet2d preinterpret(const Packet4i& a) { return vreinterpretq_f64_s32(a); } template <> EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet2d& a) { return vreinterpretq_s32_f64(a); } #endif // EIGEN_ARCH_ARM64 } // end namespace internal } // end namespace Eigen #endif // EIGEN_TYPE_CASTING_NEON_H RcppEigen/inst/include/Eigen/src/Core/arch/NEON/GeneralBlockPanelKernel.h0000644000176200001440000001523714567757725025573 0ustar liggesusersnamespace Eigen { namespace internal { #if EIGEN_ARCH_ARM && EIGEN_COMP_CLANG // Clang seems to excessively spill registers in the GEBP kernel on 32-bit arm. // Here we specialize gebp_traits to eliminate these register spills. // See #2138. template<> struct gebp_traits : gebp_traits { EIGEN_STRONG_INLINE void acc(const AccPacket& c, const ResPacket& alpha, ResPacket& r) const { // This volatile inline ASM both acts as a barrier to prevent reordering, // as well as enforces strict register use. asm volatile( "vmla.f32 %q[r], %q[c], %q[alpha]" : [r] "+w" (r) : [c] "w" (c), [alpha] "w" (alpha) : ); } template EIGEN_STRONG_INLINE void madd(const Packet4f& a, const Packet4f& b, Packet4f& c, Packet4f& tmp, const LaneIdType&) const { acc(a, b, c); } template EIGEN_STRONG_INLINE void madd(const Packet4f& a, const QuadPacket& b, Packet4f& c, Packet4f& tmp, const LaneIdType& lane) const { madd(a, b.get(lane), c, tmp, lane); } }; #endif // EIGEN_ARCH_ARM && EIGEN_COMP_CLANG #if EIGEN_ARCH_ARM64 template<> struct gebp_traits : gebp_traits { typedef float RhsPacket; typedef float32x4_t RhsPacketx4; EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const { dest = *b; } EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const { dest = vld1q_f32(b); } EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacket& dest) const { dest = *b; } EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {} EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { loadRhs(b,dest); } EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<0>&) const { c = vfmaq_n_f32(c, a, b); } // NOTE: Template parameter inference failed when compiled with Android NDK: // "candidate template ignored: could not match 'FixedInt' against 'Eigen::internal::FixedInt<0>". EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<0>&) const { madd_helper<0>(a, b, c); } EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<1>&) const { madd_helper<1>(a, b, c); } EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<2>&) const { madd_helper<2>(a, b, c); } EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<3>&) const { madd_helper<3>(a, b, c); } private: template EIGEN_STRONG_INLINE void madd_helper(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c) const { #if EIGEN_COMP_GNUC_STRICT && !(EIGEN_GNUC_AT_LEAST(9,0)) // workaround gcc issue https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89101 // vfmaq_laneq_f32 is implemented through a costly dup if(LaneID==0) asm("fmla %0.4s, %1.4s, %2.s[0]\n" : "+w" (c) : "w" (a), "w" (b) : ); else if(LaneID==1) asm("fmla %0.4s, %1.4s, %2.s[1]\n" : "+w" (c) : "w" (a), "w" (b) : ); else if(LaneID==2) asm("fmla %0.4s, %1.4s, %2.s[2]\n" : "+w" (c) : "w" (a), "w" (b) : ); else if(LaneID==3) asm("fmla %0.4s, %1.4s, %2.s[3]\n" : "+w" (c) : "w" (a), "w" (b) : ); #else c = vfmaq_laneq_f32(c, a, b, LaneID); #endif } }; template<> struct gebp_traits : gebp_traits { typedef double RhsPacket; struct RhsPacketx4 { float64x2_t B_0, B_1; }; EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const { dest = *b; } EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const { dest.B_0 = vld1q_f64(b); dest.B_1 = vld1q_f64(b+2); } EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacket& dest) const { loadRhs(b,dest); } EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {} EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { loadRhs(b,dest); } EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<0>&) const { c = vfmaq_n_f64(c, a, b); } // NOTE: Template parameter inference failed when compiled with Android NDK: // "candidate template ignored: could not match 'FixedInt' against 'Eigen::internal::FixedInt<0>". EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<0>&) const { madd_helper<0>(a, b, c); } EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<1>&) const { madd_helper<1>(a, b, c); } EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<2>&) const { madd_helper<2>(a, b, c); } EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<3>&) const { madd_helper<3>(a, b, c); } private: template EIGEN_STRONG_INLINE void madd_helper(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c) const { #if EIGEN_COMP_GNUC_STRICT && !(EIGEN_GNUC_AT_LEAST(9,0)) // workaround gcc issue https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89101 // vfmaq_laneq_f64 is implemented through a costly dup if(LaneID==0) asm("fmla %0.2d, %1.2d, %2.d[0]\n" : "+w" (c) : "w" (a), "w" (b.B_0) : ); else if(LaneID==1) asm("fmla %0.2d, %1.2d, %2.d[1]\n" : "+w" (c) : "w" (a), "w" (b.B_0) : ); else if(LaneID==2) asm("fmla %0.2d, %1.2d, %2.d[0]\n" : "+w" (c) : "w" (a), "w" (b.B_1) : ); else if(LaneID==3) asm("fmla %0.2d, %1.2d, %2.d[1]\n" : "+w" (c) : "w" (a), "w" (b.B_1) : ); #else if(LaneID==0) c = vfmaq_laneq_f64(c, a, b.B_0, 0); else if(LaneID==1) c = vfmaq_laneq_f64(c, a, b.B_0, 1); else if(LaneID==2) c = vfmaq_laneq_f64(c, a, b.B_1, 0); else if(LaneID==3) c = vfmaq_laneq_f64(c, a, b.B_1, 1); #endif } }; #endif // EIGEN_ARCH_ARM64 } // namespace internal } // namespace Eigen RcppEigen/inst/include/Eigen/src/Core/arch/NEON/Complex.h0000644000176200001440000005374714567757725022541 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2010 Gael Guennebaud // Copyright (C) 2010 Konstantinos Margaritis // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_COMPLEX_NEON_H #define EIGEN_COMPLEX_NEON_H namespace Eigen { namespace internal { inline uint32x4_t p4ui_CONJ_XOR() { // See bug 1325, clang fails to call vld1q_u64. #if EIGEN_COMP_CLANG || EIGEN_COMP_CASTXML uint32x4_t ret = { 0x00000000, 0x80000000, 0x00000000, 0x80000000 }; return ret; #else static const uint32_t conj_XOR_DATA[] = { 0x00000000, 0x80000000, 0x00000000, 0x80000000 }; return vld1q_u32( conj_XOR_DATA ); #endif } inline uint32x2_t p2ui_CONJ_XOR() { static const uint32_t conj_XOR_DATA[] = { 0x00000000, 0x80000000 }; return vld1_u32( conj_XOR_DATA ); } //---------- float ---------- struct Packet1cf { EIGEN_STRONG_INLINE Packet1cf() {} EIGEN_STRONG_INLINE explicit Packet1cf(const Packet2f& a) : v(a) {} Packet2f v; }; struct Packet2cf { EIGEN_STRONG_INLINE Packet2cf() {} EIGEN_STRONG_INLINE explicit Packet2cf(const Packet4f& a) : v(a) {} Packet4f v; }; template<> struct packet_traits > : default_packet_traits { typedef Packet2cf type; typedef Packet1cf half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 2, HasHalfPacket = 1, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, HasMax = 0, HasSetLinear = 0 }; }; template<> struct unpacket_traits { typedef std::complex type; typedef Packet1cf half; typedef Packet2f as_real; enum { size = 1, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef std::complex type; typedef Packet1cf half; typedef Packet4f as_real; enum { size = 2, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> EIGEN_STRONG_INLINE Packet1cf pcast(const float& a) { return Packet1cf(vset_lane_f32(a, vdup_n_f32(0.f), 0)); } template<> EIGEN_STRONG_INLINE Packet2cf pcast(const Packet2f& a) { return Packet2cf(vreinterpretq_f32_u64(vmovl_u32(vreinterpret_u32_f32(a)))); } template<> EIGEN_STRONG_INLINE Packet1cf pset1(const std::complex& from) { return Packet1cf(vld1_f32(reinterpret_cast(&from))); } template<> EIGEN_STRONG_INLINE Packet2cf pset1(const std::complex& from) { const float32x2_t r64 = vld1_f32(reinterpret_cast(&from)); return Packet2cf(vcombine_f32(r64, r64)); } template<> EIGEN_STRONG_INLINE Packet1cf padd(const Packet1cf& a, const Packet1cf& b) { return Packet1cf(padd(a.v, b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf padd(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(padd(a.v, b.v)); } template<> EIGEN_STRONG_INLINE Packet1cf psub(const Packet1cf& a, const Packet1cf& b) { return Packet1cf(psub(a.v, b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf psub(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(psub(a.v, b.v)); } template<> EIGEN_STRONG_INLINE Packet1cf pnegate(const Packet1cf& a) { return Packet1cf(pnegate(a.v)); } template<> EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf& a) { return Packet2cf(pnegate(a.v)); } template<> EIGEN_STRONG_INLINE Packet1cf pconj(const Packet1cf& a) { const Packet2ui b = vreinterpret_u32_f32(a.v); return Packet1cf(vreinterpret_f32_u32(veor_u32(b, p2ui_CONJ_XOR()))); } template<> EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) { const Packet4ui b = vreinterpretq_u32_f32(a.v); return Packet2cf(vreinterpretq_f32_u32(veorq_u32(b, p4ui_CONJ_XOR()))); } template<> EIGEN_STRONG_INLINE Packet1cf pmul(const Packet1cf& a, const Packet1cf& b) { Packet2f v1, v2; // Get the real values of a | a1_re | a1_re | v1 = vdup_lane_f32(a.v, 0); // Get the imag values of a | a1_im | a1_im | v2 = vdup_lane_f32(a.v, 1); // Multiply the real a with b v1 = vmul_f32(v1, b.v); // Multiply the imag a with b v2 = vmul_f32(v2, b.v); // Conjugate v2 v2 = vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(v2), p2ui_CONJ_XOR())); // Swap real/imag elements in v2. v2 = vrev64_f32(v2); // Add and return the result return Packet1cf(vadd_f32(v1, v2)); } template<> EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) { Packet4f v1, v2; // Get the real values of a | a1_re | a1_re | a2_re | a2_re | v1 = vcombine_f32(vdup_lane_f32(vget_low_f32(a.v), 0), vdup_lane_f32(vget_high_f32(a.v), 0)); // Get the imag values of a | a1_im | a1_im | a2_im | a2_im | v2 = vcombine_f32(vdup_lane_f32(vget_low_f32(a.v), 1), vdup_lane_f32(vget_high_f32(a.v), 1)); // Multiply the real a with b v1 = vmulq_f32(v1, b.v); // Multiply the imag a with b v2 = vmulq_f32(v2, b.v); // Conjugate v2 v2 = vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(v2), p4ui_CONJ_XOR())); // Swap real/imag elements in v2. v2 = vrev64q_f32(v2); // Add and return the result return Packet2cf(vaddq_f32(v1, v2)); } template<> EIGEN_STRONG_INLINE Packet1cf pcmp_eq(const Packet1cf& a, const Packet1cf& b) { // Compare real and imaginary parts of a and b to get the mask vector: // [re(a[0])==re(b[0]), im(a[0])==im(b[0])] Packet2f eq = pcmp_eq(a.v, b.v); // Swap real/imag elements in the mask in to get: // [im(a[0])==im(b[0]), re(a[0])==re(b[0])] Packet2f eq_swapped = vrev64_f32(eq); // Return re(a)==re(b) && im(a)==im(b) by computing bitwise AND of eq and eq_swapped return Packet1cf(pand(eq, eq_swapped)); } template<> EIGEN_STRONG_INLINE Packet2cf pcmp_eq(const Packet2cf& a, const Packet2cf& b) { // Compare real and imaginary parts of a and b to get the mask vector: // [re(a[0])==re(b[0]), im(a[0])==im(b[0]), re(a[1])==re(b[1]), im(a[1])==im(b[1])] Packet4f eq = pcmp_eq(a.v, b.v); // Swap real/imag elements in the mask in to get: // [im(a[0])==im(b[0]), re(a[0])==re(b[0]), im(a[1])==im(b[1]), re(a[1])==re(b[1])] Packet4f eq_swapped = vrev64q_f32(eq); // Return re(a)==re(b) && im(a)==im(b) by computing bitwise AND of eq and eq_swapped return Packet2cf(pand(eq, eq_swapped)); } template<> EIGEN_STRONG_INLINE Packet1cf pand(const Packet1cf& a, const Packet1cf& b) { return Packet1cf(vreinterpret_f32_u32(vand_u32(vreinterpret_u32_f32(a.v), vreinterpret_u32_f32(b.v)))); } template<> EIGEN_STRONG_INLINE Packet2cf pand(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(a.v), vreinterpretq_u32_f32(b.v)))); } template<> EIGEN_STRONG_INLINE Packet1cf por(const Packet1cf& a, const Packet1cf& b) { return Packet1cf(vreinterpret_f32_u32(vorr_u32(vreinterpret_u32_f32(a.v), vreinterpret_u32_f32(b.v)))); } template<> EIGEN_STRONG_INLINE Packet2cf por(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(vreinterpretq_f32_u32(vorrq_u32(vreinterpretq_u32_f32(a.v), vreinterpretq_u32_f32(b.v)))); } template<> EIGEN_STRONG_INLINE Packet1cf pxor(const Packet1cf& a, const Packet1cf& b) { return Packet1cf(vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(a.v), vreinterpret_u32_f32(b.v)))); } template<> EIGEN_STRONG_INLINE Packet2cf pxor(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(a.v), vreinterpretq_u32_f32(b.v)))); } template<> EIGEN_STRONG_INLINE Packet1cf pandnot(const Packet1cf& a, const Packet1cf& b) { return Packet1cf(vreinterpret_f32_u32(vbic_u32(vreinterpret_u32_f32(a.v), vreinterpret_u32_f32(b.v)))); } template<> EIGEN_STRONG_INLINE Packet2cf pandnot(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(vreinterpretq_f32_u32(vbicq_u32(vreinterpretq_u32_f32(a.v), vreinterpretq_u32_f32(b.v)))); } template<> EIGEN_STRONG_INLINE Packet1cf pload(const std::complex* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet1cf(pload((const float*)from)); } template<> EIGEN_STRONG_INLINE Packet2cf pload(const std::complex* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet2cf(pload(reinterpret_cast(from))); } template<> EIGEN_STRONG_INLINE Packet1cf ploadu(const std::complex* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet1cf(ploadu((const float*)from)); } template<> EIGEN_STRONG_INLINE Packet2cf ploadu(const std::complex* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cf(ploadu(reinterpret_cast(from))); } template<> EIGEN_STRONG_INLINE Packet1cf ploaddup(const std::complex* from) { return pset1(*from); } template<> EIGEN_STRONG_INLINE Packet2cf ploaddup(const std::complex* from) { return pset1(*from); } template<> EIGEN_STRONG_INLINE void pstore >(std::complex *to, const Packet1cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((float*)to, from.v); } template<> EIGEN_STRONG_INLINE void pstore >(std::complex *to, const Packet2cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore(reinterpret_cast(to), from.v); } template<> EIGEN_STRONG_INLINE void pstoreu >(std::complex *to, const Packet1cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((float*)to, from.v); } template<> EIGEN_STRONG_INLINE void pstoreu >(std::complex *to, const Packet2cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(reinterpret_cast(to), from.v); } template<> EIGEN_DEVICE_FUNC inline Packet1cf pgather, Packet1cf>( const std::complex* from, Index stride) { const Packet2f tmp = vdup_n_f32(std::real(from[0*stride])); return Packet1cf(vset_lane_f32(std::imag(from[0*stride]), tmp, 1)); } template<> EIGEN_DEVICE_FUNC inline Packet2cf pgather, Packet2cf>( const std::complex* from, Index stride) { Packet4f res = vdupq_n_f32(std::real(from[0*stride])); res = vsetq_lane_f32(std::imag(from[0*stride]), res, 1); res = vsetq_lane_f32(std::real(from[1*stride]), res, 2); res = vsetq_lane_f32(std::imag(from[1*stride]), res, 3); return Packet2cf(res); } template<> EIGEN_DEVICE_FUNC inline void pscatter, Packet1cf>( std::complex* to, const Packet1cf& from, Index stride) { to[stride*0] = std::complex(vget_lane_f32(from.v, 0), vget_lane_f32(from.v, 1)); } template<> EIGEN_DEVICE_FUNC inline void pscatter, Packet2cf>( std::complex* to, const Packet2cf& from, Index stride) { to[stride*0] = std::complex(vgetq_lane_f32(from.v, 0), vgetq_lane_f32(from.v, 1)); to[stride*1] = std::complex(vgetq_lane_f32(from.v, 2), vgetq_lane_f32(from.v, 3)); } template<> EIGEN_STRONG_INLINE void prefetch >(const std::complex *addr) { EIGEN_ARM_PREFETCH(reinterpret_cast(addr)); } template<> EIGEN_STRONG_INLINE std::complex pfirst(const Packet1cf& a) { EIGEN_ALIGN16 std::complex x; vst1_f32(reinterpret_cast(&x), a.v); return x; } template<> EIGEN_STRONG_INLINE std::complex pfirst(const Packet2cf& a) { EIGEN_ALIGN16 std::complex x[2]; vst1q_f32(reinterpret_cast(x), a.v); return x[0]; } template<> EIGEN_STRONG_INLINE Packet1cf preverse(const Packet1cf& a) { return a; } template<> EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) { return Packet2cf(vcombine_f32(vget_high_f32(a.v), vget_low_f32(a.v))); } template<> EIGEN_STRONG_INLINE Packet1cf pcplxflip(const Packet1cf& a) { return Packet1cf(vrev64_f32(a.v)); } template<> EIGEN_STRONG_INLINE Packet2cf pcplxflip(const Packet2cf& a) { return Packet2cf(vrev64q_f32(a.v)); } template<> EIGEN_STRONG_INLINE std::complex predux(const Packet1cf& a) { std::complex s; vst1_f32((float *)&s, a.v); return s; } template<> EIGEN_STRONG_INLINE std::complex predux(const Packet2cf& a) { std::complex s; vst1_f32(reinterpret_cast(&s), vadd_f32(vget_low_f32(a.v), vget_high_f32(a.v))); return s; } template<> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet1cf& a) { std::complex s; vst1_f32((float *)&s, a.v); return s; } template<> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet2cf& a) { float32x2_t a1, a2, v1, v2, prod; std::complex s; a1 = vget_low_f32(a.v); a2 = vget_high_f32(a.v); // Get the real values of a | a1_re | a1_re | a2_re | a2_re | v1 = vdup_lane_f32(a1, 0); // Get the real values of a | a1_im | a1_im | a2_im | a2_im | v2 = vdup_lane_f32(a1, 1); // Multiply the real a with b v1 = vmul_f32(v1, a2); // Multiply the imag a with b v2 = vmul_f32(v2, a2); // Conjugate v2 v2 = vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(v2), p2ui_CONJ_XOR())); // Swap real/imag elements in v2. v2 = vrev64_f32(v2); // Add v1, v2 prod = vadd_f32(v1, v2); vst1_f32(reinterpret_cast(&s), prod); return s; } EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cf,Packet2f) EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf,Packet4f) template<> EIGEN_STRONG_INLINE Packet1cf pdiv(const Packet1cf& a, const Packet1cf& b) { // TODO optimize it for NEON Packet1cf res = pmul(a, pconj(b)); Packet2f s, rev_s; // this computes the norm s = vmul_f32(b.v, b.v); rev_s = vrev64_f32(s); return Packet1cf(pdiv(res.v, vadd_f32(s, rev_s))); } template<> EIGEN_STRONG_INLINE Packet2cf pdiv(const Packet2cf& a, const Packet2cf& b) { // TODO optimize it for NEON Packet2cf res = pmul(a,pconj(b)); Packet4f s, rev_s; // this computes the norm s = vmulq_f32(b.v, b.v); rev_s = vrev64q_f32(s); return Packet2cf(pdiv(res.v, vaddq_f32(s, rev_s))); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& /*kernel*/) {} EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { Packet4f tmp = vcombine_f32(vget_high_f32(kernel.packet[0].v), vget_high_f32(kernel.packet[1].v)); kernel.packet[0].v = vcombine_f32(vget_low_f32(kernel.packet[0].v), vget_low_f32(kernel.packet[1].v)); kernel.packet[1].v = tmp; } template<> EIGEN_STRONG_INLINE Packet1cf psqrt(const Packet1cf& a) { return psqrt_complex(a); } template<> EIGEN_STRONG_INLINE Packet2cf psqrt(const Packet2cf& a) { return psqrt_complex(a); } //---------- double ---------- #if EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG // See bug 1325, clang fails to call vld1q_u64. #if EIGEN_COMP_CLANG || EIGEN_COMP_CASTXML static uint64x2_t p2ul_CONJ_XOR = {0x0, 0x8000000000000000}; #else const uint64_t p2ul_conj_XOR_DATA[] = { 0x0, 0x8000000000000000 }; static uint64x2_t p2ul_CONJ_XOR = vld1q_u64( p2ul_conj_XOR_DATA ); #endif struct Packet1cd { EIGEN_STRONG_INLINE Packet1cd() {} EIGEN_STRONG_INLINE explicit Packet1cd(const Packet2d& a) : v(a) {} Packet2d v; }; template<> struct packet_traits > : default_packet_traits { typedef Packet1cd type; typedef Packet1cd half; enum { Vectorizable = 1, AlignedOnScalar = 0, size = 1, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, HasMax = 0, HasSetLinear = 0 }; }; template<> struct unpacket_traits { typedef std::complex type; typedef Packet1cd half; typedef Packet2d as_real; enum { size=1, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false }; }; template<> EIGEN_STRONG_INLINE Packet1cd pload(const std::complex* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet1cd(pload(reinterpret_cast(from))); } template<> EIGEN_STRONG_INLINE Packet1cd ploadu(const std::complex* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet1cd(ploadu(reinterpret_cast(from))); } template<> EIGEN_STRONG_INLINE Packet1cd pset1(const std::complex& from) { /* here we really have to use unaligned loads :( */ return ploadu(&from); } template<> EIGEN_STRONG_INLINE Packet1cd padd(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(padd(a.v, b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd psub(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(psub(a.v, b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd pnegate(const Packet1cd& a) { return Packet1cd(pnegate(a.v)); } template<> EIGEN_STRONG_INLINE Packet1cd pconj(const Packet1cd& a) { return Packet1cd(vreinterpretq_f64_u64(veorq_u64(vreinterpretq_u64_f64(a.v), p2ul_CONJ_XOR))); } template<> EIGEN_STRONG_INLINE Packet1cd pmul(const Packet1cd& a, const Packet1cd& b) { Packet2d v1, v2; // Get the real values of a v1 = vdupq_lane_f64(vget_low_f64(a.v), 0); // Get the imag values of a v2 = vdupq_lane_f64(vget_high_f64(a.v), 0); // Multiply the real a with b v1 = vmulq_f64(v1, b.v); // Multiply the imag a with b v2 = vmulq_f64(v2, b.v); // Conjugate v2 v2 = vreinterpretq_f64_u64(veorq_u64(vreinterpretq_u64_f64(v2), p2ul_CONJ_XOR)); // Swap real/imag elements in v2. v2 = preverse(v2); // Add and return the result return Packet1cd(vaddq_f64(v1, v2)); } template<> EIGEN_STRONG_INLINE Packet1cd pcmp_eq(const Packet1cd& a, const Packet1cd& b) { // Compare real and imaginary parts of a and b to get the mask vector: // [re(a)==re(b), im(a)==im(b)] Packet2d eq = pcmp_eq(a.v, b.v); // Swap real/imag elements in the mask in to get: // [im(a)==im(b), re(a)==re(b)] Packet2d eq_swapped = vreinterpretq_f64_u32(vrev64q_u32(vreinterpretq_u32_f64(eq))); // Return re(a)==re(b) & im(a)==im(b) by computing bitwise AND of eq and eq_swapped return Packet1cd(pand(eq, eq_swapped)); } template<> EIGEN_STRONG_INLINE Packet1cd pand(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vreinterpretq_f64_u64(vandq_u64(vreinterpretq_u64_f64(a.v),vreinterpretq_u64_f64(b.v)))); } template<> EIGEN_STRONG_INLINE Packet1cd por(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vreinterpretq_f64_u64(vorrq_u64(vreinterpretq_u64_f64(a.v),vreinterpretq_u64_f64(b.v)))); } template<> EIGEN_STRONG_INLINE Packet1cd pxor(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vreinterpretq_f64_u64(veorq_u64(vreinterpretq_u64_f64(a.v),vreinterpretq_u64_f64(b.v)))); } template<> EIGEN_STRONG_INLINE Packet1cd pandnot(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vreinterpretq_f64_u64(vbicq_u64(vreinterpretq_u64_f64(a.v),vreinterpretq_u64_f64(b.v)))); } template<> EIGEN_STRONG_INLINE Packet1cd ploaddup(const std::complex* from) { return pset1(*from); } template<> EIGEN_STRONG_INLINE void pstore >(std::complex *to, const Packet1cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore(reinterpret_cast(to), from.v); } template<> EIGEN_STRONG_INLINE void pstoreu >(std::complex *to, const Packet1cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(reinterpret_cast(to), from.v); } template<> EIGEN_STRONG_INLINE void prefetch >(const std::complex *addr) { EIGEN_ARM_PREFETCH(reinterpret_cast(addr)); } template<> EIGEN_DEVICE_FUNC inline Packet1cd pgather, Packet1cd>( const std::complex* from, Index stride) { Packet2d res = pset1(0.0); res = vsetq_lane_f64(std::real(from[0*stride]), res, 0); res = vsetq_lane_f64(std::imag(from[0*stride]), res, 1); return Packet1cd(res); } template<> EIGEN_DEVICE_FUNC inline void pscatter, Packet1cd>( std::complex* to, const Packet1cd& from, Index stride) { to[stride*0] = std::complex(vgetq_lane_f64(from.v, 0), vgetq_lane_f64(from.v, 1)); } template<> EIGEN_STRONG_INLINE std::complex pfirst(const Packet1cd& a) { EIGEN_ALIGN16 std::complex res; pstore >(&res, a); return res; } template<> EIGEN_STRONG_INLINE Packet1cd preverse(const Packet1cd& a) { return a; } template<> EIGEN_STRONG_INLINE std::complex predux(const Packet1cd& a) { return pfirst(a); } template<> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet1cd& a) { return pfirst(a); } EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd,Packet2d) template<> EIGEN_STRONG_INLINE Packet1cd pdiv(const Packet1cd& a, const Packet1cd& b) { // TODO optimize it for NEON Packet1cd res = pmul(a,pconj(b)); Packet2d s = pmul(b.v, b.v); Packet2d rev_s = preverse(s); return Packet1cd(pdiv(res.v, padd(s,rev_s))); } EIGEN_STRONG_INLINE Packet1cd pcplxflip/**/(const Packet1cd& x) { return Packet1cd(preverse(Packet2d(x.v))); } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { Packet2d tmp = vcombine_f64(vget_high_f64(kernel.packet[0].v), vget_high_f64(kernel.packet[1].v)); kernel.packet[0].v = vcombine_f64(vget_low_f64(kernel.packet[0].v), vget_low_f64(kernel.packet[1].v)); kernel.packet[1].v = tmp; } template<> EIGEN_STRONG_INLINE Packet1cd psqrt(const Packet1cd& a) { return psqrt_complex(a); } #endif // EIGEN_ARCH_ARM64 } // end namespace internal } // end namespace Eigen #endif // EIGEN_COMPLEX_NEON_H RcppEigen/inst/include/Eigen/src/Core/arch/NEON/PacketMath.h0000644000176200001440000056212514567757725023146 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2009 Gael Guennebaud // Copyright (C) 2010 Konstantinos Margaritis // Heavily based on Gael's SSE version. // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_PACKET_MATH_NEON_H #define EIGEN_PACKET_MATH_NEON_H namespace Eigen { namespace internal { #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 #endif #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD #endif #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS #if EIGEN_ARCH_ARM64 #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32 #else #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 16 #endif #endif #if EIGEN_COMP_MSVC_STRICT // In MSVC's arm_neon.h header file, all NEON vector types // are aliases to the same underlying type __n128. // We thus have to wrap them to make them different C++ types. // (See also bug 1428) typedef eigen_packet_wrapper Packet2f; typedef eigen_packet_wrapper Packet4f; typedef eigen_packet_wrapper Packet4c; typedef eigen_packet_wrapper Packet8c; typedef eigen_packet_wrapper Packet16c; typedef eigen_packet_wrapper Packet4uc; typedef eigen_packet_wrapper Packet8uc; typedef eigen_packet_wrapper Packet16uc; typedef eigen_packet_wrapper Packet4s; typedef eigen_packet_wrapper Packet8s; typedef eigen_packet_wrapper Packet4us; typedef eigen_packet_wrapper Packet8us; typedef eigen_packet_wrapper Packet2i; typedef eigen_packet_wrapper Packet4i; typedef eigen_packet_wrapper Packet2ui; typedef eigen_packet_wrapper Packet4ui; typedef eigen_packet_wrapper Packet2l; typedef eigen_packet_wrapper Packet2ul; #else typedef float32x2_t Packet2f; typedef float32x4_t Packet4f; typedef eigen_packet_wrapper Packet4c; typedef int8x8_t Packet8c; typedef int8x16_t Packet16c; typedef eigen_packet_wrapper Packet4uc; typedef uint8x8_t Packet8uc; typedef uint8x16_t Packet16uc; typedef int16x4_t Packet4s; typedef int16x8_t Packet8s; typedef uint16x4_t Packet4us; typedef uint16x8_t Packet8us; typedef int32x2_t Packet2i; typedef int32x4_t Packet4i; typedef uint32x2_t Packet2ui; typedef uint32x4_t Packet4ui; typedef int64x2_t Packet2l; typedef uint64x2_t Packet2ul; #endif // EIGEN_COMP_MSVC_STRICT EIGEN_STRONG_INLINE Packet4f shuffle1(const Packet4f& m, int mask){ const float* a = reinterpret_cast(&m); Packet4f res = {*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3 )), *(a + ((mask >> 6) & 3))}; return res; } // fuctionally equivalent to _mm_shuffle_ps in SSE when interleave // == false (i.e. shuffle(m, n, mask) equals _mm_shuffle_ps(m, n, mask)), // interleave m and n when interleave == true. Currently used in LU/arch/InverseSize4.h // to enable a shared implementation for fast inversion of matrices of size 4. template EIGEN_STRONG_INLINE Packet4f shuffle2(const Packet4f &m, const Packet4f &n, int mask) { const float* a = reinterpret_cast(&m); const float* b = reinterpret_cast(&n); Packet4f res = {*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(b + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3))}; return res; } template<> EIGEN_STRONG_INLINE Packet4f shuffle2(const Packet4f &m, const Packet4f &n, int mask) { const float* a = reinterpret_cast(&m); const float* b = reinterpret_cast(&n); Packet4f res = {*(a + (mask & 3)), *(b + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3))}; return res; } EIGEN_STRONG_INLINE static int eigen_neon_shuffle_mask(int p, int q, int r, int s) {return ((s)<<6|(r)<<4|(q)<<2|(p));} EIGEN_STRONG_INLINE Packet4f vec4f_swizzle1(const Packet4f& a, int p, int q, int r, int s) { return shuffle1(a, eigen_neon_shuffle_mask(p, q, r, s)); } EIGEN_STRONG_INLINE Packet4f vec4f_swizzle2(const Packet4f& a, const Packet4f& b, int p, int q, int r, int s) { return shuffle2(a,b,eigen_neon_shuffle_mask(p, q, r, s)); } EIGEN_STRONG_INLINE Packet4f vec4f_movelh(const Packet4f& a, const Packet4f& b) { return shuffle2(a,b,eigen_neon_shuffle_mask(0, 1, 0, 1)); } EIGEN_STRONG_INLINE Packet4f vec4f_movehl(const Packet4f& a, const Packet4f& b) { return shuffle2(b,a,eigen_neon_shuffle_mask(2, 3, 2, 3)); } EIGEN_STRONG_INLINE Packet4f vec4f_unpacklo(const Packet4f& a, const Packet4f& b) { return shuffle2(a,b,eigen_neon_shuffle_mask(0, 0, 1, 1)); } EIGEN_STRONG_INLINE Packet4f vec4f_unpackhi(const Packet4f& a, const Packet4f& b) { return shuffle2(a,b,eigen_neon_shuffle_mask(2, 2, 3, 3)); } #define vec4f_duplane(a, p) \ vdupq_lane_f32(vget_low_f32(a), p) #define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ const Packet4f p4f_##NAME = pset1(X) #define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ const Packet4f p4f_##NAME = vreinterpretq_f32_u32(pset1(X)) #define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ const Packet4i p4i_##NAME = pset1(X) #if EIGEN_ARCH_ARM64 // __builtin_prefetch tends to do nothing on ARM64 compilers because the // prefetch instructions there are too detailed for __builtin_prefetch to map // meaningfully to them. #define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__("prfm pldl1keep, [%[addr]]\n" ::[addr] "r"(ADDR) : ); #elif EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC #define EIGEN_ARM_PREFETCH(ADDR) __builtin_prefetch(ADDR); #elif defined __pld #define EIGEN_ARM_PREFETCH(ADDR) __pld(ADDR) #elif EIGEN_ARCH_ARM32 #define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__ ("pld [%[addr]]\n" :: [addr] "r" (ADDR) : ); #else // by default no explicit prefetching #define EIGEN_ARM_PREFETCH(ADDR) #endif template <> struct packet_traits : default_packet_traits { typedef Packet4f type; typedef Packet2f half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 4, HasHalfPacket = 1, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasNegate = 1, HasAbs = 1, HasArg = 0, HasAbs2 = 1, HasAbsDiff = 1, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasBlend = 0, HasDiv = 1, HasFloor = 1, HasCeil = 1, HasRint = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasLog = 1, HasExp = 1, HasSqrt = 1, HasRsqrt = 1, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, HasBessel = 0, // Issues with accuracy. HasNdtri = 0 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet16c type; typedef Packet8c half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 16, HasHalfPacket = 1, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasNegate = 1, HasAbs = 1, HasAbsDiff = 1, HasArg = 0, HasAbs2 = 1, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasBlend = 0 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet16uc type; typedef Packet8uc half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 16, HasHalfPacket = 1, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasNegate = 0, HasAbs = 1, HasAbsDiff = 1, HasArg = 0, HasAbs2 = 1, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasBlend = 0, HasSqrt = 1 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet8s type; typedef Packet4s half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 8, HasHalfPacket = 1, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasNegate = 1, HasAbs = 1, HasAbsDiff = 1, HasArg = 0, HasAbs2 = 1, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasBlend = 0 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet8us type; typedef Packet4us half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 8, HasHalfPacket = 1, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasNegate = 0, HasAbs = 0, HasAbsDiff = 1, HasArg = 0, HasAbs2 = 1, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasBlend = 0, HasSqrt = 1 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet4i type; typedef Packet2i half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 4, HasHalfPacket = 1, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasNegate = 1, HasAbs = 1, HasArg = 0, HasAbs2 = 1, HasAbsDiff = 1, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasBlend = 0 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet4ui type; typedef Packet2ui half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 4, HasHalfPacket = 1, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasNegate = 0, HasAbs = 0, HasArg = 0, HasAbs2 = 1, HasAbsDiff = 1, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasBlend = 0, HasSqrt = 1 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet2l type; typedef Packet2l half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 2, HasHalfPacket = 0, HasCmp = 1, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasNegate = 1, HasAbs = 1, HasArg = 0, HasAbs2 = 1, HasAbsDiff = 1, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasBlend = 0 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet2ul type; typedef Packet2ul half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 2, HasHalfPacket = 0, HasCmp = 1, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasNegate = 0, HasAbs = 0, HasArg = 0, HasAbs2 = 1, HasAbsDiff = 1, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasBlend = 0 }; }; #if EIGEN_GNUC_AT_MOST(4, 4) && !EIGEN_COMP_LLVM // workaround gcc 4.2, 4.3 and 4.4 compilation issue EIGEN_STRONG_INLINE float32x4_t vld1q_f32(const float* x) { return ::vld1q_f32((const float32_t*)x); } EIGEN_STRONG_INLINE float32x2_t vld1_f32(const float* x) { return ::vld1_f32 ((const float32_t*)x); } EIGEN_STRONG_INLINE float32x2_t vld1_dup_f32(const float* x) { return ::vld1_dup_f32 ((const float32_t*)x); } EIGEN_STRONG_INLINE void vst1q_f32(float* to, float32x4_t from) { ::vst1q_f32((float32_t*)to,from); } EIGEN_STRONG_INLINE void vst1_f32 (float* to, float32x2_t from) { ::vst1_f32 ((float32_t*)to,from); } #endif template<> struct unpacket_traits { typedef float type; typedef Packet2f half; typedef Packet2i integer_packet; enum { size = 2, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef float type; typedef Packet2f half; typedef Packet4i integer_packet; enum { size = 4, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef int8_t type; typedef Packet4c half; enum { size = 4, alignment = Unaligned, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef int8_t type; typedef Packet4c half; enum { size = 8, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef int8_t type; typedef Packet8c half; enum { size = 16, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef uint8_t type; typedef Packet4uc half; enum { size = 4, alignment = Unaligned, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef uint8_t type; typedef Packet4uc half; enum { size = 8, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef uint8_t type; typedef Packet8uc half; enum { size = 16, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false}; }; template<> struct unpacket_traits { typedef int16_t type; typedef Packet4s half; enum { size = 4, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef int16_t type; typedef Packet4s half; enum { size = 8, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef uint16_t type; typedef Packet4us half; enum { size = 4, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef uint16_t type; typedef Packet4us half; enum { size = 8, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef int32_t type; typedef Packet2i half; enum { size = 2, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef int32_t type; typedef Packet2i half; enum { size = 4, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef uint32_t type; typedef Packet2ui half; enum { size = 2, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef uint32_t type; typedef Packet2ui half; enum { size = 4, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef int64_t type; typedef Packet2l half; enum { size = 2, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> struct unpacket_traits { typedef uint64_t type; typedef Packet2ul half; enum { size = 2, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> EIGEN_STRONG_INLINE Packet2f pset1(const float& from) { return vdup_n_f32(from); } template<> EIGEN_STRONG_INLINE Packet4f pset1(const float& from) { return vdupq_n_f32(from); } template<> EIGEN_STRONG_INLINE Packet4c pset1(const int8_t& from) { return vget_lane_s32(vreinterpret_s32_s8(vdup_n_s8(from)), 0); } template<> EIGEN_STRONG_INLINE Packet8c pset1(const int8_t& from) { return vdup_n_s8(from); } template<> EIGEN_STRONG_INLINE Packet16c pset1(const int8_t& from) { return vdupq_n_s8(from); } template<> EIGEN_STRONG_INLINE Packet4uc pset1(const uint8_t& from) { return vget_lane_u32(vreinterpret_u32_u8(vdup_n_u8(from)), 0); } template<> EIGEN_STRONG_INLINE Packet8uc pset1(const uint8_t& from) { return vdup_n_u8(from); } template<> EIGEN_STRONG_INLINE Packet16uc pset1(const uint8_t& from) { return vdupq_n_u8(from); } template<> EIGEN_STRONG_INLINE Packet4s pset1(const int16_t& from) { return vdup_n_s16(from); } template<> EIGEN_STRONG_INLINE Packet8s pset1(const int16_t& from) { return vdupq_n_s16(from); } template<> EIGEN_STRONG_INLINE Packet4us pset1(const uint16_t& from) { return vdup_n_u16(from); } template<> EIGEN_STRONG_INLINE Packet8us pset1(const uint16_t& from) { return vdupq_n_u16(from); } template<> EIGEN_STRONG_INLINE Packet2i pset1(const int32_t& from) { return vdup_n_s32(from); } template<> EIGEN_STRONG_INLINE Packet4i pset1(const int32_t& from) { return vdupq_n_s32(from); } template<> EIGEN_STRONG_INLINE Packet2ui pset1(const uint32_t& from) { return vdup_n_u32(from); } template<> EIGEN_STRONG_INLINE Packet4ui pset1(const uint32_t& from) { return vdupq_n_u32(from); } template<> EIGEN_STRONG_INLINE Packet2l pset1(const int64_t& from) { return vdupq_n_s64(from); } template<> EIGEN_STRONG_INLINE Packet2ul pset1(const uint64_t& from) { return vdupq_n_u64(from); } template<> EIGEN_STRONG_INLINE Packet2f pset1frombits(unsigned int from) { return vreinterpret_f32_u32(vdup_n_u32(from)); } template<> EIGEN_STRONG_INLINE Packet4f pset1frombits(unsigned int from) { return vreinterpretq_f32_u32(vdupq_n_u32(from)); } template<> EIGEN_STRONG_INLINE Packet2f plset(const float& a) { const float c[] = {0.0f,1.0f}; return vadd_f32(pset1(a), vld1_f32(c)); } template<> EIGEN_STRONG_INLINE Packet4f plset(const float& a) { const float c[] = {0.0f,1.0f,2.0f,3.0f}; return vaddq_f32(pset1(a), vld1q_f32(c)); } template<> EIGEN_STRONG_INLINE Packet4c plset(const int8_t& a) { return vget_lane_s32(vreinterpret_s32_s8(vadd_s8(vreinterpret_s8_u32(vdup_n_u32(0x03020100)), vdup_n_s8(a))), 0); } template<> EIGEN_STRONG_INLINE Packet8c plset(const int8_t& a) { const int8_t c[] = {0,1,2,3,4,5,6,7}; return vadd_s8(pset1(a), vld1_s8(c)); } template<> EIGEN_STRONG_INLINE Packet16c plset(const int8_t& a) { const int8_t c[] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}; return vaddq_s8(pset1(a), vld1q_s8(c)); } template<> EIGEN_STRONG_INLINE Packet4uc plset(const uint8_t& a) { return vget_lane_u32(vreinterpret_u32_u8(vadd_u8(vreinterpret_u8_u32(vdup_n_u32(0x03020100)), vdup_n_u8(a))), 0); } template<> EIGEN_STRONG_INLINE Packet8uc plset(const uint8_t& a) { const uint8_t c[] = {0,1,2,3,4,5,6,7}; return vadd_u8(pset1(a), vld1_u8(c)); } template<> EIGEN_STRONG_INLINE Packet16uc plset(const uint8_t& a) { const uint8_t c[] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}; return vaddq_u8(pset1(a), vld1q_u8(c)); } template<> EIGEN_STRONG_INLINE Packet4s plset(const int16_t& a) { const int16_t c[] = {0,1,2,3}; return vadd_s16(pset1(a), vld1_s16(c)); } template<> EIGEN_STRONG_INLINE Packet4us plset(const uint16_t& a) { const uint16_t c[] = {0,1,2,3}; return vadd_u16(pset1(a), vld1_u16(c)); } template<> EIGEN_STRONG_INLINE Packet8s plset(const int16_t& a) { const int16_t c[] = {0,1,2,3,4,5,6,7}; return vaddq_s16(pset1(a), vld1q_s16(c)); } template<> EIGEN_STRONG_INLINE Packet8us plset(const uint16_t& a) { const uint16_t c[] = {0,1,2,3,4,5,6,7}; return vaddq_u16(pset1(a), vld1q_u16(c)); } template<> EIGEN_STRONG_INLINE Packet2i plset(const int32_t& a) { const int32_t c[] = {0,1}; return vadd_s32(pset1(a), vld1_s32(c)); } template<> EIGEN_STRONG_INLINE Packet4i plset(const int32_t& a) { const int32_t c[] = {0,1,2,3}; return vaddq_s32(pset1(a), vld1q_s32(c)); } template<> EIGEN_STRONG_INLINE Packet2ui plset(const uint32_t& a) { const uint32_t c[] = {0,1}; return vadd_u32(pset1(a), vld1_u32(c)); } template<> EIGEN_STRONG_INLINE Packet4ui plset(const uint32_t& a) { const uint32_t c[] = {0,1,2,3}; return vaddq_u32(pset1(a), vld1q_u32(c)); } template<> EIGEN_STRONG_INLINE Packet2l plset(const int64_t& a) { const int64_t c[] = {0,1}; return vaddq_s64(pset1(a), vld1q_s64(c)); } template<> EIGEN_STRONG_INLINE Packet2ul plset(const uint64_t& a) { const uint64_t c[] = {0,1}; return vaddq_u64(pset1(a), vld1q_u64(c)); } template<> EIGEN_STRONG_INLINE Packet2f padd(const Packet2f& a, const Packet2f& b) { return vadd_f32(a,b); } template<> EIGEN_STRONG_INLINE Packet4f padd(const Packet4f& a, const Packet4f& b) { return vaddq_f32(a,b); } template<> EIGEN_STRONG_INLINE Packet4c padd(const Packet4c& a, const Packet4c& b) { return vget_lane_s32(vreinterpret_s32_s8(vadd_s8( vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8c padd(const Packet8c& a, const Packet8c& b) { return vadd_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet16c padd(const Packet16c& a, const Packet16c& b) { return vaddq_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet4uc padd(const Packet4uc& a, const Packet4uc& b) { return vget_lane_u32(vreinterpret_u32_u8(vadd_u8( vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8uc padd(const Packet8uc& a, const Packet8uc& b) { return vadd_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet16uc padd(const Packet16uc& a, const Packet16uc& b) { return vaddq_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet4s padd(const Packet4s& a, const Packet4s& b) { return vadd_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet8s padd(const Packet8s& a, const Packet8s& b) { return vaddq_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet4us padd(const Packet4us& a, const Packet4us& b) { return vadd_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet8us padd(const Packet8us& a, const Packet8us& b) { return vaddq_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet2i padd(const Packet2i& a, const Packet2i& b) { return vadd_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet4i padd(const Packet4i& a, const Packet4i& b) { return vaddq_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet2ui padd(const Packet2ui& a, const Packet2ui& b) { return vadd_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet4ui padd(const Packet4ui& a, const Packet4ui& b) { return vaddq_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet2l padd(const Packet2l& a, const Packet2l& b) { return vaddq_s64(a,b); } template<> EIGEN_STRONG_INLINE Packet2ul padd(const Packet2ul& a, const Packet2ul& b) { return vaddq_u64(a,b); } template<> EIGEN_STRONG_INLINE Packet2f psub(const Packet2f& a, const Packet2f& b) { return vsub_f32(a,b); } template<> EIGEN_STRONG_INLINE Packet4f psub(const Packet4f& a, const Packet4f& b) { return vsubq_f32(a,b); } template<> EIGEN_STRONG_INLINE Packet4c psub(const Packet4c& a, const Packet4c& b) { return vget_lane_s32(vreinterpret_s32_s8(vsub_s8( vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8c psub(const Packet8c& a, const Packet8c& b) { return vsub_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet16c psub(const Packet16c& a, const Packet16c& b) { return vsubq_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet4uc psub(const Packet4uc& a, const Packet4uc& b) { return vget_lane_u32(vreinterpret_u32_u8(vsub_u8( vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8uc psub(const Packet8uc& a, const Packet8uc& b) { return vsub_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet16uc psub(const Packet16uc& a, const Packet16uc& b) { return vsubq_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet4s psub(const Packet4s& a, const Packet4s& b) { return vsub_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet8s psub(const Packet8s& a, const Packet8s& b) { return vsubq_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet4us psub(const Packet4us& a, const Packet4us& b) { return vsub_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet8us psub(const Packet8us& a, const Packet8us& b) { return vsubq_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet2i psub(const Packet2i& a, const Packet2i& b) { return vsub_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet4i psub(const Packet4i& a, const Packet4i& b) { return vsubq_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet2ui psub(const Packet2ui& a, const Packet2ui& b) { return vsub_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet4ui psub(const Packet4ui& a, const Packet4ui& b) { return vsubq_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet2l psub(const Packet2l& a, const Packet2l& b) { return vsubq_s64(a,b); } template<> EIGEN_STRONG_INLINE Packet2ul psub(const Packet2ul& a, const Packet2ul& b) { return vsubq_u64(a,b); } template<> EIGEN_STRONG_INLINE Packet2f pxor(const Packet2f& a, const Packet2f& b); template<> EIGEN_STRONG_INLINE Packet2f paddsub(const Packet2f& a, const Packet2f & b) { Packet2f mask = {numext::bit_cast(0x80000000u), 0.0f}; return padd(a, pxor(mask, b)); } template<> EIGEN_STRONG_INLINE Packet4f pxor(const Packet4f& a, const Packet4f& b); template<> EIGEN_STRONG_INLINE Packet4f paddsub(const Packet4f& a, const Packet4f& b) { Packet4f mask = {numext::bit_cast(0x80000000u), 0.0f, numext::bit_cast(0x80000000u), 0.0f}; return padd(a, pxor(mask, b)); } template<> EIGEN_STRONG_INLINE Packet2f pnegate(const Packet2f& a) { return vneg_f32(a); } template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { return vnegq_f32(a); } template<> EIGEN_STRONG_INLINE Packet4c pnegate(const Packet4c& a) { return vget_lane_s32(vreinterpret_s32_s8(vneg_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0); } template<> EIGEN_STRONG_INLINE Packet8c pnegate(const Packet8c& a) { return vneg_s8(a); } template<> EIGEN_STRONG_INLINE Packet16c pnegate(const Packet16c& a) { return vnegq_s8(a); } template<> EIGEN_STRONG_INLINE Packet4s pnegate(const Packet4s& a) { return vneg_s16(a); } template<> EIGEN_STRONG_INLINE Packet8s pnegate(const Packet8s& a) { return vnegq_s16(a); } template<> EIGEN_STRONG_INLINE Packet2i pnegate(const Packet2i& a) { return vneg_s32(a); } template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return vnegq_s32(a); } template<> EIGEN_STRONG_INLINE Packet2l pnegate(const Packet2l& a) { #if EIGEN_ARCH_ARM64 return vnegq_s64(a); #else return vcombine_s64( vdup_n_s64(-vgetq_lane_s64(a, 0)), vdup_n_s64(-vgetq_lane_s64(a, 1))); #endif } template<> EIGEN_STRONG_INLINE Packet2f pconj(const Packet2f& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4c pconj(const Packet4c& a) { return a; } template<> EIGEN_STRONG_INLINE Packet8c pconj(const Packet8c& a) { return a; } template<> EIGEN_STRONG_INLINE Packet16c pconj(const Packet16c& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4uc pconj(const Packet4uc& a) { return a; } template<> EIGEN_STRONG_INLINE Packet8uc pconj(const Packet8uc& a) { return a; } template<> EIGEN_STRONG_INLINE Packet16uc pconj(const Packet16uc& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4s pconj(const Packet4s& a) { return a; } template<> EIGEN_STRONG_INLINE Packet8s pconj(const Packet8s& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4us pconj(const Packet4us& a) { return a; } template<> EIGEN_STRONG_INLINE Packet8us pconj(const Packet8us& a) { return a; } template<> EIGEN_STRONG_INLINE Packet2i pconj(const Packet2i& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; } template<> EIGEN_STRONG_INLINE Packet2ui pconj(const Packet2ui& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4ui pconj(const Packet4ui& a) { return a; } template<> EIGEN_STRONG_INLINE Packet2l pconj(const Packet2l& a) { return a; } template<> EIGEN_STRONG_INLINE Packet2ul pconj(const Packet2ul& a) { return a; } template<> EIGEN_STRONG_INLINE Packet2f pmul(const Packet2f& a, const Packet2f& b) { return vmul_f32(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pmul(const Packet4f& a, const Packet4f& b) { return vmulq_f32(a,b); } template<> EIGEN_STRONG_INLINE Packet4c pmul(const Packet4c& a, const Packet4c& b) { return vget_lane_s32(vreinterpret_s32_s8(vmul_s8( vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8c pmul(const Packet8c& a, const Packet8c& b) { return vmul_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet16c pmul(const Packet16c& a, const Packet16c& b) { return vmulq_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet4uc pmul(const Packet4uc& a, const Packet4uc& b) { return vget_lane_u32(vreinterpret_u32_u8(vmul_u8( vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8uc pmul(const Packet8uc& a, const Packet8uc& b) { return vmul_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet16uc pmul(const Packet16uc& a, const Packet16uc& b) { return vmulq_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet4s pmul(const Packet4s& a, const Packet4s& b) { return vmul_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet8s pmul(const Packet8s& a, const Packet8s& b) { return vmulq_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet4us pmul(const Packet4us& a, const Packet4us& b) { return vmul_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet8us pmul(const Packet8us& a, const Packet8us& b) { return vmulq_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet2i pmul(const Packet2i& a, const Packet2i& b) { return vmul_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pmul(const Packet4i& a, const Packet4i& b) { return vmulq_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet2ui pmul(const Packet2ui& a, const Packet2ui& b) { return vmul_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet4ui pmul(const Packet4ui& a, const Packet4ui& b) { return vmulq_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet2l pmul(const Packet2l& a, const Packet2l& b) { return vcombine_s64( vdup_n_s64(vgetq_lane_s64(a, 0)*vgetq_lane_s64(b, 0)), vdup_n_s64(vgetq_lane_s64(a, 1)*vgetq_lane_s64(b, 1))); } template<> EIGEN_STRONG_INLINE Packet2ul pmul(const Packet2ul& a, const Packet2ul& b) { return vcombine_u64( vdup_n_u64(vgetq_lane_u64(a, 0)*vgetq_lane_u64(b, 0)), vdup_n_u64(vgetq_lane_u64(a, 1)*vgetq_lane_u64(b, 1))); } template<> EIGEN_STRONG_INLINE Packet2f pdiv(const Packet2f& a, const Packet2f& b) { #if EIGEN_ARCH_ARM64 return vdiv_f32(a,b); #else Packet2f inv, restep, div; // NEON does not offer a divide instruction, we have to do a reciprocal approximation // However NEON in contrast to other SIMD engines (AltiVec/SSE), offers // a reciprocal estimate AND a reciprocal step -which saves a few instructions // vrecpeq_f32() returns an estimate to 1/b, which we will finetune with // Newton-Raphson and vrecpsq_f32() inv = vrecpe_f32(b); // This returns a differential, by which we will have to multiply inv to get a better // approximation of 1/b. restep = vrecps_f32(b, inv); inv = vmul_f32(restep, inv); // Finally, multiply a by 1/b and get the wanted result of the division. div = vmul_f32(a, inv); return div; #endif } template<> EIGEN_STRONG_INLINE Packet4f pdiv(const Packet4f& a, const Packet4f& b) { #if EIGEN_ARCH_ARM64 return vdivq_f32(a,b); #else Packet4f inv, restep, div; // NEON does not offer a divide instruction, we have to do a reciprocal approximation // However NEON in contrast to other SIMD engines (AltiVec/SSE), offers // a reciprocal estimate AND a reciprocal step -which saves a few instructions // vrecpeq_f32() returns an estimate to 1/b, which we will finetune with // Newton-Raphson and vrecpsq_f32() inv = vrecpeq_f32(b); // This returns a differential, by which we will have to multiply inv to get a better // approximation of 1/b. restep = vrecpsq_f32(b, inv); inv = vmulq_f32(restep, inv); // Finally, multiply a by 1/b and get the wanted result of the division. div = vmulq_f32(a, inv); return div; #endif } template<> EIGEN_STRONG_INLINE Packet4c pdiv(const Packet4c& /*a*/, const Packet4c& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); return pset1(0); } template<> EIGEN_STRONG_INLINE Packet8c pdiv(const Packet8c& /*a*/, const Packet8c& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); return pset1(0); } template<> EIGEN_STRONG_INLINE Packet16c pdiv(const Packet16c& /*a*/, const Packet16c& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); return pset1(0); } template<> EIGEN_STRONG_INLINE Packet4uc pdiv(const Packet4uc& /*a*/, const Packet4uc& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); return pset1(0); } template<> EIGEN_STRONG_INLINE Packet8uc pdiv(const Packet8uc& /*a*/, const Packet8uc& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); return pset1(0); } template<> EIGEN_STRONG_INLINE Packet16uc pdiv(const Packet16uc& /*a*/, const Packet16uc& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); return pset1(0); } template<> EIGEN_STRONG_INLINE Packet4s pdiv(const Packet4s& /*a*/, const Packet4s& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); return pset1(0); } template<> EIGEN_STRONG_INLINE Packet8s pdiv(const Packet8s& /*a*/, const Packet8s& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); return pset1(0); } template<> EIGEN_STRONG_INLINE Packet4us pdiv(const Packet4us& /*a*/, const Packet4us& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); return pset1(0); } template<> EIGEN_STRONG_INLINE Packet8us pdiv(const Packet8us& /*a*/, const Packet8us& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); return pset1(0); } template<> EIGEN_STRONG_INLINE Packet2i pdiv(const Packet2i& /*a*/, const Packet2i& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); return pset1(0); } template<> EIGEN_STRONG_INLINE Packet4i pdiv(const Packet4i& /*a*/, const Packet4i& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); return pset1(0); } template<> EIGEN_STRONG_INLINE Packet2ui pdiv(const Packet2ui& /*a*/, const Packet2ui& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); return pset1(0); } template<> EIGEN_STRONG_INLINE Packet4ui pdiv(const Packet4ui& /*a*/, const Packet4ui& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); return pset1(0); } template<> EIGEN_STRONG_INLINE Packet2l pdiv(const Packet2l& /*a*/, const Packet2l& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); return pset1(0LL); } template<> EIGEN_STRONG_INLINE Packet2ul pdiv(const Packet2ul& /*a*/, const Packet2ul& /*b*/) { eigen_assert(false && "packet integer division are not supported by NEON"); return pset1(0ULL); } #ifdef __ARM_FEATURE_FMA template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return vfmaq_f32(c,a,b); } template<> EIGEN_STRONG_INLINE Packet2f pmadd(const Packet2f& a, const Packet2f& b, const Packet2f& c) { return vfma_f32(c,a,b); } #else template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return vmlaq_f32(c,a,b); } template<> EIGEN_STRONG_INLINE Packet2f pmadd(const Packet2f& a, const Packet2f& b, const Packet2f& c) { return vmla_f32(c,a,b); } #endif // No FMA instruction for int, so use MLA unconditionally. template<> EIGEN_STRONG_INLINE Packet4c pmadd(const Packet4c& a, const Packet4c& b, const Packet4c& c) { return vget_lane_s32(vreinterpret_s32_s8(vmla_s8( vreinterpret_s8_s32(vdup_n_s32(c)), vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8c pmadd(const Packet8c& a, const Packet8c& b, const Packet8c& c) { return vmla_s8(c,a,b); } template<> EIGEN_STRONG_INLINE Packet16c pmadd(const Packet16c& a, const Packet16c& b, const Packet16c& c) { return vmlaq_s8(c,a,b); } template<> EIGEN_STRONG_INLINE Packet4uc pmadd(const Packet4uc& a, const Packet4uc& b, const Packet4uc& c) { return vget_lane_u32(vreinterpret_u32_u8(vmla_u8( vreinterpret_u8_u32(vdup_n_u32(c)), vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8uc pmadd(const Packet8uc& a, const Packet8uc& b, const Packet8uc& c) { return vmla_u8(c,a,b); } template<> EIGEN_STRONG_INLINE Packet16uc pmadd(const Packet16uc& a, const Packet16uc& b, const Packet16uc& c) { return vmlaq_u8(c,a,b); } template<> EIGEN_STRONG_INLINE Packet4s pmadd(const Packet4s& a, const Packet4s& b, const Packet4s& c) { return vmla_s16(c,a,b); } template<> EIGEN_STRONG_INLINE Packet8s pmadd(const Packet8s& a, const Packet8s& b, const Packet8s& c) { return vmlaq_s16(c,a,b); } template<> EIGEN_STRONG_INLINE Packet4us pmadd(const Packet4us& a, const Packet4us& b, const Packet4us& c) { return vmla_u16(c,a,b); } template<> EIGEN_STRONG_INLINE Packet8us pmadd(const Packet8us& a, const Packet8us& b, const Packet8us& c) { return vmlaq_u16(c,a,b); } template<> EIGEN_STRONG_INLINE Packet2i pmadd(const Packet2i& a, const Packet2i& b, const Packet2i& c) { return vmla_s32(c,a,b); } template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return vmlaq_s32(c,a,b); } template<> EIGEN_STRONG_INLINE Packet2ui pmadd(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c) { return vmla_u32(c,a,b); } template<> EIGEN_STRONG_INLINE Packet4ui pmadd(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c) { return vmlaq_u32(c,a,b); } template<> EIGEN_STRONG_INLINE Packet2f pabsdiff(const Packet2f& a, const Packet2f& b) { return vabd_f32(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pabsdiff(const Packet4f& a, const Packet4f& b) { return vabdq_f32(a,b); } template<> EIGEN_STRONG_INLINE Packet4c pabsdiff(const Packet4c& a, const Packet4c& b) { return vget_lane_s32(vreinterpret_s32_s8(vabd_s8( vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8c pabsdiff(const Packet8c& a, const Packet8c& b) { return vabd_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet16c pabsdiff(const Packet16c& a, const Packet16c& b) { return vabdq_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet4uc pabsdiff(const Packet4uc& a, const Packet4uc& b) { return vget_lane_u32(vreinterpret_u32_u8(vabd_u8( vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8uc pabsdiff(const Packet8uc& a, const Packet8uc& b) { return vabd_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet16uc pabsdiff(const Packet16uc& a, const Packet16uc& b) { return vabdq_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet4s pabsdiff(const Packet4s& a, const Packet4s& b) { return vabd_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet8s pabsdiff(const Packet8s& a, const Packet8s& b) { return vabdq_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet4us pabsdiff(const Packet4us& a, const Packet4us& b) { return vabd_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet8us pabsdiff(const Packet8us& a, const Packet8us& b) { return vabdq_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet2i pabsdiff(const Packet2i& a, const Packet2i& b) { return vabd_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pabsdiff(const Packet4i& a, const Packet4i& b) { return vabdq_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet2ui pabsdiff(const Packet2ui& a, const Packet2ui& b) { return vabd_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet4ui pabsdiff(const Packet4ui& a, const Packet4ui& b) { return vabdq_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet2f pmin(const Packet2f& a, const Packet2f& b) { return vmin_f32(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pmin(const Packet4f& a, const Packet4f& b) { return vminq_f32(a,b); } #ifdef __ARM_FEATURE_NUMERIC_MAXMIN // numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8 systems). template<> EIGEN_STRONG_INLINE Packet4f pmin(const Packet4f& a, const Packet4f& b) { return vminnmq_f32(a, b); } template<> EIGEN_STRONG_INLINE Packet2f pmin(const Packet2f& a, const Packet2f& b) { return vminnm_f32(a, b); } #endif template<> EIGEN_STRONG_INLINE Packet4f pmin(const Packet4f& a, const Packet4f& b) { return pmin(a, b); } template<> EIGEN_STRONG_INLINE Packet2f pmin(const Packet2f& a, const Packet2f& b) { return pmin(a, b); } template<> EIGEN_STRONG_INLINE Packet4c pmin(const Packet4c& a, const Packet4c& b) { return vget_lane_s32(vreinterpret_s32_s8(vmin_s8( vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8c pmin(const Packet8c& a, const Packet8c& b) { return vmin_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet16c pmin(const Packet16c& a, const Packet16c& b) { return vminq_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet4uc pmin(const Packet4uc& a, const Packet4uc& b) { return vget_lane_u32(vreinterpret_u32_u8(vmin_u8( vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8uc pmin(const Packet8uc& a, const Packet8uc& b) { return vmin_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet16uc pmin(const Packet16uc& a, const Packet16uc& b) { return vminq_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet4s pmin(const Packet4s& a, const Packet4s& b) { return vmin_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet8s pmin(const Packet8s& a, const Packet8s& b) { return vminq_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet4us pmin(const Packet4us& a, const Packet4us& b) { return vmin_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet8us pmin(const Packet8us& a, const Packet8us& b) { return vminq_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet2i pmin(const Packet2i& a, const Packet2i& b) { return vmin_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pmin(const Packet4i& a, const Packet4i& b) { return vminq_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet2ui pmin(const Packet2ui& a, const Packet2ui& b) { return vmin_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet4ui pmin(const Packet4ui& a, const Packet4ui& b) { return vminq_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet2l pmin(const Packet2l& a, const Packet2l& b) { return vcombine_s64( vdup_n_s64((std::min)(vgetq_lane_s64(a, 0), vgetq_lane_s64(b, 0))), vdup_n_s64((std::min)(vgetq_lane_s64(a, 1), vgetq_lane_s64(b, 1)))); } template<> EIGEN_STRONG_INLINE Packet2ul pmin(const Packet2ul& a, const Packet2ul& b) { return vcombine_u64( vdup_n_u64((std::min)(vgetq_lane_u64(a, 0), vgetq_lane_u64(b, 0))), vdup_n_u64((std::min)(vgetq_lane_u64(a, 1), vgetq_lane_u64(b, 1)))); } template<> EIGEN_STRONG_INLINE Packet2f pmax(const Packet2f& a, const Packet2f& b) { return vmax_f32(a,b); } template<> EIGEN_STRONG_INLINE Packet4f pmax(const Packet4f& a, const Packet4f& b) { return vmaxq_f32(a,b); } #ifdef __ARM_FEATURE_NUMERIC_MAXMIN // numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8 systems). template<> EIGEN_STRONG_INLINE Packet4f pmax(const Packet4f& a, const Packet4f& b) { return vmaxnmq_f32(a, b); } template<> EIGEN_STRONG_INLINE Packet2f pmax(const Packet2f& a, const Packet2f& b) { return vmaxnm_f32(a, b); } #endif template<> EIGEN_STRONG_INLINE Packet4f pmax(const Packet4f& a, const Packet4f& b) { return pmax(a, b); } template<> EIGEN_STRONG_INLINE Packet2f pmax(const Packet2f& a, const Packet2f& b) { return pmax(a, b); } template<> EIGEN_STRONG_INLINE Packet4c pmax(const Packet4c& a, const Packet4c& b) { return vget_lane_s32(vreinterpret_s32_s8(vmax_s8( vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8c pmax(const Packet8c& a, const Packet8c& b) { return vmax_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet16c pmax(const Packet16c& a, const Packet16c& b) { return vmaxq_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet4uc pmax(const Packet4uc& a, const Packet4uc& b) { return vget_lane_u32(vreinterpret_u32_u8(vmax_u8( vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8uc pmax(const Packet8uc& a, const Packet8uc& b) { return vmax_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet16uc pmax(const Packet16uc& a, const Packet16uc& b) { return vmaxq_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet4s pmax(const Packet4s& a, const Packet4s& b) { return vmax_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet8s pmax(const Packet8s& a, const Packet8s& b) { return vmaxq_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet4us pmax(const Packet4us& a, const Packet4us& b) { return vmax_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet8us pmax(const Packet8us& a, const Packet8us& b) { return vmaxq_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet2i pmax(const Packet2i& a, const Packet2i& b) { return vmax_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pmax(const Packet4i& a, const Packet4i& b) { return vmaxq_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet2ui pmax(const Packet2ui& a, const Packet2ui& b) { return vmax_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet4ui pmax(const Packet4ui& a, const Packet4ui& b) { return vmaxq_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet2l pmax(const Packet2l& a, const Packet2l& b) { return vcombine_s64( vdup_n_s64((std::max)(vgetq_lane_s64(a, 0), vgetq_lane_s64(b, 0))), vdup_n_s64((std::max)(vgetq_lane_s64(a, 1), vgetq_lane_s64(b, 1)))); } template<> EIGEN_STRONG_INLINE Packet2ul pmax(const Packet2ul& a, const Packet2ul& b) { return vcombine_u64( vdup_n_u64((std::max)(vgetq_lane_u64(a, 0), vgetq_lane_u64(b, 0))), vdup_n_u64((std::max)(vgetq_lane_u64(a, 1), vgetq_lane_u64(b, 1)))); } template<> EIGEN_STRONG_INLINE Packet2f pcmp_le(const Packet2f& a, const Packet2f& b) { return vreinterpret_f32_u32(vcle_f32(a,b)); } template<> EIGEN_STRONG_INLINE Packet4f pcmp_le(const Packet4f& a, const Packet4f& b) { return vreinterpretq_f32_u32(vcleq_f32(a,b)); } template<> EIGEN_STRONG_INLINE Packet4c pcmp_le(const Packet4c& a, const Packet4c& b) { return vget_lane_s32(vreinterpret_s32_u8(vcle_s8( vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8c pcmp_le(const Packet8c& a, const Packet8c& b) { return vreinterpret_s8_u8(vcle_s8(a,b)); } template<> EIGEN_STRONG_INLINE Packet16c pcmp_le(const Packet16c& a, const Packet16c& b) { return vreinterpretq_s8_u8(vcleq_s8(a,b)); } template<> EIGEN_STRONG_INLINE Packet4uc pcmp_le(const Packet4uc& a, const Packet4uc& b) { return vget_lane_u32(vreinterpret_u32_u8(vcle_u8( vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8uc pcmp_le(const Packet8uc& a, const Packet8uc& b) { return vcle_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet16uc pcmp_le(const Packet16uc& a, const Packet16uc& b) { return vcleq_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet4s pcmp_le(const Packet4s& a, const Packet4s& b) { return vreinterpret_s16_u16(vcle_s16(a,b)); } template<> EIGEN_STRONG_INLINE Packet8s pcmp_le(const Packet8s& a, const Packet8s& b) { return vreinterpretq_s16_u16(vcleq_s16(a,b)); } template<> EIGEN_STRONG_INLINE Packet4us pcmp_le(const Packet4us& a, const Packet4us& b) { return vcle_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet8us pcmp_le(const Packet8us& a, const Packet8us& b) { return vcleq_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet2i pcmp_le(const Packet2i& a, const Packet2i& b) { return vreinterpret_s32_u32(vcle_s32(a,b)); } template<> EIGEN_STRONG_INLINE Packet4i pcmp_le(const Packet4i& a, const Packet4i& b) { return vreinterpretq_s32_u32(vcleq_s32(a,b)); } template<> EIGEN_STRONG_INLINE Packet2ui pcmp_le(const Packet2ui& a, const Packet2ui& b) { return vcle_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet4ui pcmp_le(const Packet4ui& a, const Packet4ui& b) { return vcleq_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet2l pcmp_le(const Packet2l& a, const Packet2l& b) { #if EIGEN_ARCH_ARM64 return vreinterpretq_s64_u64(vcleq_s64(a,b)); #else return vcombine_s64( vdup_n_s64(vgetq_lane_s64(a, 0) <= vgetq_lane_s64(b, 0) ? numext::int64_t(-1) : 0), vdup_n_s64(vgetq_lane_s64(a, 1) <= vgetq_lane_s64(b, 1) ? numext::int64_t(-1) : 0)); #endif } template<> EIGEN_STRONG_INLINE Packet2ul pcmp_le(const Packet2ul& a, const Packet2ul& b) { #if EIGEN_ARCH_ARM64 return vcleq_u64(a,b); #else return vcombine_u64( vdup_n_u64(vgetq_lane_u64(a, 0) <= vgetq_lane_u64(b, 0) ? numext::uint64_t(-1) : 0), vdup_n_u64(vgetq_lane_u64(a, 1) <= vgetq_lane_u64(b, 1) ? numext::uint64_t(-1) : 0)); #endif } template<> EIGEN_STRONG_INLINE Packet2f pcmp_lt(const Packet2f& a, const Packet2f& b) { return vreinterpret_f32_u32(vclt_f32(a,b)); } template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt(const Packet4f& a, const Packet4f& b) { return vreinterpretq_f32_u32(vcltq_f32(a,b)); } template<> EIGEN_STRONG_INLINE Packet4c pcmp_lt(const Packet4c& a, const Packet4c& b) { return vget_lane_s32(vreinterpret_s32_u8(vclt_s8( vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8c pcmp_lt(const Packet8c& a, const Packet8c& b) { return vreinterpret_s8_u8(vclt_s8(a,b)); } template<> EIGEN_STRONG_INLINE Packet16c pcmp_lt(const Packet16c& a, const Packet16c& b) { return vreinterpretq_s8_u8(vcltq_s8(a,b)); } template<> EIGEN_STRONG_INLINE Packet4uc pcmp_lt(const Packet4uc& a, const Packet4uc& b) { return vget_lane_u32(vreinterpret_u32_u8(vclt_u8( vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8uc pcmp_lt(const Packet8uc& a, const Packet8uc& b) { return vclt_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet16uc pcmp_lt(const Packet16uc& a, const Packet16uc& b) { return vcltq_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet4s pcmp_lt(const Packet4s& a, const Packet4s& b) { return vreinterpret_s16_u16(vclt_s16(a,b)); } template<> EIGEN_STRONG_INLINE Packet8s pcmp_lt(const Packet8s& a, const Packet8s& b) { return vreinterpretq_s16_u16(vcltq_s16(a,b)); } template<> EIGEN_STRONG_INLINE Packet4us pcmp_lt(const Packet4us& a, const Packet4us& b) { return vclt_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet8us pcmp_lt(const Packet8us& a, const Packet8us& b) { return vcltq_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet2i pcmp_lt(const Packet2i& a, const Packet2i& b) { return vreinterpret_s32_u32(vclt_s32(a,b)); } template<> EIGEN_STRONG_INLINE Packet4i pcmp_lt(const Packet4i& a, const Packet4i& b) { return vreinterpretq_s32_u32(vcltq_s32(a,b)); } template<> EIGEN_STRONG_INLINE Packet2ui pcmp_lt(const Packet2ui& a, const Packet2ui& b) { return vclt_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet4ui pcmp_lt(const Packet4ui& a, const Packet4ui& b) { return vcltq_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet2l pcmp_lt(const Packet2l& a, const Packet2l& b) { #if EIGEN_ARCH_ARM64 return vreinterpretq_s64_u64(vcltq_s64(a,b)); #else return vcombine_s64( vdup_n_s64(vgetq_lane_s64(a, 0) < vgetq_lane_s64(b, 0) ? numext::int64_t(-1) : 0), vdup_n_s64(vgetq_lane_s64(a, 1) < vgetq_lane_s64(b, 1) ? numext::int64_t(-1) : 0)); #endif } template<> EIGEN_STRONG_INLINE Packet2ul pcmp_lt(const Packet2ul& a, const Packet2ul& b) { #if EIGEN_ARCH_ARM64 return vcltq_u64(a,b); #else return vcombine_u64( vdup_n_u64(vgetq_lane_u64(a, 0) < vgetq_lane_u64(b, 0) ? numext::uint64_t(-1) : 0), vdup_n_u64(vgetq_lane_u64(a, 1) < vgetq_lane_u64(b, 1) ? numext::uint64_t(-1) : 0)); #endif } template<> EIGEN_STRONG_INLINE Packet2f pcmp_eq(const Packet2f& a, const Packet2f& b) { return vreinterpret_f32_u32(vceq_f32(a,b)); } template<> EIGEN_STRONG_INLINE Packet4f pcmp_eq(const Packet4f& a, const Packet4f& b) { return vreinterpretq_f32_u32(vceqq_f32(a,b)); } template<> EIGEN_STRONG_INLINE Packet4c pcmp_eq(const Packet4c& a, const Packet4c& b) { return vget_lane_s32(vreinterpret_s32_u8(vceq_s8( vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8c pcmp_eq(const Packet8c& a, const Packet8c& b) { return vreinterpret_s8_u8(vceq_s8(a,b)); } template<> EIGEN_STRONG_INLINE Packet16c pcmp_eq(const Packet16c& a, const Packet16c& b) { return vreinterpretq_s8_u8(vceqq_s8(a,b)); } template<> EIGEN_STRONG_INLINE Packet4uc pcmp_eq(const Packet4uc& a, const Packet4uc& b) { return vget_lane_u32(vreinterpret_u32_u8(vceq_u8( vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0); } template<> EIGEN_STRONG_INLINE Packet8uc pcmp_eq(const Packet8uc& a, const Packet8uc& b) { return vceq_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet16uc pcmp_eq(const Packet16uc& a, const Packet16uc& b) { return vceqq_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet4s pcmp_eq(const Packet4s& a, const Packet4s& b) { return vreinterpret_s16_u16(vceq_s16(a,b)); } template<> EIGEN_STRONG_INLINE Packet8s pcmp_eq(const Packet8s& a, const Packet8s& b) { return vreinterpretq_s16_u16(vceqq_s16(a,b)); } template<> EIGEN_STRONG_INLINE Packet4us pcmp_eq(const Packet4us& a, const Packet4us& b) { return vceq_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet8us pcmp_eq(const Packet8us& a, const Packet8us& b) { return vceqq_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet2i pcmp_eq(const Packet2i& a, const Packet2i& b) { return vreinterpret_s32_u32(vceq_s32(a,b)); } template<> EIGEN_STRONG_INLINE Packet4i pcmp_eq(const Packet4i& a, const Packet4i& b) { return vreinterpretq_s32_u32(vceqq_s32(a,b)); } template<> EIGEN_STRONG_INLINE Packet2ui pcmp_eq(const Packet2ui& a, const Packet2ui& b) { return vceq_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet4ui pcmp_eq(const Packet4ui& a, const Packet4ui& b) { return vceqq_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet2l pcmp_eq(const Packet2l& a, const Packet2l& b) { #if EIGEN_ARCH_ARM64 return vreinterpretq_s64_u64(vceqq_s64(a,b)); #else return vcombine_s64( vdup_n_s64(vgetq_lane_s64(a, 0) == vgetq_lane_s64(b, 0) ? numext::int64_t(-1) : 0), vdup_n_s64(vgetq_lane_s64(a, 1) == vgetq_lane_s64(b, 1) ? numext::int64_t(-1) : 0)); #endif } template<> EIGEN_STRONG_INLINE Packet2ul pcmp_eq(const Packet2ul& a, const Packet2ul& b) { #if EIGEN_ARCH_ARM64 return vceqq_u64(a,b); #else return vcombine_u64( vdup_n_u64(vgetq_lane_u64(a, 0) == vgetq_lane_u64(b, 0) ? numext::uint64_t(-1) : 0), vdup_n_u64(vgetq_lane_u64(a, 1) == vgetq_lane_u64(b, 1) ? numext::uint64_t(-1) : 0)); #endif } template<> EIGEN_STRONG_INLINE Packet2f pcmp_lt_or_nan(const Packet2f& a, const Packet2f& b) { return vreinterpret_f32_u32(vmvn_u32(vcge_f32(a,b))); } template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan(const Packet4f& a, const Packet4f& b) { return vreinterpretq_f32_u32(vmvnq_u32(vcgeq_f32(a,b))); } // Logical Operations are not supported for float, so we have to reinterpret casts using NEON intrinsics template<> EIGEN_STRONG_INLINE Packet2f pand(const Packet2f& a, const Packet2f& b) { return vreinterpret_f32_u32(vand_u32(vreinterpret_u32_f32(a),vreinterpret_u32_f32(b))); } template<> EIGEN_STRONG_INLINE Packet4f pand(const Packet4f& a, const Packet4f& b) { return vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b))); } template<> EIGEN_STRONG_INLINE Packet4c pand(const Packet4c& a, const Packet4c& b) { return a & b; } template<> EIGEN_STRONG_INLINE Packet8c pand(const Packet8c& a, const Packet8c& b) { return vand_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet16c pand(const Packet16c& a, const Packet16c& b) { return vandq_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet4uc pand(const Packet4uc& a, const Packet4uc& b) { return a & b; } template<> EIGEN_STRONG_INLINE Packet8uc pand(const Packet8uc& a, const Packet8uc& b) { return vand_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet16uc pand(const Packet16uc& a, const Packet16uc& b) { return vandq_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet4s pand(const Packet4s& a, const Packet4s& b) { return vand_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet8s pand(const Packet8s& a, const Packet8s& b) { return vandq_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet4us pand(const Packet4us& a, const Packet4us& b) { return vand_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet8us pand(const Packet8us& a, const Packet8us& b) { return vandq_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet2i pand(const Packet2i& a, const Packet2i& b) { return vand_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pand(const Packet4i& a, const Packet4i& b) { return vandq_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet2ui pand(const Packet2ui& a, const Packet2ui& b) { return vand_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet4ui pand(const Packet4ui& a, const Packet4ui& b) { return vandq_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet2l pand(const Packet2l& a, const Packet2l& b) { return vandq_s64(a,b); } template<> EIGEN_STRONG_INLINE Packet2ul pand(const Packet2ul& a, const Packet2ul& b) { return vandq_u64(a,b); } template<> EIGEN_STRONG_INLINE Packet2f por(const Packet2f& a, const Packet2f& b) { return vreinterpret_f32_u32(vorr_u32(vreinterpret_u32_f32(a),vreinterpret_u32_f32(b))); } template<> EIGEN_STRONG_INLINE Packet4f por(const Packet4f& a, const Packet4f& b) { return vreinterpretq_f32_u32(vorrq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b))); } template<> EIGEN_STRONG_INLINE Packet4c por(const Packet4c& a, const Packet4c& b) { return a | b; } template<> EIGEN_STRONG_INLINE Packet8c por(const Packet8c& a, const Packet8c& b) { return vorr_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet16c por(const Packet16c& a, const Packet16c& b) { return vorrq_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet4uc por(const Packet4uc& a, const Packet4uc& b) { return a | b; } template<> EIGEN_STRONG_INLINE Packet8uc por(const Packet8uc& a, const Packet8uc& b) { return vorr_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet16uc por(const Packet16uc& a, const Packet16uc& b) { return vorrq_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet4s por(const Packet4s& a, const Packet4s& b) { return vorr_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet8s por(const Packet8s& a, const Packet8s& b) { return vorrq_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet4us por(const Packet4us& a, const Packet4us& b) { return vorr_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet8us por(const Packet8us& a, const Packet8us& b) { return vorrq_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet2i por(const Packet2i& a, const Packet2i& b) { return vorr_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet4i por(const Packet4i& a, const Packet4i& b) { return vorrq_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet2ui por(const Packet2ui& a, const Packet2ui& b) { return vorr_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet4ui por(const Packet4ui& a, const Packet4ui& b) { return vorrq_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet2l por(const Packet2l& a, const Packet2l& b) { return vorrq_s64(a,b); } template<> EIGEN_STRONG_INLINE Packet2ul por(const Packet2ul& a, const Packet2ul& b) { return vorrq_u64(a,b); } template<> EIGEN_STRONG_INLINE Packet2f pxor(const Packet2f& a, const Packet2f& b) { return vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(a),vreinterpret_u32_f32(b))); } template<> EIGEN_STRONG_INLINE Packet4f pxor(const Packet4f& a, const Packet4f& b) { return vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b))); } template<> EIGEN_STRONG_INLINE Packet4c pxor(const Packet4c& a, const Packet4c& b) { return a ^ b; } template<> EIGEN_STRONG_INLINE Packet8c pxor(const Packet8c& a, const Packet8c& b) { return veor_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet16c pxor(const Packet16c& a, const Packet16c& b) { return veorq_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet4uc pxor(const Packet4uc& a, const Packet4uc& b) { return a ^ b; } template<> EIGEN_STRONG_INLINE Packet8uc pxor(const Packet8uc& a, const Packet8uc& b) { return veor_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet16uc pxor(const Packet16uc& a, const Packet16uc& b) { return veorq_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet4s pxor(const Packet4s& a, const Packet4s& b) { return veor_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet8s pxor(const Packet8s& a, const Packet8s& b) { return veorq_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet4us pxor(const Packet4us& a, const Packet4us& b) { return veor_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet8us pxor(const Packet8us& a, const Packet8us& b) { return veorq_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet2i pxor(const Packet2i& a, const Packet2i& b) { return veor_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pxor(const Packet4i& a, const Packet4i& b) { return veorq_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet2ui pxor(const Packet2ui& a, const Packet2ui& b) { return veor_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet4ui pxor(const Packet4ui& a, const Packet4ui& b) { return veorq_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet2l pxor(const Packet2l& a, const Packet2l& b) { return veorq_s64(a,b); } template<> EIGEN_STRONG_INLINE Packet2ul pxor(const Packet2ul& a, const Packet2ul& b) { return veorq_u64(a,b); } template<> EIGEN_STRONG_INLINE Packet2f pandnot(const Packet2f& a, const Packet2f& b) { return vreinterpret_f32_u32(vbic_u32(vreinterpret_u32_f32(a),vreinterpret_u32_f32(b))); } template<> EIGEN_STRONG_INLINE Packet4f pandnot(const Packet4f& a, const Packet4f& b) { return vreinterpretq_f32_u32(vbicq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b))); } template<> EIGEN_STRONG_INLINE Packet4c pandnot(const Packet4c& a, const Packet4c& b) { return a & ~b; } template<> EIGEN_STRONG_INLINE Packet8c pandnot(const Packet8c& a, const Packet8c& b) { return vbic_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet16c pandnot(const Packet16c& a, const Packet16c& b) { return vbicq_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet4uc pandnot(const Packet4uc& a, const Packet4uc& b) { return a & ~b; } template<> EIGEN_STRONG_INLINE Packet8uc pandnot(const Packet8uc& a, const Packet8uc& b) { return vbic_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet16uc pandnot(const Packet16uc& a, const Packet16uc& b) { return vbicq_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet4s pandnot(const Packet4s& a, const Packet4s& b) { return vbic_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet8s pandnot(const Packet8s& a, const Packet8s& b) { return vbicq_s16(a,b); } template<> EIGEN_STRONG_INLINE Packet4us pandnot(const Packet4us& a, const Packet4us& b) { return vbic_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet8us pandnot(const Packet8us& a, const Packet8us& b) { return vbicq_u16(a,b); } template<> EIGEN_STRONG_INLINE Packet2i pandnot(const Packet2i& a, const Packet2i& b) { return vbic_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pandnot(const Packet4i& a, const Packet4i& b) { return vbicq_s32(a,b); } template<> EIGEN_STRONG_INLINE Packet2ui pandnot(const Packet2ui& a, const Packet2ui& b) { return vbic_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet4ui pandnot(const Packet4ui& a, const Packet4ui& b) { return vbicq_u32(a,b); } template<> EIGEN_STRONG_INLINE Packet2l pandnot(const Packet2l& a, const Packet2l& b) { return vbicq_s64(a,b); } template<> EIGEN_STRONG_INLINE Packet2ul pandnot(const Packet2ul& a, const Packet2ul& b) { return vbicq_u64(a,b); } template EIGEN_STRONG_INLINE Packet4c parithmetic_shift_right(Packet4c& a) { return vget_lane_s32(vreinterpret_s32_s8(vshr_n_s8(vreinterpret_s8_s32(vdup_n_s32(a)), N)), 0); } template EIGEN_STRONG_INLINE Packet8c parithmetic_shift_right(Packet8c a) { return vshr_n_s8(a,N); } template EIGEN_STRONG_INLINE Packet16c parithmetic_shift_right(Packet16c a) { return vshrq_n_s8(a,N); } template EIGEN_STRONG_INLINE Packet4uc parithmetic_shift_right(Packet4uc& a) { return vget_lane_u32(vreinterpret_u32_u8(vshr_n_u8(vreinterpret_u8_u32(vdup_n_u32(a)), N)), 0); } template EIGEN_STRONG_INLINE Packet8uc parithmetic_shift_right(Packet8uc a) { return vshr_n_u8(a,N); } template EIGEN_STRONG_INLINE Packet16uc parithmetic_shift_right(Packet16uc a) { return vshrq_n_u8(a,N); } template EIGEN_STRONG_INLINE Packet4s parithmetic_shift_right(Packet4s a) { return vshr_n_s16(a,N); } template EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(Packet8s a) { return vshrq_n_s16(a,N); } template EIGEN_STRONG_INLINE Packet4us parithmetic_shift_right(Packet4us a) { return vshr_n_u16(a,N); } template EIGEN_STRONG_INLINE Packet8us parithmetic_shift_right(Packet8us a) { return vshrq_n_u16(a,N); } template EIGEN_STRONG_INLINE Packet2i parithmetic_shift_right(Packet2i a) { return vshr_n_s32(a,N); } template EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(Packet4i a) { return vshrq_n_s32(a,N); } template EIGEN_STRONG_INLINE Packet2ui parithmetic_shift_right(Packet2ui a) { return vshr_n_u32(a,N); } template EIGEN_STRONG_INLINE Packet4ui parithmetic_shift_right(Packet4ui a) { return vshrq_n_u32(a,N); } template EIGEN_STRONG_INLINE Packet2l parithmetic_shift_right(Packet2l a) { return vshrq_n_s64(a,N); } template EIGEN_STRONG_INLINE Packet2ul parithmetic_shift_right(Packet2ul a) { return vshrq_n_u64(a,N); } template EIGEN_STRONG_INLINE Packet4c plogical_shift_right(Packet4c& a) { return vget_lane_s32(vreinterpret_s32_u8(vshr_n_u8(vreinterpret_u8_s32(vdup_n_s32(a)), N)), 0); } template EIGEN_STRONG_INLINE Packet8c plogical_shift_right(Packet8c a) { return vreinterpret_s8_u8(vshr_n_u8(vreinterpret_u8_s8(a),N)); } template EIGEN_STRONG_INLINE Packet16c plogical_shift_right(Packet16c a) { return vreinterpretq_s8_u8(vshrq_n_u8(vreinterpretq_u8_s8(a),N)); } template EIGEN_STRONG_INLINE Packet4uc plogical_shift_right(Packet4uc& a) { return vget_lane_u32(vreinterpret_u32_s8(vshr_n_s8(vreinterpret_s8_u32(vdup_n_u32(a)), N)), 0); } template EIGEN_STRONG_INLINE Packet8uc plogical_shift_right(Packet8uc a) { return vshr_n_u8(a,N); } template EIGEN_STRONG_INLINE Packet16uc plogical_shift_right(Packet16uc a) { return vshrq_n_u8(a,N); } template EIGEN_STRONG_INLINE Packet4s plogical_shift_right(Packet4s a) { return vreinterpret_s16_u16(vshr_n_u16(vreinterpret_u16_s16(a),N)); } template EIGEN_STRONG_INLINE Packet8s plogical_shift_right(Packet8s a) { return vreinterpretq_s16_u16(vshrq_n_u16(vreinterpretq_u16_s16(a),N)); } template EIGEN_STRONG_INLINE Packet4us plogical_shift_right(Packet4us a) { return vshr_n_u16(a,N); } template EIGEN_STRONG_INLINE Packet8us plogical_shift_right(Packet8us a) { return vshrq_n_u16(a,N); } template EIGEN_STRONG_INLINE Packet2i plogical_shift_right(Packet2i a) { return vreinterpret_s32_u32(vshr_n_u32(vreinterpret_u32_s32(a),N)); } template EIGEN_STRONG_INLINE Packet4i plogical_shift_right(Packet4i a) { return vreinterpretq_s32_u32(vshrq_n_u32(vreinterpretq_u32_s32(a),N)); } template EIGEN_STRONG_INLINE Packet2ui plogical_shift_right(Packet2ui a) { return vshr_n_u32(a,N); } template EIGEN_STRONG_INLINE Packet4ui plogical_shift_right(Packet4ui a) { return vshrq_n_u32(a,N); } template EIGEN_STRONG_INLINE Packet2l plogical_shift_right(Packet2l a) { return vreinterpretq_s64_u64(vshrq_n_u64(vreinterpretq_u64_s64(a),N)); } template EIGEN_STRONG_INLINE Packet2ul plogical_shift_right(Packet2ul a) { return vshrq_n_u64(a,N); } template EIGEN_STRONG_INLINE Packet4c plogical_shift_left(Packet4c& a) { return vget_lane_s32(vreinterpret_s32_s8(vshl_n_s8(vreinterpret_s8_s32(vdup_n_s32(a)), N)), 0); } template EIGEN_STRONG_INLINE Packet8c plogical_shift_left(Packet8c a) { return vshl_n_s8(a,N); } template EIGEN_STRONG_INLINE Packet16c plogical_shift_left(Packet16c a) { return vshlq_n_s8(a,N); } template EIGEN_STRONG_INLINE Packet4uc plogical_shift_left(Packet4uc& a) { return vget_lane_u32(vreinterpret_u32_u8(vshl_n_u8(vreinterpret_u8_u32(vdup_n_u32(a)), N)), 0); } template EIGEN_STRONG_INLINE Packet8uc plogical_shift_left(Packet8uc a) { return vshl_n_u8(a,N); } template EIGEN_STRONG_INLINE Packet16uc plogical_shift_left(Packet16uc a) { return vshlq_n_u8(a,N); } template EIGEN_STRONG_INLINE Packet4s plogical_shift_left(Packet4s a) { return vshl_n_s16(a,N); } template EIGEN_STRONG_INLINE Packet8s plogical_shift_left(Packet8s a) { return vshlq_n_s16(a,N); } template EIGEN_STRONG_INLINE Packet4us plogical_shift_left(Packet4us a) { return vshl_n_u16(a,N); } template EIGEN_STRONG_INLINE Packet8us plogical_shift_left(Packet8us a) { return vshlq_n_u16(a,N); } template EIGEN_STRONG_INLINE Packet2i plogical_shift_left(Packet2i a) { return vshl_n_s32(a,N); } template EIGEN_STRONG_INLINE Packet4i plogical_shift_left(Packet4i a) { return vshlq_n_s32(a,N); } template EIGEN_STRONG_INLINE Packet2ui plogical_shift_left(Packet2ui a) { return vshl_n_u32(a,N); } template EIGEN_STRONG_INLINE Packet4ui plogical_shift_left(Packet4ui a) { return vshlq_n_u32(a,N); } template EIGEN_STRONG_INLINE Packet2l plogical_shift_left(Packet2l a) { return vshlq_n_s64(a,N); } template EIGEN_STRONG_INLINE Packet2ul plogical_shift_left(Packet2ul a) { return vshlq_n_u64(a,N); } template<> EIGEN_STRONG_INLINE Packet2f pload(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1_f32(from); } template<> EIGEN_STRONG_INLINE Packet4f pload(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f32(from); } template<> EIGEN_STRONG_INLINE Packet4c pload(const int8_t* from) { Packet4c res; memcpy(&res, from, sizeof(Packet4c)); return res; } template<> EIGEN_STRONG_INLINE Packet8c pload(const int8_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1_s8(from); } template<> EIGEN_STRONG_INLINE Packet16c pload(const int8_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s8(from); } template<> EIGEN_STRONG_INLINE Packet4uc pload(const uint8_t* from) { Packet4uc res; memcpy(&res, from, sizeof(Packet4uc)); return res; } template<> EIGEN_STRONG_INLINE Packet8uc pload(const uint8_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1_u8(from); } template<> EIGEN_STRONG_INLINE Packet16uc pload(const uint8_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u8(from); } template<> EIGEN_STRONG_INLINE Packet4s pload(const int16_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1_s16(from); } template<> EIGEN_STRONG_INLINE Packet8s pload(const int16_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s16(from); } template<> EIGEN_STRONG_INLINE Packet4us pload(const uint16_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1_u16(from); } template<> EIGEN_STRONG_INLINE Packet8us pload(const uint16_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u16(from); } template<> EIGEN_STRONG_INLINE Packet2i pload(const int32_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1_s32(from); } template<> EIGEN_STRONG_INLINE Packet4i pload(const int32_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s32(from); } template<> EIGEN_STRONG_INLINE Packet2ui pload(const uint32_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1_u32(from); } template<> EIGEN_STRONG_INLINE Packet4ui pload(const uint32_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u32(from); } template<> EIGEN_STRONG_INLINE Packet2l pload(const int64_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s64(from); } template<> EIGEN_STRONG_INLINE Packet2ul pload(const uint64_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u64(from); } template<> EIGEN_STRONG_INLINE Packet2f ploadu(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1_f32(from); } template<> EIGEN_STRONG_INLINE Packet4f ploadu(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f32(from); } template<> EIGEN_STRONG_INLINE Packet4c ploadu(const int8_t* from) { Packet4c res; memcpy(&res, from, sizeof(Packet4c)); return res; } template<> EIGEN_STRONG_INLINE Packet8c ploadu(const int8_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1_s8(from); } template<> EIGEN_STRONG_INLINE Packet16c ploadu(const int8_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s8(from); } template<> EIGEN_STRONG_INLINE Packet4uc ploadu(const uint8_t* from) { Packet4uc res; memcpy(&res, from, sizeof(Packet4uc)); return res; } template<> EIGEN_STRONG_INLINE Packet8uc ploadu(const uint8_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1_u8(from); } template<> EIGEN_STRONG_INLINE Packet16uc ploadu(const uint8_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_u8(from); } template<> EIGEN_STRONG_INLINE Packet4s ploadu(const int16_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1_s16(from); } template<> EIGEN_STRONG_INLINE Packet8s ploadu(const int16_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s16(from); } template<> EIGEN_STRONG_INLINE Packet4us ploadu(const uint16_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1_u16(from); } template<> EIGEN_STRONG_INLINE Packet8us ploadu(const uint16_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_u16(from); } template<> EIGEN_STRONG_INLINE Packet2i ploadu(const int32_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1_s32(from); } template<> EIGEN_STRONG_INLINE Packet4i ploadu(const int32_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s32(from); } template<> EIGEN_STRONG_INLINE Packet2ui ploadu(const uint32_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1_u32(from); } template<> EIGEN_STRONG_INLINE Packet4ui ploadu(const uint32_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_u32(from); } template<> EIGEN_STRONG_INLINE Packet2l ploadu(const int64_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s64(from); } template<> EIGEN_STRONG_INLINE Packet2ul ploadu(const uint64_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_u64(from); } template<> EIGEN_STRONG_INLINE Packet2f ploaddup(const float* from) { return vld1_dup_f32(from); } template<> EIGEN_STRONG_INLINE Packet4f ploaddup(const float* from) { return vcombine_f32(vld1_dup_f32(from), vld1_dup_f32(from+1)); } template<> EIGEN_STRONG_INLINE Packet4c ploaddup(const int8_t* from) { const int8x8_t a = vreinterpret_s8_s32(vdup_n_s32(pload(from))); return vget_lane_s32(vreinterpret_s32_s8(vzip_s8(a,a).val[0]), 0); } template<> EIGEN_STRONG_INLINE Packet8c ploaddup(const int8_t* from) { const int8x8_t a = vld1_s8(from); return vzip_s8(a,a).val[0]; } template<> EIGEN_STRONG_INLINE Packet16c ploaddup(const int8_t* from) { const int8x8_t a = vld1_s8(from); const int8x8x2_t b = vzip_s8(a,a); return vcombine_s8(b.val[0], b.val[1]); } template<> EIGEN_STRONG_INLINE Packet4uc ploaddup(const uint8_t* from) { const uint8x8_t a = vreinterpret_u8_u32(vdup_n_u32(pload(from))); return vget_lane_u32(vreinterpret_u32_u8(vzip_u8(a,a).val[0]), 0); } template<> EIGEN_STRONG_INLINE Packet8uc ploaddup(const uint8_t* from) { const uint8x8_t a = vld1_u8(from); return vzip_u8(a,a).val[0]; } template<> EIGEN_STRONG_INLINE Packet16uc ploaddup(const uint8_t* from) { const uint8x8_t a = vld1_u8(from); const uint8x8x2_t b = vzip_u8(a,a); return vcombine_u8(b.val[0], b.val[1]); } template<> EIGEN_STRONG_INLINE Packet4s ploaddup(const int16_t* from) { return vreinterpret_s16_u32(vzip_u32(vreinterpret_u32_s16(vld1_dup_s16(from)), vreinterpret_u32_s16(vld1_dup_s16(from+1))).val[0]); } template<> EIGEN_STRONG_INLINE Packet8s ploaddup(const int16_t* from) { const int16x4_t a = vld1_s16(from); const int16x4x2_t b = vzip_s16(a,a); return vcombine_s16(b.val[0], b.val[1]); } template<> EIGEN_STRONG_INLINE Packet4us ploaddup(const uint16_t* from) { return vreinterpret_u16_u32(vzip_u32(vreinterpret_u32_u16(vld1_dup_u16(from)), vreinterpret_u32_u16(vld1_dup_u16(from+1))).val[0]); } template<> EIGEN_STRONG_INLINE Packet8us ploaddup(const uint16_t* from) { const uint16x4_t a = vld1_u16(from); const uint16x4x2_t b = vzip_u16(a,a); return vcombine_u16(b.val[0], b.val[1]); } template<> EIGEN_STRONG_INLINE Packet2i ploaddup(const int32_t* from) { return vld1_dup_s32(from); } template<> EIGEN_STRONG_INLINE Packet4i ploaddup(const int32_t* from) { return vcombine_s32(vld1_dup_s32(from), vld1_dup_s32(from+1)); } template<> EIGEN_STRONG_INLINE Packet2ui ploaddup(const uint32_t* from) { return vld1_dup_u32(from); } template<> EIGEN_STRONG_INLINE Packet4ui ploaddup(const uint32_t* from) { return vcombine_u32(vld1_dup_u32(from), vld1_dup_u32(from+1)); } template<> EIGEN_STRONG_INLINE Packet2l ploaddup(const int64_t* from) { return vld1q_dup_s64(from); } template<> EIGEN_STRONG_INLINE Packet2ul ploaddup(const uint64_t* from) { return vld1q_dup_u64(from); } template<> EIGEN_STRONG_INLINE Packet4f ploadquad(const float* from) { return vld1q_dup_f32(from); } template<> EIGEN_STRONG_INLINE Packet4c ploadquad(const int8_t* from) { return vget_lane_s32(vreinterpret_s32_s8(vld1_dup_s8(from)), 0); } template<> EIGEN_STRONG_INLINE Packet8c ploadquad(const int8_t* from) { return vreinterpret_s8_u32(vzip_u32( vreinterpret_u32_s8(vld1_dup_s8(from)), vreinterpret_u32_s8(vld1_dup_s8(from+1))).val[0]); } template<> EIGEN_STRONG_INLINE Packet16c ploadquad(const int8_t* from) { const int8x8_t a = vreinterpret_s8_u32(vzip_u32( vreinterpret_u32_s8(vld1_dup_s8(from)), vreinterpret_u32_s8(vld1_dup_s8(from+1))).val[0]); const int8x8_t b = vreinterpret_s8_u32(vzip_u32( vreinterpret_u32_s8(vld1_dup_s8(from+2)), vreinterpret_u32_s8(vld1_dup_s8(from+3))).val[0]); return vcombine_s8(a,b); } template<> EIGEN_STRONG_INLINE Packet4uc ploadquad(const uint8_t* from) { return vget_lane_u32(vreinterpret_u32_u8(vld1_dup_u8(from)), 0); } template<> EIGEN_STRONG_INLINE Packet8uc ploadquad(const uint8_t* from) { return vreinterpret_u8_u32(vzip_u32( vreinterpret_u32_u8(vld1_dup_u8(from)), vreinterpret_u32_u8(vld1_dup_u8(from+1))).val[0]); } template<> EIGEN_STRONG_INLINE Packet16uc ploadquad(const uint8_t* from) { const uint8x8_t a = vreinterpret_u8_u32(vzip_u32( vreinterpret_u32_u8(vld1_dup_u8(from)), vreinterpret_u32_u8(vld1_dup_u8(from+1))).val[0]); const uint8x8_t b = vreinterpret_u8_u32(vzip_u32( vreinterpret_u32_u8(vld1_dup_u8(from+2)), vreinterpret_u32_u8(vld1_dup_u8(from+3))).val[0]); return vcombine_u8(a,b); } template<> EIGEN_STRONG_INLINE Packet8s ploadquad(const int16_t* from) { return vcombine_s16(vld1_dup_s16(from), vld1_dup_s16(from+1)); } template<> EIGEN_STRONG_INLINE Packet8us ploadquad(const uint16_t* from) { return vcombine_u16(vld1_dup_u16(from), vld1_dup_u16(from+1)); } template<> EIGEN_STRONG_INLINE Packet4i ploadquad(const int32_t* from) { return vld1q_dup_s32(from); } template<> EIGEN_STRONG_INLINE Packet4ui ploadquad(const uint32_t* from) { return vld1q_dup_u32(from); } template<> EIGEN_STRONG_INLINE void pstore(float* to, const Packet2f& from) { EIGEN_DEBUG_ALIGNED_STORE vst1_f32(to,from); } template<> EIGEN_STRONG_INLINE void pstore(float* to, const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_f32(to,from); } template<> EIGEN_STRONG_INLINE void pstore(int8_t* to, const Packet4c& from) { memcpy(to, &from, sizeof(from)); } template<> EIGEN_STRONG_INLINE void pstore(int8_t* to, const Packet8c& from) { EIGEN_DEBUG_ALIGNED_STORE vst1_s8(to,from); } template<> EIGEN_STRONG_INLINE void pstore(int8_t* to, const Packet16c& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_s8(to,from); } template<> EIGEN_STRONG_INLINE void pstore(uint8_t* to, const Packet4uc& from) { memcpy(to, &from, sizeof(from)); } template<> EIGEN_STRONG_INLINE void pstore(uint8_t* to, const Packet8uc& from) { EIGEN_DEBUG_ALIGNED_STORE vst1_u8(to,from); } template<> EIGEN_STRONG_INLINE void pstore(uint8_t* to, const Packet16uc& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_u8(to,from); } template<> EIGEN_STRONG_INLINE void pstore(int16_t* to, const Packet4s& from) { EIGEN_DEBUG_ALIGNED_STORE vst1_s16(to,from); } template<> EIGEN_STRONG_INLINE void pstore(int16_t* to, const Packet8s& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_s16(to,from); } template<> EIGEN_STRONG_INLINE void pstore(uint16_t* to, const Packet4us& from) { EIGEN_DEBUG_ALIGNED_STORE vst1_u16(to,from); } template<> EIGEN_STRONG_INLINE void pstore(uint16_t* to, const Packet8us& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_u16(to,from); } template<> EIGEN_STRONG_INLINE void pstore(int32_t* to, const Packet2i& from) { EIGEN_DEBUG_ALIGNED_STORE vst1_s32(to,from); } template<> EIGEN_STRONG_INLINE void pstore(int32_t* to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_s32(to,from); } template<> EIGEN_STRONG_INLINE void pstore(uint32_t* to, const Packet2ui& from) { EIGEN_DEBUG_ALIGNED_STORE vst1_u32(to,from); } template<> EIGEN_STRONG_INLINE void pstore(uint32_t* to, const Packet4ui& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_u32(to,from); } template<> EIGEN_STRONG_INLINE void pstore(int64_t* to, const Packet2l& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_s64(to,from); } template<> EIGEN_STRONG_INLINE void pstore(uint64_t* to, const Packet2ul& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_u64(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet2f& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1_f32(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_f32(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(int8_t* to, const Packet4c& from) { memcpy(to, &from, sizeof(from)); } template<> EIGEN_STRONG_INLINE void pstoreu(int8_t* to, const Packet8c& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1_s8(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(int8_t* to, const Packet16c& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s8(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(uint8_t* to, const Packet4uc& from) { memcpy(to, &from, sizeof(from)); } template<> EIGEN_STRONG_INLINE void pstoreu(uint8_t* to, const Packet8uc& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1_u8(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(uint8_t* to, const Packet16uc& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_u8(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(int16_t* to, const Packet4s& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1_s16(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(int16_t* to, const Packet8s& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s16(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(uint16_t* to, const Packet4us& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1_u16(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(uint16_t* to, const Packet8us& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_u16(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(int32_t* to, const Packet2i& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1_s32(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(int32_t* to, const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s32(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(uint32_t* to, const Packet2ui& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1_u32(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(uint32_t* to, const Packet4ui& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_u32(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(int64_t* to, const Packet2l& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s64(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(uint64_t* to, const Packet2ul& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_u64(to,from); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f pgather(const float* from, Index stride) { Packet2f res = vld1_dup_f32(from); res = vld1_lane_f32(from + 1*stride, res, 1); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pgather(const float* from, Index stride) { Packet4f res = vld1q_dup_f32(from); res = vld1q_lane_f32(from + 1*stride, res, 1); res = vld1q_lane_f32(from + 2*stride, res, 2); res = vld1q_lane_f32(from + 3*stride, res, 3); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c pgather(const int8_t* from, Index stride) { Packet4c res; for (int i = 0; i != 4; i++) reinterpret_cast(&res)[i] = *(from + i * stride); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c pgather(const int8_t* from, Index stride) { Packet8c res = vld1_dup_s8(from); res = vld1_lane_s8(from + 1*stride, res, 1); res = vld1_lane_s8(from + 2*stride, res, 2); res = vld1_lane_s8(from + 3*stride, res, 3); res = vld1_lane_s8(from + 4*stride, res, 4); res = vld1_lane_s8(from + 5*stride, res, 5); res = vld1_lane_s8(from + 6*stride, res, 6); res = vld1_lane_s8(from + 7*stride, res, 7); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pgather(const int8_t* from, Index stride) { Packet16c res = vld1q_dup_s8(from); res = vld1q_lane_s8(from + 1*stride, res, 1); res = vld1q_lane_s8(from + 2*stride, res, 2); res = vld1q_lane_s8(from + 3*stride, res, 3); res = vld1q_lane_s8(from + 4*stride, res, 4); res = vld1q_lane_s8(from + 5*stride, res, 5); res = vld1q_lane_s8(from + 6*stride, res, 6); res = vld1q_lane_s8(from + 7*stride, res, 7); res = vld1q_lane_s8(from + 8*stride, res, 8); res = vld1q_lane_s8(from + 9*stride, res, 9); res = vld1q_lane_s8(from + 10*stride, res, 10); res = vld1q_lane_s8(from + 11*stride, res, 11); res = vld1q_lane_s8(from + 12*stride, res, 12); res = vld1q_lane_s8(from + 13*stride, res, 13); res = vld1q_lane_s8(from + 14*stride, res, 14); res = vld1q_lane_s8(from + 15*stride, res, 15); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc pgather(const uint8_t* from, Index stride) { Packet4uc res; for (int i = 0; i != 4; i++) reinterpret_cast(&res)[i] = *(from + i * stride); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc pgather(const uint8_t* from, Index stride) { Packet8uc res = vld1_dup_u8(from); res = vld1_lane_u8(from + 1*stride, res, 1); res = vld1_lane_u8(from + 2*stride, res, 2); res = vld1_lane_u8(from + 3*stride, res, 3); res = vld1_lane_u8(from + 4*stride, res, 4); res = vld1_lane_u8(from + 5*stride, res, 5); res = vld1_lane_u8(from + 6*stride, res, 6); res = vld1_lane_u8(from + 7*stride, res, 7); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pgather(const uint8_t* from, Index stride) { Packet16uc res = vld1q_dup_u8(from); res = vld1q_lane_u8(from + 1*stride, res, 1); res = vld1q_lane_u8(from + 2*stride, res, 2); res = vld1q_lane_u8(from + 3*stride, res, 3); res = vld1q_lane_u8(from + 4*stride, res, 4); res = vld1q_lane_u8(from + 5*stride, res, 5); res = vld1q_lane_u8(from + 6*stride, res, 6); res = vld1q_lane_u8(from + 7*stride, res, 7); res = vld1q_lane_u8(from + 8*stride, res, 8); res = vld1q_lane_u8(from + 9*stride, res, 9); res = vld1q_lane_u8(from + 10*stride, res, 10); res = vld1q_lane_u8(from + 11*stride, res, 11); res = vld1q_lane_u8(from + 12*stride, res, 12); res = vld1q_lane_u8(from + 13*stride, res, 13); res = vld1q_lane_u8(from + 14*stride, res, 14); res = vld1q_lane_u8(from + 15*stride, res, 15); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s pgather(const int16_t* from, Index stride) { Packet4s res = vld1_dup_s16(from); res = vld1_lane_s16(from + 1*stride, res, 1); res = vld1_lane_s16(from + 2*stride, res, 2); res = vld1_lane_s16(from + 3*stride, res, 3); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pgather(const int16_t* from, Index stride) { Packet8s res = vld1q_dup_s16(from); res = vld1q_lane_s16(from + 1*stride, res, 1); res = vld1q_lane_s16(from + 2*stride, res, 2); res = vld1q_lane_s16(from + 3*stride, res, 3); res = vld1q_lane_s16(from + 4*stride, res, 4); res = vld1q_lane_s16(from + 5*stride, res, 5); res = vld1q_lane_s16(from + 6*stride, res, 6); res = vld1q_lane_s16(from + 7*stride, res, 7); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us pgather(const uint16_t* from, Index stride) { Packet4us res = vld1_dup_u16(from); res = vld1_lane_u16(from + 1*stride, res, 1); res = vld1_lane_u16(from + 2*stride, res, 2); res = vld1_lane_u16(from + 3*stride, res, 3); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pgather(const uint16_t* from, Index stride) { Packet8us res = vld1q_dup_u16(from); res = vld1q_lane_u16(from + 1*stride, res, 1); res = vld1q_lane_u16(from + 2*stride, res, 2); res = vld1q_lane_u16(from + 3*stride, res, 3); res = vld1q_lane_u16(from + 4*stride, res, 4); res = vld1q_lane_u16(from + 5*stride, res, 5); res = vld1q_lane_u16(from + 6*stride, res, 6); res = vld1q_lane_u16(from + 7*stride, res, 7); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i pgather(const int32_t* from, Index stride) { Packet2i res = vld1_dup_s32(from); res = vld1_lane_s32(from + 1*stride, res, 1); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pgather(const int32_t* from, Index stride) { Packet4i res = vld1q_dup_s32(from); res = vld1q_lane_s32(from + 1*stride, res, 1); res = vld1q_lane_s32(from + 2*stride, res, 2); res = vld1q_lane_s32(from + 3*stride, res, 3); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui pgather(const uint32_t* from, Index stride) { Packet2ui res = vld1_dup_u32(from); res = vld1_lane_u32(from + 1*stride, res, 1); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pgather(const uint32_t* from, Index stride) { Packet4ui res = vld1q_dup_u32(from); res = vld1q_lane_u32(from + 1*stride, res, 1); res = vld1q_lane_u32(from + 2*stride, res, 2); res = vld1q_lane_u32(from + 3*stride, res, 3); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pgather(const int64_t* from, Index stride) { Packet2l res = vld1q_dup_s64(from); res = vld1q_lane_s64(from + 1*stride, res, 1); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pgather(const uint64_t* from, Index stride) { Packet2ul res = vld1q_dup_u64(from); res = vld1q_lane_u64(from + 1*stride, res, 1); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(float* to, const Packet2f& from, Index stride) { vst1_lane_f32(to + stride*0, from, 0); vst1_lane_f32(to + stride*1, from, 1); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(float* to, const Packet4f& from, Index stride) { vst1q_lane_f32(to + stride*0, from, 0); vst1q_lane_f32(to + stride*1, from, 1); vst1q_lane_f32(to + stride*2, from, 2); vst1q_lane_f32(to + stride*3, from, 3); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int8_t* to, const Packet4c& from, Index stride) { for (int i = 0; i != 4; i++) *(to + i * stride) = reinterpret_cast(&from)[i]; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int8_t* to, const Packet8c& from, Index stride) { vst1_lane_s8(to + stride*0, from, 0); vst1_lane_s8(to + stride*1, from, 1); vst1_lane_s8(to + stride*2, from, 2); vst1_lane_s8(to + stride*3, from, 3); vst1_lane_s8(to + stride*4, from, 4); vst1_lane_s8(to + stride*5, from, 5); vst1_lane_s8(to + stride*6, from, 6); vst1_lane_s8(to + stride*7, from, 7); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int8_t* to, const Packet16c& from, Index stride) { vst1q_lane_s8(to + stride*0, from, 0); vst1q_lane_s8(to + stride*1, from, 1); vst1q_lane_s8(to + stride*2, from, 2); vst1q_lane_s8(to + stride*3, from, 3); vst1q_lane_s8(to + stride*4, from, 4); vst1q_lane_s8(to + stride*5, from, 5); vst1q_lane_s8(to + stride*6, from, 6); vst1q_lane_s8(to + stride*7, from, 7); vst1q_lane_s8(to + stride*8, from, 8); vst1q_lane_s8(to + stride*9, from, 9); vst1q_lane_s8(to + stride*10, from, 10); vst1q_lane_s8(to + stride*11, from, 11); vst1q_lane_s8(to + stride*12, from, 12); vst1q_lane_s8(to + stride*13, from, 13); vst1q_lane_s8(to + stride*14, from, 14); vst1q_lane_s8(to + stride*15, from, 15); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint8_t* to, const Packet4uc& from, Index stride) { for (int i = 0; i != 4; i++) *(to + i * stride) = reinterpret_cast(&from)[i]; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint8_t* to, const Packet8uc& from, Index stride) { vst1_lane_u8(to + stride*0, from, 0); vst1_lane_u8(to + stride*1, from, 1); vst1_lane_u8(to + stride*2, from, 2); vst1_lane_u8(to + stride*3, from, 3); vst1_lane_u8(to + stride*4, from, 4); vst1_lane_u8(to + stride*5, from, 5); vst1_lane_u8(to + stride*6, from, 6); vst1_lane_u8(to + stride*7, from, 7); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint8_t* to, const Packet16uc& from, Index stride) { vst1q_lane_u8(to + stride*0, from, 0); vst1q_lane_u8(to + stride*1, from, 1); vst1q_lane_u8(to + stride*2, from, 2); vst1q_lane_u8(to + stride*3, from, 3); vst1q_lane_u8(to + stride*4, from, 4); vst1q_lane_u8(to + stride*5, from, 5); vst1q_lane_u8(to + stride*6, from, 6); vst1q_lane_u8(to + stride*7, from, 7); vst1q_lane_u8(to + stride*8, from, 8); vst1q_lane_u8(to + stride*9, from, 9); vst1q_lane_u8(to + stride*10, from, 10); vst1q_lane_u8(to + stride*11, from, 11); vst1q_lane_u8(to + stride*12, from, 12); vst1q_lane_u8(to + stride*13, from, 13); vst1q_lane_u8(to + stride*14, from, 14); vst1q_lane_u8(to + stride*15, from, 15); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int16_t* to, const Packet4s& from, Index stride) { vst1_lane_s16(to + stride*0, from, 0); vst1_lane_s16(to + stride*1, from, 1); vst1_lane_s16(to + stride*2, from, 2); vst1_lane_s16(to + stride*3, from, 3); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int16_t* to, const Packet8s& from, Index stride) { vst1q_lane_s16(to + stride*0, from, 0); vst1q_lane_s16(to + stride*1, from, 1); vst1q_lane_s16(to + stride*2, from, 2); vst1q_lane_s16(to + stride*3, from, 3); vst1q_lane_s16(to + stride*4, from, 4); vst1q_lane_s16(to + stride*5, from, 5); vst1q_lane_s16(to + stride*6, from, 6); vst1q_lane_s16(to + stride*7, from, 7); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint16_t* to, const Packet4us& from, Index stride) { vst1_lane_u16(to + stride*0, from, 0); vst1_lane_u16(to + stride*1, from, 1); vst1_lane_u16(to + stride*2, from, 2); vst1_lane_u16(to + stride*3, from, 3); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint16_t* to, const Packet8us& from, Index stride) { vst1q_lane_u16(to + stride*0, from, 0); vst1q_lane_u16(to + stride*1, from, 1); vst1q_lane_u16(to + stride*2, from, 2); vst1q_lane_u16(to + stride*3, from, 3); vst1q_lane_u16(to + stride*4, from, 4); vst1q_lane_u16(to + stride*5, from, 5); vst1q_lane_u16(to + stride*6, from, 6); vst1q_lane_u16(to + stride*7, from, 7); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int32_t* to, const Packet2i& from, Index stride) { vst1_lane_s32(to + stride*0, from, 0); vst1_lane_s32(to + stride*1, from, 1); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int32_t* to, const Packet4i& from, Index stride) { vst1q_lane_s32(to + stride*0, from, 0); vst1q_lane_s32(to + stride*1, from, 1); vst1q_lane_s32(to + stride*2, from, 2); vst1q_lane_s32(to + stride*3, from, 3); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint32_t* to, const Packet2ui& from, Index stride) { vst1_lane_u32(to + stride*0, from, 0); vst1_lane_u32(to + stride*1, from, 1); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint32_t* to, const Packet4ui& from, Index stride) { vst1q_lane_u32(to + stride*0, from, 0); vst1q_lane_u32(to + stride*1, from, 1); vst1q_lane_u32(to + stride*2, from, 2); vst1q_lane_u32(to + stride*3, from, 3); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int64_t* to, const Packet2l& from, Index stride) { vst1q_lane_s64(to + stride*0, from, 0); vst1q_lane_s64(to + stride*1, from, 1); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint64_t* to, const Packet2ul& from, Index stride) { vst1q_lane_u64(to + stride*0, from, 0); vst1q_lane_u64(to + stride*1, from, 1); } template<> EIGEN_STRONG_INLINE void prefetch(const float* addr) { EIGEN_ARM_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE void prefetch(const int8_t* addr) { EIGEN_ARM_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE void prefetch(const uint8_t* addr) { EIGEN_ARM_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE void prefetch(const int16_t* addr) { EIGEN_ARM_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE void prefetch(const uint16_t* addr) { EIGEN_ARM_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE void prefetch(const int32_t* addr) { EIGEN_ARM_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE void prefetch(const uint32_t* addr) { EIGEN_ARM_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE void prefetch(const int64_t* addr) { EIGEN_ARM_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE void prefetch(const uint64_t* addr) { EIGEN_ARM_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE float pfirst(const Packet2f& a) { return vget_lane_f32(a,0); } template<> EIGEN_STRONG_INLINE float pfirst(const Packet4f& a) { return vgetq_lane_f32(a,0); } template<> EIGEN_STRONG_INLINE int8_t pfirst(const Packet4c& a) { return static_cast(a & 0xff); } template<> EIGEN_STRONG_INLINE int8_t pfirst(const Packet8c& a) { return vget_lane_s8(a,0); } template<> EIGEN_STRONG_INLINE int8_t pfirst(const Packet16c& a) { return vgetq_lane_s8(a,0); } template<> EIGEN_STRONG_INLINE uint8_t pfirst(const Packet4uc& a) { return static_cast(a & 0xff); } template<> EIGEN_STRONG_INLINE uint8_t pfirst(const Packet8uc& a) { return vget_lane_u8(a,0); } template<> EIGEN_STRONG_INLINE uint8_t pfirst(const Packet16uc& a) { return vgetq_lane_u8(a,0); } template<> EIGEN_STRONG_INLINE int16_t pfirst(const Packet4s& a) { return vget_lane_s16(a,0); } template<> EIGEN_STRONG_INLINE int16_t pfirst(const Packet8s& a) { return vgetq_lane_s16(a,0); } template<> EIGEN_STRONG_INLINE uint16_t pfirst(const Packet4us& a) { return vget_lane_u16(a,0); } template<> EIGEN_STRONG_INLINE uint16_t pfirst(const Packet8us& a) { return vgetq_lane_u16(a,0); } template<> EIGEN_STRONG_INLINE int32_t pfirst(const Packet2i& a) { return vget_lane_s32(a,0); } template<> EIGEN_STRONG_INLINE int32_t pfirst(const Packet4i& a) { return vgetq_lane_s32(a,0); } template<> EIGEN_STRONG_INLINE uint32_t pfirst(const Packet2ui& a) { return vget_lane_u32(a,0); } template<> EIGEN_STRONG_INLINE uint32_t pfirst(const Packet4ui& a) { return vgetq_lane_u32(a,0); } template<> EIGEN_STRONG_INLINE int64_t pfirst(const Packet2l& a) { return vgetq_lane_s64(a,0); } template<> EIGEN_STRONG_INLINE uint64_t pfirst(const Packet2ul& a) { return vgetq_lane_u64(a,0); } template<> EIGEN_STRONG_INLINE Packet2f preverse(const Packet2f& a) { return vrev64_f32(a); } template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) { const float32x4_t a_r64 = vrev64q_f32(a); return vcombine_f32(vget_high_f32(a_r64), vget_low_f32(a_r64)); } template<> EIGEN_STRONG_INLINE Packet4c preverse(const Packet4c& a) { return vget_lane_s32(vreinterpret_s32_s8(vrev64_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0); } template<> EIGEN_STRONG_INLINE Packet8c preverse(const Packet8c& a) { return vrev64_s8(a); } template<> EIGEN_STRONG_INLINE Packet16c preverse(const Packet16c& a) { const int8x16_t a_r64 = vrev64q_s8(a); return vcombine_s8(vget_high_s8(a_r64), vget_low_s8(a_r64)); } template<> EIGEN_STRONG_INLINE Packet4uc preverse(const Packet4uc& a) { return vget_lane_u32(vreinterpret_u32_u8(vrev64_u8(vreinterpret_u8_u32(vdup_n_u32(a)))), 0); } template<> EIGEN_STRONG_INLINE Packet8uc preverse(const Packet8uc& a) { return vrev64_u8(a); } template<> EIGEN_STRONG_INLINE Packet16uc preverse(const Packet16uc& a) { const uint8x16_t a_r64 = vrev64q_u8(a); return vcombine_u8(vget_high_u8(a_r64), vget_low_u8(a_r64)); } template<> EIGEN_STRONG_INLINE Packet4s preverse(const Packet4s& a) { return vrev64_s16(a); } template<> EIGEN_STRONG_INLINE Packet8s preverse(const Packet8s& a) { const int16x8_t a_r64 = vrev64q_s16(a); return vcombine_s16(vget_high_s16(a_r64), vget_low_s16(a_r64)); } template<> EIGEN_STRONG_INLINE Packet4us preverse(const Packet4us& a) { return vrev64_u16(a); } template<> EIGEN_STRONG_INLINE Packet8us preverse(const Packet8us& a) { const uint16x8_t a_r64 = vrev64q_u16(a); return vcombine_u16(vget_high_u16(a_r64), vget_low_u16(a_r64)); } template<> EIGEN_STRONG_INLINE Packet2i preverse(const Packet2i& a) { return vrev64_s32(a); } template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) { const int32x4_t a_r64 = vrev64q_s32(a); return vcombine_s32(vget_high_s32(a_r64), vget_low_s32(a_r64)); } template<> EIGEN_STRONG_INLINE Packet2ui preverse(const Packet2ui& a) { return vrev64_u32(a); } template<> EIGEN_STRONG_INLINE Packet4ui preverse(const Packet4ui& a) { const uint32x4_t a_r64 = vrev64q_u32(a); return vcombine_u32(vget_high_u32(a_r64), vget_low_u32(a_r64)); } template<> EIGEN_STRONG_INLINE Packet2l preverse(const Packet2l& a) { return vcombine_s64(vget_high_s64(a), vget_low_s64(a)); } template<> EIGEN_STRONG_INLINE Packet2ul preverse(const Packet2ul& a) { return vcombine_u64(vget_high_u64(a), vget_low_u64(a)); } template<> EIGEN_STRONG_INLINE Packet2f pabs(const Packet2f& a) { return vabs_f32(a); } template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { return vabsq_f32(a); } template<> EIGEN_STRONG_INLINE Packet4c pabs(const Packet4c& a) { return vget_lane_s32(vreinterpret_s32_s8(vabs_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0); } template<> EIGEN_STRONG_INLINE Packet8c pabs(const Packet8c& a) { return vabs_s8(a); } template<> EIGEN_STRONG_INLINE Packet16c pabs(const Packet16c& a) { return vabsq_s8(a); } template<> EIGEN_STRONG_INLINE Packet4uc pabs(const Packet4uc& a) { return a; } template<> EIGEN_STRONG_INLINE Packet8uc pabs(const Packet8uc& a) { return a; } template<> EIGEN_STRONG_INLINE Packet16uc pabs(const Packet16uc& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4s pabs(const Packet4s& a) { return vabs_s16(a); } template<> EIGEN_STRONG_INLINE Packet8s pabs(const Packet8s& a) { return vabsq_s16(a); } template<> EIGEN_STRONG_INLINE Packet4us pabs(const Packet4us& a) { return a; } template<> EIGEN_STRONG_INLINE Packet8us pabs(const Packet8us& a) { return a; } template<> EIGEN_STRONG_INLINE Packet2i pabs(const Packet2i& a) { return vabs_s32(a); } template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { return vabsq_s32(a); } template<> EIGEN_STRONG_INLINE Packet2ui pabs(const Packet2ui& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4ui pabs(const Packet4ui& a) { return a; } template<> EIGEN_STRONG_INLINE Packet2l pabs(const Packet2l& a) { #if EIGEN_ARCH_ARM64 return vabsq_s64(a); #else return vcombine_s64( vdup_n_s64((std::abs)(vgetq_lane_s64(a, 0))), vdup_n_s64((std::abs)(vgetq_lane_s64(a, 1)))); #endif } template<> EIGEN_STRONG_INLINE Packet2ul pabs(const Packet2ul& a) { return a; } template<> EIGEN_STRONG_INLINE Packet2f pfrexp(const Packet2f& a, Packet2f& exponent) { return pfrexp_generic(a,exponent); } template<> EIGEN_STRONG_INLINE Packet4f pfrexp(const Packet4f& a, Packet4f& exponent) { return pfrexp_generic(a,exponent); } template<> EIGEN_STRONG_INLINE Packet2f pldexp(const Packet2f& a, const Packet2f& exponent) { return pldexp_generic(a,exponent); } template<> EIGEN_STRONG_INLINE Packet4f pldexp(const Packet4f& a, const Packet4f& exponent) { return pldexp_generic(a,exponent); } template<> EIGEN_STRONG_INLINE float predux(const Packet2f& a) { return vget_lane_f32(vpadd_f32(a,a), 0); } template<> EIGEN_STRONG_INLINE float predux(const Packet4f& a) { const float32x2_t sum = vadd_f32(vget_low_f32(a), vget_high_f32(a)); return vget_lane_f32(vpadd_f32(sum, sum), 0); } template<> EIGEN_STRONG_INLINE int8_t predux(const Packet4c& a) { const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a)); int8x8_t sum = vpadd_s8(a_dup, a_dup); sum = vpadd_s8(sum, sum); return vget_lane_s8(sum, 0); } template<> EIGEN_STRONG_INLINE int8_t predux(const Packet8c& a) { int8x8_t sum = vpadd_s8(a,a); sum = vpadd_s8(sum, sum); sum = vpadd_s8(sum, sum); return vget_lane_s8(sum, 0); } template<> EIGEN_STRONG_INLINE int8_t predux(const Packet16c& a) { int8x8_t sum = vadd_s8(vget_low_s8(a), vget_high_s8(a)); sum = vpadd_s8(sum, sum); sum = vpadd_s8(sum, sum); sum = vpadd_s8(sum, sum); return vget_lane_s8(sum, 0); } template<> EIGEN_STRONG_INLINE uint8_t predux(const Packet4uc& a) { const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a)); uint8x8_t sum = vpadd_u8(a_dup, a_dup); sum = vpadd_u8(sum, sum); return vget_lane_u8(sum, 0); } template<> EIGEN_STRONG_INLINE uint8_t predux(const Packet8uc& a) { uint8x8_t sum = vpadd_u8(a,a); sum = vpadd_u8(sum, sum); sum = vpadd_u8(sum, sum); return vget_lane_u8(sum, 0); } template<> EIGEN_STRONG_INLINE uint8_t predux(const Packet16uc& a) { uint8x8_t sum = vadd_u8(vget_low_u8(a), vget_high_u8(a)); sum = vpadd_u8(sum, sum); sum = vpadd_u8(sum, sum); sum = vpadd_u8(sum, sum); return vget_lane_u8(sum, 0); } template<> EIGEN_STRONG_INLINE int16_t predux(const Packet4s& a) { const int16x4_t sum = vpadd_s16(a,a); return vget_lane_s16(vpadd_s16(sum, sum), 0); } template<> EIGEN_STRONG_INLINE int16_t predux(const Packet8s& a) { int16x4_t sum = vadd_s16(vget_low_s16(a), vget_high_s16(a)); sum = vpadd_s16(sum, sum); sum = vpadd_s16(sum, sum); return vget_lane_s16(sum, 0); } template<> EIGEN_STRONG_INLINE uint16_t predux(const Packet4us& a) { const uint16x4_t sum = vpadd_u16(a,a); return vget_lane_u16(vpadd_u16(sum, sum), 0); } template<> EIGEN_STRONG_INLINE uint16_t predux(const Packet8us& a) { uint16x4_t sum = vadd_u16(vget_low_u16(a), vget_high_u16(a)); sum = vpadd_u16(sum, sum); sum = vpadd_u16(sum, sum); return vget_lane_u16(sum, 0); } template<> EIGEN_STRONG_INLINE int32_t predux(const Packet2i& a) { return vget_lane_s32(vpadd_s32(a,a), 0); } template<> EIGEN_STRONG_INLINE int32_t predux(const Packet4i& a) { const int32x2_t sum = vadd_s32(vget_low_s32(a), vget_high_s32(a)); return vget_lane_s32(vpadd_s32(sum, sum), 0); } template<> EIGEN_STRONG_INLINE uint32_t predux(const Packet2ui& a) { return vget_lane_u32(vpadd_u32(a,a), 0); } template<> EIGEN_STRONG_INLINE uint32_t predux(const Packet4ui& a) { const uint32x2_t sum = vadd_u32(vget_low_u32(a), vget_high_u32(a)); return vget_lane_u32(vpadd_u32(sum, sum), 0); } template<> EIGEN_STRONG_INLINE int64_t predux(const Packet2l& a) { return vgetq_lane_s64(a, 0) + vgetq_lane_s64(a, 1); } template<> EIGEN_STRONG_INLINE uint64_t predux(const Packet2ul& a) { return vgetq_lane_u64(a, 0) + vgetq_lane_u64(a, 1); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c predux_half_dowto4(const Packet8c& a) { return vget_lane_s32(vreinterpret_s32_s8(vadd_s8(a, vreinterpret_s8_s32(vrev64_s32(vreinterpret_s32_s8(a))))), 0); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c predux_half_dowto4(const Packet16c& a) { return vadd_s8(vget_high_s8(a), vget_low_s8(a)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc predux_half_dowto4(const Packet8uc& a) { return vget_lane_u32(vreinterpret_u32_u8(vadd_u8(a, vreinterpret_u8_u32(vrev64_u32(vreinterpret_u32_u8(a))))), 0); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc predux_half_dowto4(const Packet16uc& a) { return vadd_u8(vget_high_u8(a), vget_low_u8(a)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s predux_half_dowto4(const Packet8s& a) { return vadd_s16(vget_high_s16(a), vget_low_s16(a)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us predux_half_dowto4(const Packet8us& a) { return vadd_u16(vget_high_u16(a), vget_low_u16(a)); } // Other reduction functions: // mul template<> EIGEN_STRONG_INLINE float predux_mul(const Packet2f& a) { return vget_lane_f32(a, 0) * vget_lane_f32(a, 1); } template<> EIGEN_STRONG_INLINE float predux_mul(const Packet4f& a) { return predux_mul(vmul_f32(vget_low_f32(a), vget_high_f32(a))); } template<> EIGEN_STRONG_INLINE int8_t predux_mul(const Packet4c& a) { int8x8_t prod = vreinterpret_s8_s32(vdup_n_s32(a)); prod = vmul_s8(prod, vrev16_s8(prod)); return vget_lane_s8(prod, 0) * vget_lane_s8(prod, 2); } template<> EIGEN_STRONG_INLINE int8_t predux_mul(const Packet8c& a) { int8x8_t prod = vmul_s8(a, vrev16_s8(a)); prod = vmul_s8(prod, vrev32_s8(prod)); return vget_lane_s8(prod, 0) * vget_lane_s8(prod, 4); } template<> EIGEN_STRONG_INLINE int8_t predux_mul(const Packet16c& a) { return predux_mul(vmul_s8(vget_low_s8(a), vget_high_s8(a))); } template<> EIGEN_STRONG_INLINE uint8_t predux_mul(const Packet4uc& a) { uint8x8_t prod = vreinterpret_u8_u32(vdup_n_u32(a)); prod = vmul_u8(prod, vrev16_u8(prod)); return vget_lane_u8(prod, 0) * vget_lane_u8(prod, 2); } template<> EIGEN_STRONG_INLINE uint8_t predux_mul(const Packet8uc& a) { uint8x8_t prod = vmul_u8(a, vrev16_u8(a)); prod = vmul_u8(prod, vrev32_u8(prod)); return vget_lane_u8(prod, 0) * vget_lane_u8(prod, 4); } template<> EIGEN_STRONG_INLINE uint8_t predux_mul(const Packet16uc& a) { return predux_mul(vmul_u8(vget_low_u8(a), vget_high_u8(a))); } template<> EIGEN_STRONG_INLINE int16_t predux_mul(const Packet4s& a) { const int16x4_t prod = vmul_s16(a, vrev32_s16(a)); return vget_lane_s16(prod, 0) * vget_lane_s16(prod, 2); } template<> EIGEN_STRONG_INLINE int16_t predux_mul(const Packet8s& a) { int16x4_t prod; // Get the product of a_lo * a_hi -> |a1*a5|a2*a6|a3*a7|a4*a8| prod = vmul_s16(vget_low_s16(a), vget_high_s16(a)); // Swap and multiply |a1*a5*a2*a6|a3*a7*a4*a8| prod = vmul_s16(prod, vrev32_s16(prod)); // Multiply |a1*a5*a2*a6*a3*a7*a4*a8| return vget_lane_s16(prod, 0) * vget_lane_s16(prod, 2); } template<> EIGEN_STRONG_INLINE uint16_t predux_mul(const Packet4us& a) { const uint16x4_t prod = vmul_u16(a, vrev32_u16(a)); return vget_lane_u16(prod, 0) * vget_lane_u16(prod, 2); } template<> EIGEN_STRONG_INLINE uint16_t predux_mul(const Packet8us& a) { uint16x4_t prod; // Get the product of a_lo * a_hi -> |a1*a5|a2*a6|a3*a7|a4*a8| prod = vmul_u16(vget_low_u16(a), vget_high_u16(a)); // Swap and multiply |a1*a5*a2*a6|a3*a7*a4*a8| prod = vmul_u16(prod, vrev32_u16(prod)); // Multiply |a1*a5*a2*a6*a3*a7*a4*a8| return vget_lane_u16(prod, 0) * vget_lane_u16(prod, 2); } template<> EIGEN_STRONG_INLINE int32_t predux_mul(const Packet2i& a) { return vget_lane_s32(a, 0) * vget_lane_s32(a, 1); } template<> EIGEN_STRONG_INLINE int32_t predux_mul(const Packet4i& a) { return predux_mul(vmul_s32(vget_low_s32(a), vget_high_s32(a))); } template<> EIGEN_STRONG_INLINE uint32_t predux_mul(const Packet2ui& a) { return vget_lane_u32(a, 0) * vget_lane_u32(a, 1); } template<> EIGEN_STRONG_INLINE uint32_t predux_mul(const Packet4ui& a) { return predux_mul(vmul_u32(vget_low_u32(a), vget_high_u32(a))); } template<> EIGEN_STRONG_INLINE int64_t predux_mul(const Packet2l& a) { return vgetq_lane_s64(a, 0) * vgetq_lane_s64(a, 1); } template<> EIGEN_STRONG_INLINE uint64_t predux_mul(const Packet2ul& a) { return vgetq_lane_u64(a, 0) * vgetq_lane_u64(a, 1); } // min template<> EIGEN_STRONG_INLINE float predux_min(const Packet2f& a) { return vget_lane_f32(vpmin_f32(a,a), 0); } template<> EIGEN_STRONG_INLINE float predux_min(const Packet4f& a) { const float32x2_t min = vmin_f32(vget_low_f32(a), vget_high_f32(a)); return vget_lane_f32(vpmin_f32(min, min), 0); } template<> EIGEN_STRONG_INLINE int8_t predux_min(const Packet4c& a) { const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a)); int8x8_t min = vpmin_s8(a_dup, a_dup); min = vpmin_s8(min, min); return vget_lane_s8(min, 0); } template<> EIGEN_STRONG_INLINE int8_t predux_min(const Packet8c& a) { int8x8_t min = vpmin_s8(a,a); min = vpmin_s8(min, min); min = vpmin_s8(min, min); return vget_lane_s8(min, 0); } template<> EIGEN_STRONG_INLINE int8_t predux_min(const Packet16c& a) { int8x8_t min = vmin_s8(vget_low_s8(a), vget_high_s8(a)); min = vpmin_s8(min, min); min = vpmin_s8(min, min); min = vpmin_s8(min, min); return vget_lane_s8(min, 0); } template<> EIGEN_STRONG_INLINE uint8_t predux_min(const Packet4uc& a) { const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a)); uint8x8_t min = vpmin_u8(a_dup, a_dup); min = vpmin_u8(min, min); return vget_lane_u8(min, 0); } template<> EIGEN_STRONG_INLINE uint8_t predux_min(const Packet8uc& a) { uint8x8_t min = vpmin_u8(a,a); min = vpmin_u8(min, min); min = vpmin_u8(min, min); return vget_lane_u8(min, 0); } template<> EIGEN_STRONG_INLINE uint8_t predux_min(const Packet16uc& a) { uint8x8_t min = vmin_u8(vget_low_u8(a), vget_high_u8(a)); min = vpmin_u8(min, min); min = vpmin_u8(min, min); min = vpmin_u8(min, min); return vget_lane_u8(min, 0); } template<> EIGEN_STRONG_INLINE int16_t predux_min(const Packet4s& a) { const int16x4_t min = vpmin_s16(a,a); return vget_lane_s16(vpmin_s16(min, min), 0); } template<> EIGEN_STRONG_INLINE int16_t predux_min(const Packet8s& a) { int16x4_t min = vmin_s16(vget_low_s16(a), vget_high_s16(a)); min = vpmin_s16(min, min); min = vpmin_s16(min, min); return vget_lane_s16(min, 0); } template<> EIGEN_STRONG_INLINE uint16_t predux_min(const Packet4us& a) { const uint16x4_t min = vpmin_u16(a,a); return vget_lane_u16(vpmin_u16(min, min), 0); } template<> EIGEN_STRONG_INLINE uint16_t predux_min(const Packet8us& a) { uint16x4_t min = vmin_u16(vget_low_u16(a), vget_high_u16(a)); min = vpmin_u16(min, min); min = vpmin_u16(min, min); return vget_lane_u16(min, 0); } template<> EIGEN_STRONG_INLINE int32_t predux_min(const Packet2i& a) { return vget_lane_s32(vpmin_s32(a,a), 0); } template<> EIGEN_STRONG_INLINE int32_t predux_min(const Packet4i& a) { const int32x2_t min = vmin_s32(vget_low_s32(a), vget_high_s32(a)); return vget_lane_s32(vpmin_s32(min, min), 0); } template<> EIGEN_STRONG_INLINE uint32_t predux_min(const Packet2ui& a) { return vget_lane_u32(vpmin_u32(a,a), 0); } template<> EIGEN_STRONG_INLINE uint32_t predux_min(const Packet4ui& a) { const uint32x2_t min = vmin_u32(vget_low_u32(a), vget_high_u32(a)); return vget_lane_u32(vpmin_u32(min, min), 0); } template<> EIGEN_STRONG_INLINE int64_t predux_min(const Packet2l& a) { return (std::min)(vgetq_lane_s64(a, 0), vgetq_lane_s64(a, 1)); } template<> EIGEN_STRONG_INLINE uint64_t predux_min(const Packet2ul& a) { return (std::min)(vgetq_lane_u64(a, 0), vgetq_lane_u64(a, 1)); } // max template<> EIGEN_STRONG_INLINE float predux_max(const Packet2f& a) { return vget_lane_f32(vpmax_f32(a,a), 0); } template<> EIGEN_STRONG_INLINE float predux_max(const Packet4f& a) { const float32x2_t max = vmax_f32(vget_low_f32(a), vget_high_f32(a)); return vget_lane_f32(vpmax_f32(max, max), 0); } template<> EIGEN_STRONG_INLINE int8_t predux_max(const Packet4c& a) { const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a)); int8x8_t max = vpmax_s8(a_dup, a_dup); max = vpmax_s8(max, max); return vget_lane_s8(max, 0); } template<> EIGEN_STRONG_INLINE int8_t predux_max(const Packet8c& a) { int8x8_t max = vpmax_s8(a,a); max = vpmax_s8(max, max); max = vpmax_s8(max, max); return vget_lane_s8(max, 0); } template<> EIGEN_STRONG_INLINE int8_t predux_max(const Packet16c& a) { int8x8_t max = vmax_s8(vget_low_s8(a), vget_high_s8(a)); max = vpmax_s8(max, max); max = vpmax_s8(max, max); max = vpmax_s8(max, max); return vget_lane_s8(max, 0); } template<> EIGEN_STRONG_INLINE uint8_t predux_max(const Packet4uc& a) { const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a)); uint8x8_t max = vpmax_u8(a_dup, a_dup); max = vpmax_u8(max, max); return vget_lane_u8(max, 0); } template<> EIGEN_STRONG_INLINE uint8_t predux_max(const Packet8uc& a) { uint8x8_t max = vpmax_u8(a,a); max = vpmax_u8(max, max); max = vpmax_u8(max, max); return vget_lane_u8(max, 0); } template<> EIGEN_STRONG_INLINE uint8_t predux_max(const Packet16uc& a) { uint8x8_t max = vmax_u8(vget_low_u8(a), vget_high_u8(a)); max = vpmax_u8(max, max); max = vpmax_u8(max, max); max = vpmax_u8(max, max); return vget_lane_u8(max, 0); } template<> EIGEN_STRONG_INLINE int16_t predux_max(const Packet4s& a) { const int16x4_t max = vpmax_s16(a,a); return vget_lane_s16(vpmax_s16(max, max), 0); } template<> EIGEN_STRONG_INLINE int16_t predux_max(const Packet8s& a) { int16x4_t max = vmax_s16(vget_low_s16(a), vget_high_s16(a)); max = vpmax_s16(max, max); max = vpmax_s16(max, max); return vget_lane_s16(max, 0); } template<> EIGEN_STRONG_INLINE uint16_t predux_max(const Packet4us& a) { const uint16x4_t max = vpmax_u16(a,a); return vget_lane_u16(vpmax_u16(max, max), 0); } template<> EIGEN_STRONG_INLINE uint16_t predux_max(const Packet8us& a) { uint16x4_t max = vmax_u16(vget_low_u16(a), vget_high_u16(a)); max = vpmax_u16(max, max); max = vpmax_u16(max, max); return vget_lane_u16(max, 0); } template<> EIGEN_STRONG_INLINE int32_t predux_max(const Packet2i& a) { return vget_lane_s32(vpmax_s32(a,a), 0); } template<> EIGEN_STRONG_INLINE int32_t predux_max(const Packet4i& a) { const int32x2_t max = vmax_s32(vget_low_s32(a), vget_high_s32(a)); return vget_lane_s32(vpmax_s32(max, max), 0); } template<> EIGEN_STRONG_INLINE uint32_t predux_max(const Packet2ui& a) { return vget_lane_u32(vpmax_u32(a,a), 0); } template<> EIGEN_STRONG_INLINE uint32_t predux_max(const Packet4ui& a) { const uint32x2_t max = vmax_u32(vget_low_u32(a), vget_high_u32(a)); return vget_lane_u32(vpmax_u32(max, max), 0); } template<> EIGEN_STRONG_INLINE int64_t predux_max(const Packet2l& a) { return (std::max)(vgetq_lane_s64(a, 0), vgetq_lane_s64(a, 1)); } template<> EIGEN_STRONG_INLINE uint64_t predux_max(const Packet2ul& a) { return (std::max)(vgetq_lane_u64(a, 0), vgetq_lane_u64(a, 1)); } template<> EIGEN_STRONG_INLINE bool predux_any(const Packet4f& x) { uint32x2_t tmp = vorr_u32(vget_low_u32( vreinterpretq_u32_f32(x)), vget_high_u32(vreinterpretq_u32_f32(x))); return vget_lane_u32(vpmax_u32(tmp, tmp), 0); } // Helpers for ptranspose. namespace detail { template void zip_in_place(Packet& p1, Packet& p2); template<> EIGEN_ALWAYS_INLINE void zip_in_place(Packet2f& p1, Packet2f& p2) { const float32x2x2_t tmp = vzip_f32(p1, p2); p1 = tmp.val[0]; p2 = tmp.val[1]; } template<> EIGEN_ALWAYS_INLINE void zip_in_place(Packet4f& p1, Packet4f& p2) { const float32x4x2_t tmp = vzipq_f32(p1, p2); p1 = tmp.val[0]; p2 = tmp.val[1]; } template<> EIGEN_ALWAYS_INLINE void zip_in_place(Packet8c& p1, Packet8c& p2) { const int8x8x2_t tmp = vzip_s8(p1, p2); p1 = tmp.val[0]; p2 = tmp.val[1]; } template<> EIGEN_ALWAYS_INLINE void zip_in_place(Packet16c& p1, Packet16c& p2) { const int8x16x2_t tmp = vzipq_s8(p1, p2); p1 = tmp.val[0]; p2 = tmp.val[1]; } template<> EIGEN_ALWAYS_INLINE void zip_in_place(Packet8uc& p1, Packet8uc& p2) { const uint8x8x2_t tmp = vzip_u8(p1, p2); p1 = tmp.val[0]; p2 = tmp.val[1]; } template<> EIGEN_ALWAYS_INLINE void zip_in_place(Packet16uc& p1, Packet16uc& p2) { const uint8x16x2_t tmp = vzipq_u8(p1, p2); p1 = tmp.val[0]; p2 = tmp.val[1]; } template<> EIGEN_ALWAYS_INLINE void zip_in_place(Packet2i& p1, Packet2i& p2) { const int32x2x2_t tmp = vzip_s32(p1, p2); p1 = tmp.val[0]; p2 = tmp.val[1]; } template<> EIGEN_ALWAYS_INLINE void zip_in_place(Packet4i& p1, Packet4i& p2) { const int32x4x2_t tmp = vzipq_s32(p1, p2); p1 = tmp.val[0]; p2 = tmp.val[1]; } template<> EIGEN_ALWAYS_INLINE void zip_in_place(Packet2ui& p1, Packet2ui& p2) { const uint32x2x2_t tmp = vzip_u32(p1, p2); p1 = tmp.val[0]; p2 = tmp.val[1]; } template<> EIGEN_ALWAYS_INLINE void zip_in_place(Packet4ui& p1, Packet4ui& p2) { const uint32x4x2_t tmp = vzipq_u32(p1, p2); p1 = tmp.val[0]; p2 = tmp.val[1]; } template<> EIGEN_ALWAYS_INLINE void zip_in_place(Packet4s& p1, Packet4s& p2) { const int16x4x2_t tmp = vzip_s16(p1, p2); p1 = tmp.val[0]; p2 = tmp.val[1]; } template<> EIGEN_ALWAYS_INLINE void zip_in_place(Packet8s& p1, Packet8s& p2) { const int16x8x2_t tmp = vzipq_s16(p1, p2); p1 = tmp.val[0]; p2 = tmp.val[1]; } template<> EIGEN_ALWAYS_INLINE void zip_in_place(Packet4us& p1, Packet4us& p2) { const uint16x4x2_t tmp = vzip_u16(p1, p2); p1 = tmp.val[0]; p2 = tmp.val[1]; } template<> EIGEN_ALWAYS_INLINE void zip_in_place(Packet8us& p1, Packet8us& p2) { const uint16x8x2_t tmp = vzipq_u16(p1, p2); p1 = tmp.val[0]; p2 = tmp.val[1]; } template EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock& kernel) { zip_in_place(kernel.packet[0], kernel.packet[1]); } template EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock& kernel) { zip_in_place(kernel.packet[0], kernel.packet[2]); zip_in_place(kernel.packet[1], kernel.packet[3]); zip_in_place(kernel.packet[0], kernel.packet[1]); zip_in_place(kernel.packet[2], kernel.packet[3]); } template EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock& kernel) { zip_in_place(kernel.packet[0], kernel.packet[4]); zip_in_place(kernel.packet[1], kernel.packet[5]); zip_in_place(kernel.packet[2], kernel.packet[6]); zip_in_place(kernel.packet[3], kernel.packet[7]); zip_in_place(kernel.packet[0], kernel.packet[2]); zip_in_place(kernel.packet[1], kernel.packet[3]); zip_in_place(kernel.packet[4], kernel.packet[6]); zip_in_place(kernel.packet[5], kernel.packet[7]); zip_in_place(kernel.packet[0], kernel.packet[1]); zip_in_place(kernel.packet[2], kernel.packet[3]); zip_in_place(kernel.packet[4], kernel.packet[5]); zip_in_place(kernel.packet[6], kernel.packet[7]); } template EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock& kernel) { EIGEN_UNROLL_LOOP for (int i=0; i<4; ++i) { const int m = (1 << i); EIGEN_UNROLL_LOOP for (int j=0; j& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const int8x8_t a = vreinterpret_s8_s32(vset_lane_s32(kernel.packet[2], vdup_n_s32(kernel.packet[0]), 1)); const int8x8_t b = vreinterpret_s8_s32(vset_lane_s32(kernel.packet[3], vdup_n_s32(kernel.packet[1]), 1)); const int8x8x2_t zip8 = vzip_s8(a,b); const int16x4x2_t zip16 = vzip_s16(vreinterpret_s16_s8(zip8.val[0]), vreinterpret_s16_s8(zip8.val[1])); kernel.packet[0] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[0]), 0); kernel.packet[1] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[0]), 1); kernel.packet[2] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[1]), 0); kernel.packet[3] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[1]), 1); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const uint8x8_t a = vreinterpret_u8_u32(vset_lane_u32(kernel.packet[2], vdup_n_u32(kernel.packet[0]), 1)); const uint8x8_t b = vreinterpret_u8_u32(vset_lane_u32(kernel.packet[3], vdup_n_u32(kernel.packet[1]), 1)); const uint8x8x2_t zip8 = vzip_u8(a,b); const uint16x4x2_t zip16 = vzip_u16(vreinterpret_u16_u8(zip8.val[0]), vreinterpret_u16_u8(zip8.val[1])); kernel.packet[0] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[0]), 0); kernel.packet[1] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[0]), 1); kernel.packet[2] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[1]), 0); kernel.packet[3] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[1]), 1); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::zip_in_place(kernel.packet[0], kernel.packet[1]); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { #if EIGEN_ARCH_ARM64 const int64x2_t tmp1 = vzip1q_s64(kernel.packet[0], kernel.packet[1]); kernel.packet[1] = vzip2q_s64(kernel.packet[0], kernel.packet[1]); kernel.packet[0] = tmp1; #else const int64x1_t tmp[2][2] = { { vget_low_s64(kernel.packet[0]), vget_high_s64(kernel.packet[0]) }, { vget_low_s64(kernel.packet[1]), vget_high_s64(kernel.packet[1]) } }; kernel.packet[0] = vcombine_s64(tmp[0][0], tmp[1][0]); kernel.packet[1] = vcombine_s64(tmp[0][1], tmp[1][1]); #endif } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { #if EIGEN_ARCH_ARM64 const uint64x2_t tmp1 = vzip1q_u64(kernel.packet[0], kernel.packet[1]); kernel.packet[1] = vzip2q_u64(kernel.packet[0], kernel.packet[1]); kernel.packet[0] = tmp1; #else const uint64x1_t tmp[2][2] = { { vget_low_u64(kernel.packet[0]), vget_high_u64(kernel.packet[0]) }, { vget_low_u64(kernel.packet[1]), vget_high_u64(kernel.packet[1]) } }; kernel.packet[0] = vcombine_u64(tmp[0][0], tmp[1][0]); kernel.packet[1] = vcombine_u64(tmp[0][1], tmp[1][1]); #endif } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f pselect( const Packet2f& mask, const Packet2f& a, const Packet2f& b) { return vbsl_f32(vreinterpret_u32_f32(mask), a, b); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pselect(const Packet4f& mask, const Packet4f& a, const Packet4f& b) { return vbslq_f32(vreinterpretq_u32_f32(mask), a, b); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c pselect(const Packet8c& mask, const Packet8c& a, const Packet8c& b) { return vbsl_s8(vreinterpret_u8_s8(mask), a, b); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pselect(const Packet16c& mask, const Packet16c& a, const Packet16c& b) { return vbslq_s8(vreinterpretq_u8_s8(mask), a, b); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc pselect(const Packet8uc& mask, const Packet8uc& a, const Packet8uc& b) { return vbsl_u8(mask, a, b); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pselect(const Packet16uc& mask, const Packet16uc& a, const Packet16uc& b) { return vbslq_u8(mask, a, b); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s pselect(const Packet4s& mask, const Packet4s& a, const Packet4s& b) { return vbsl_s16(vreinterpret_u16_s16(mask), a, b); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pselect(const Packet8s& mask, const Packet8s& a, const Packet8s& b) { return vbslq_s16(vreinterpretq_u16_s16(mask), a, b); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us pselect(const Packet4us& mask, const Packet4us& a, const Packet4us& b) { return vbsl_u16(mask, a, b); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pselect(const Packet8us& mask, const Packet8us& a, const Packet8us& b) { return vbslq_u16(mask, a, b); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i pselect(const Packet2i& mask, const Packet2i& a, const Packet2i& b) { return vbsl_s32(vreinterpret_u32_s32(mask), a, b); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pselect(const Packet4i& mask, const Packet4i& a, const Packet4i& b) { return vbslq_s32(vreinterpretq_u32_s32(mask), a, b); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui pselect(const Packet2ui& mask, const Packet2ui& a, const Packet2ui& b) { return vbsl_u32(mask, a, b); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pselect(const Packet4ui& mask, const Packet4ui& a, const Packet4ui& b) { return vbslq_u32(mask, a, b); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pselect(const Packet2l& mask, const Packet2l& a, const Packet2l& b) { return vbslq_s64(vreinterpretq_u64_s64(mask), a, b); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pselect(const Packet2ul& mask, const Packet2ul& a, const Packet2ul& b) { return vbslq_u64(mask, a, b); } // Use armv8 rounding intinsics if available. #if EIGEN_ARCH_ARMV8 template<> EIGEN_STRONG_INLINE Packet2f print(const Packet2f& a) { return vrndn_f32(a); } template<> EIGEN_STRONG_INLINE Packet4f print(const Packet4f& a) { return vrndnq_f32(a); } template<> EIGEN_STRONG_INLINE Packet2f pfloor(const Packet2f& a) { return vrndm_f32(a); } template<> EIGEN_STRONG_INLINE Packet4f pfloor(const Packet4f& a) { return vrndmq_f32(a); } template<> EIGEN_STRONG_INLINE Packet2f pceil(const Packet2f& a) { return vrndp_f32(a); } template<> EIGEN_STRONG_INLINE Packet4f pceil(const Packet4f& a) { return vrndpq_f32(a); } #else template<> EIGEN_STRONG_INLINE Packet4f print(const Packet4f& a) { // Adds and subtracts signum(a) * 2^23 to force rounding. const Packet4f limit = pset1(static_cast(1<<23)); const Packet4f abs_a = pabs(a); Packet4f r = padd(abs_a, limit); // Don't compile-away addition and subtraction. EIGEN_OPTIMIZATION_BARRIER(r); r = psub(r, limit); // If greater than limit, simply return a. Otherwise, account for sign. r = pselect(pcmp_lt(abs_a, limit), pselect(pcmp_lt(a, pzero(a)), pnegate(r), r), a); return r; } template<> EIGEN_STRONG_INLINE Packet2f print(const Packet2f& a) { // Adds and subtracts signum(a) * 2^23 to force rounding. const Packet2f limit = pset1(static_cast(1<<23)); const Packet2f abs_a = pabs(a); Packet2f r = padd(abs_a, limit); // Don't compile-away addition and subtraction. EIGEN_OPTIMIZATION_BARRIER(r); r = psub(r, limit); // If greater than limit, simply return a. Otherwise, account for sign. r = pselect(pcmp_lt(abs_a, limit), pselect(pcmp_lt(a, pzero(a)), pnegate(r), r), a); return r; } template<> EIGEN_STRONG_INLINE Packet4f pfloor(const Packet4f& a) { const Packet4f cst_1 = pset1(1.0f); Packet4f tmp = print(a); // If greater, subtract one. Packet4f mask = pcmp_lt(a, tmp); mask = pand(mask, cst_1); return psub(tmp, mask); } template<> EIGEN_STRONG_INLINE Packet2f pfloor(const Packet2f& a) { const Packet2f cst_1 = pset1(1.0f); Packet2f tmp = print(a); // If greater, subtract one. Packet2f mask = pcmp_lt(a, tmp); mask = pand(mask, cst_1); return psub(tmp, mask); } template<> EIGEN_STRONG_INLINE Packet4f pceil(const Packet4f& a) { const Packet4f cst_1 = pset1(1.0f); Packet4f tmp = print(a); // If smaller, add one. Packet4f mask = pcmp_lt(tmp, a); mask = pand(mask, cst_1); return padd(tmp, mask); } template<> EIGEN_STRONG_INLINE Packet2f pceil(const Packet2f& a) { const Packet2f cst_1 = pset1(1.0); Packet2f tmp = print(a); // If smaller, add one. Packet2f mask = pcmp_lt(tmp, a); mask = pand(mask, cst_1); return padd(tmp, mask); } #endif /** * Computes the integer square root * @remarks The calculation is performed using an algorithm which iterates through each binary digit of the result * and tests whether setting that digit to 1 would cause the square of the value to be greater than the argument * value. The algorithm is described in detail here: http://ww1.microchip.com/downloads/en/AppNotes/91040a.pdf . */ template<> EIGEN_STRONG_INLINE Packet4uc psqrt(const Packet4uc& a) { uint8x8_t x = vreinterpret_u8_u32(vdup_n_u32(a)); uint8x8_t res = vdup_n_u8(0); uint8x8_t add = vdup_n_u8(0x8); for (int i = 0; i < 4; i++) { const uint8x8_t temp = vorr_u8(res, add); res = vbsl_u8(vcge_u8(x, vmul_u8(temp, temp)), temp, res); add = vshr_n_u8(add, 1); } return vget_lane_u32(vreinterpret_u32_u8(res), 0); } /// @copydoc Eigen::internal::psqrt(const Packet4uc& a) template<> EIGEN_STRONG_INLINE Packet8uc psqrt(const Packet8uc& a) { uint8x8_t res = vdup_n_u8(0); uint8x8_t add = vdup_n_u8(0x8); for (int i = 0; i < 4; i++) { const uint8x8_t temp = vorr_u8(res, add); res = vbsl_u8(vcge_u8(a, vmul_u8(temp, temp)), temp, res); add = vshr_n_u8(add, 1); } return res; } /// @copydoc Eigen::internal::psqrt(const Packet4uc& a) template<> EIGEN_STRONG_INLINE Packet16uc psqrt(const Packet16uc& a) { uint8x16_t res = vdupq_n_u8(0); uint8x16_t add = vdupq_n_u8(0x8); for (int i = 0; i < 4; i++) { const uint8x16_t temp = vorrq_u8(res, add); res = vbslq_u8(vcgeq_u8(a, vmulq_u8(temp, temp)), temp, res); add = vshrq_n_u8(add, 1); } return res; } /// @copydoc Eigen::internal::psqrt(const Packet4uc& a) template<> EIGEN_STRONG_INLINE Packet4us psqrt(const Packet4us& a) { uint16x4_t res = vdup_n_u16(0); uint16x4_t add = vdup_n_u16(0x80); for (int i = 0; i < 8; i++) { const uint16x4_t temp = vorr_u16(res, add); res = vbsl_u16(vcge_u16(a, vmul_u16(temp, temp)), temp, res); add = vshr_n_u16(add, 1); } return res; } /// @copydoc Eigen::internal::psqrt(const Packet4uc& a) template<> EIGEN_STRONG_INLINE Packet8us psqrt(const Packet8us& a) { uint16x8_t res = vdupq_n_u16(0); uint16x8_t add = vdupq_n_u16(0x80); for (int i = 0; i < 8; i++) { const uint16x8_t temp = vorrq_u16(res, add); res = vbslq_u16(vcgeq_u16(a, vmulq_u16(temp, temp)), temp, res); add = vshrq_n_u16(add, 1); } return res; } /// @copydoc Eigen::internal::psqrt(const Packet4uc& a) template<> EIGEN_STRONG_INLINE Packet2ui psqrt(const Packet2ui& a) { uint32x2_t res = vdup_n_u32(0); uint32x2_t add = vdup_n_u32(0x8000); for (int i = 0; i < 16; i++) { const uint32x2_t temp = vorr_u32(res, add); res = vbsl_u32(vcge_u32(a, vmul_u32(temp, temp)), temp, res); add = vshr_n_u32(add, 1); } return res; } /// @copydoc Eigen::internal::psqrt(const Packet4uc& a) template<> EIGEN_STRONG_INLINE Packet4ui psqrt(const Packet4ui& a) { uint32x4_t res = vdupq_n_u32(0); uint32x4_t add = vdupq_n_u32(0x8000); for (int i = 0; i < 16; i++) { const uint32x4_t temp = vorrq_u32(res, add); res = vbslq_u32(vcgeq_u32(a, vmulq_u32(temp, temp)), temp, res); add = vshrq_n_u32(add, 1); } return res; } template<> EIGEN_STRONG_INLINE Packet4f prsqrt(const Packet4f& a) { // Compute approximate reciprocal sqrt. Packet4f x = vrsqrteq_f32(a); // Do Newton iterations for 1/sqrt(x). x = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a, x), x), x); x = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a, x), x), x); const Packet4f infinity = pset1(NumTraits::infinity()); return pselect(pcmp_eq(a, pzero(a)), infinity, x); } template<> EIGEN_STRONG_INLINE Packet2f prsqrt(const Packet2f& a) { // Compute approximate reciprocal sqrt. Packet2f x = vrsqrte_f32(a); // Do Newton iterations for 1/sqrt(x). x = vmul_f32(vrsqrts_f32(vmul_f32(a, x), x), x); x = vmul_f32(vrsqrts_f32(vmul_f32(a, x), x), x); const Packet2f infinity = pset1(NumTraits::infinity()); return pselect(pcmp_eq(a, pzero(a)), infinity, x); } // Unfortunately vsqrt_f32 is only available for A64. #if EIGEN_ARCH_ARM64 template<> EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& _x){return vsqrtq_f32(_x);} template<> EIGEN_STRONG_INLINE Packet2f psqrt(const Packet2f& _x){return vsqrt_f32(_x); } #else template<> EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& a) { const Packet4f infinity = pset1(NumTraits::infinity()); const Packet4f is_zero_or_inf = por(pcmp_eq(a, pzero(a)), pcmp_eq(a, infinity)); return pselect(is_zero_or_inf, a, pmul(a, prsqrt(a))); } template<> EIGEN_STRONG_INLINE Packet2f psqrt(const Packet2f& a) { const Packet2f infinity = pset1(NumTraits::infinity()); const Packet2f is_zero_or_inf = por(pcmp_eq(a, pzero(a)), pcmp_eq(a, infinity)); return pselect(is_zero_or_inf, a, pmul(a, prsqrt(a))); } #endif //---------- bfloat16 ---------- // TODO: Add support for native armv8.6-a bfloat16_t // TODO: Guard if we have native bfloat16 support typedef eigen_packet_wrapper Packet4bf; template<> struct is_arithmetic { enum { value = true }; }; template<> struct packet_traits : default_packet_traits { typedef Packet4bf type; typedef Packet4bf half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 4, HasHalfPacket = 0, HasCmp = 1, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasNegate = 1, HasAbs = 1, HasArg = 0, HasAbs2 = 1, HasAbsDiff = 1, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasBlend = 0, HasDiv = 1, HasFloor = 1, HasCeil = 1, HasRint = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasLog = 1, HasExp = 1, HasSqrt = 0, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, HasBessel = 0, // Issues with accuracy. HasNdtri = 0 }; }; template<> struct unpacket_traits { typedef bfloat16 type; typedef Packet4bf half; enum { size = 4, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; namespace detail { template<> EIGEN_ALWAYS_INLINE void zip_in_place(Packet4bf& p1, Packet4bf& p2) { const uint16x4x2_t tmp = vzip_u16(p1, p2); p1 = tmp.val[0]; p2 = tmp.val[1]; } } // namespace detail EIGEN_STRONG_INLINE Packet4bf F32ToBf16(const Packet4f& p) { // See the scalar implemention in BFloat16.h for a comprehensible explanation // of this fast rounding algorithm Packet4ui input = reinterpret_cast(p); // lsb = (input >> 16) & 1 Packet4ui lsb = vandq_u32(vshrq_n_u32(input, 16), vdupq_n_u32(1)); // rounding_bias = 0x7fff + lsb Packet4ui rounding_bias = vaddq_u32(lsb, vdupq_n_u32(0x7fff)); // input += rounding_bias input = vaddq_u32(input, rounding_bias); // input = input >> 16 input = vshrq_n_u32(input, 16); // Replace float-nans by bfloat16-nans, that is 0x7fc0 const Packet4ui bf16_nan = vdupq_n_u32(0x7fc0); const Packet4ui mask = vceqq_f32(p, p); input = vbslq_u32(mask, input, bf16_nan); // output = static_cast(input) return vmovn_u32(input); } EIGEN_STRONG_INLINE Packet4f Bf16ToF32(const Packet4bf& p) { return reinterpret_cast(vshlq_n_u32(vmovl_u16(p), 16)); } EIGEN_STRONG_INLINE Packet4bf F32MaskToBf16Mask(const Packet4f& p) { return vmovn_u32(vreinterpretq_u32_f32(p)); } template<> EIGEN_STRONG_INLINE Packet4bf pset1(const bfloat16& from) { return pset1(from.value); } template<> EIGEN_STRONG_INLINE bfloat16 pfirst(const Packet4bf& from) { return bfloat16_impl::raw_uint16_to_bfloat16(static_cast(pfirst(from))); } template<> EIGEN_STRONG_INLINE Packet4bf pload(const bfloat16* from) { return pload(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE Packet4bf ploadu(const bfloat16* from) { return ploadu(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE void pstore(bfloat16* to, const Packet4bf& from) { EIGEN_DEBUG_ALIGNED_STORE vst1_u16(reinterpret_cast(to), from); } template<> EIGEN_STRONG_INLINE void pstoreu(bfloat16* to, const Packet4bf& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1_u16(reinterpret_cast(to), from); } template<> EIGEN_STRONG_INLINE Packet4bf ploaddup(const bfloat16* from) { return ploaddup(reinterpret_cast(from)); } template <> EIGEN_STRONG_INLINE Packet4bf pabs(const Packet4bf& a) { return F32ToBf16(pabs(Bf16ToF32(a))); } template <> EIGEN_STRONG_INLINE Packet4bf pmin(const Packet4bf &a, const Packet4bf &b) { return F32ToBf16(pmin(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet4bf pmin(const Packet4bf &a, const Packet4bf &b) { return F32ToBf16(pmin(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet4bf pmin(const Packet4bf &a, const Packet4bf &b) { return F32ToBf16(pmin(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet4bf pmax(const Packet4bf &a, const Packet4bf &b) { return F32ToBf16(pmax(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet4bf pmax(const Packet4bf &a, const Packet4bf &b) { return F32ToBf16(pmax(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet4bf pmax(const Packet4bf &a, const Packet4bf &b) { return F32ToBf16(pmax(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet4bf plset(const bfloat16& a) { return F32ToBf16(plset(static_cast(a))); } template<> EIGEN_STRONG_INLINE Packet4bf por(const Packet4bf& a,const Packet4bf& b) { return por(a, b); } template<> EIGEN_STRONG_INLINE Packet4bf pxor(const Packet4bf& a,const Packet4bf& b) { return pxor(a, b); } template<> EIGEN_STRONG_INLINE Packet4bf pand(const Packet4bf& a,const Packet4bf& b) { return pand(a, b); } template<> EIGEN_STRONG_INLINE Packet4bf pandnot(const Packet4bf& a,const Packet4bf& b) { return pandnot(a, b); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4bf pselect(const Packet4bf& mask, const Packet4bf& a, const Packet4bf& b) { return pselect(mask, a, b); } template<> EIGEN_STRONG_INLINE Packet4bf print(const Packet4bf& a) { return F32ToBf16(print(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE Packet4bf pfloor(const Packet4bf& a) { return F32ToBf16(pfloor(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE Packet4bf pceil(const Packet4bf& a) { return F32ToBf16(pceil(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE Packet4bf pconj(const Packet4bf& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4bf padd(const Packet4bf& a, const Packet4bf& b) { return F32ToBf16(padd(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet4bf psub(const Packet4bf& a, const Packet4bf& b) { return F32ToBf16(psub(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet4bf pmul(const Packet4bf& a, const Packet4bf& b) { return F32ToBf16(pmul(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet4bf pdiv(const Packet4bf& a, const Packet4bf& b) { return F32ToBf16(pdiv(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet4bf pgather(const bfloat16* from, Index stride) { return pgather(reinterpret_cast(from), stride); } template<> EIGEN_STRONG_INLINE void pscatter(bfloat16* to, const Packet4bf& from, Index stride) { pscatter(reinterpret_cast(to), from, stride); } template<> EIGEN_STRONG_INLINE bfloat16 predux(const Packet4bf& a) { return static_cast(predux(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE bfloat16 predux_max(const Packet4bf& a) { return static_cast(predux_max(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE bfloat16 predux_min(const Packet4bf& a) { return static_cast(predux_min(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE bfloat16 predux_mul(const Packet4bf& a) { return static_cast(predux_mul(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE Packet4bf preverse(const Packet4bf& a) { return preverse(a); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { detail::ptranspose_impl(kernel); } template<> EIGEN_STRONG_INLINE Packet4bf pabsdiff(const Packet4bf& a, const Packet4bf& b) { return F32ToBf16(pabsdiff(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet4bf pcmp_eq(const Packet4bf& a, const Packet4bf& b) { return F32MaskToBf16Mask(pcmp_eq(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet4bf pcmp_lt(const Packet4bf& a, const Packet4bf& b) { return F32MaskToBf16Mask(pcmp_lt(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet4bf pcmp_lt_or_nan(const Packet4bf& a, const Packet4bf& b) { return F32MaskToBf16Mask(pcmp_lt_or_nan(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet4bf pcmp_le(const Packet4bf& a, const Packet4bf& b) { return F32MaskToBf16Mask(pcmp_le(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet4bf pnegate(const Packet4bf& a) { return pxor(a, pset1(static_cast(0x8000))); } //---------- double ---------- // Clang 3.5 in the iOS toolchain has an ICE triggered by NEON intrisics for double. // Confirmed at least with __apple_build_version__ = 6000054. #ifdef __apple_build_version__ // Let's hope that by the time __apple_build_version__ hits the 601* range, the bug will be fixed. // https://gist.github.com/yamaya/2924292 suggests that the 3 first digits are only updated with // major toolchain updates. #define EIGEN_APPLE_DOUBLE_NEON_BUG (__apple_build_version__ < 6010000) #else #define EIGEN_APPLE_DOUBLE_NEON_BUG 0 #endif #if EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG // Bug 907: workaround missing declarations of the following two functions in the ADK // Defining these functions as templates ensures that if these intrinsics are // already defined in arm_neon.h, then our workaround doesn't cause a conflict // and has lower priority in overload resolution. template uint64x2_t vreinterpretq_u64_f64(T a) { return (uint64x2_t) a; } template float64x2_t vreinterpretq_f64_u64(T a) { return (float64x2_t) a; } typedef float64x2_t Packet2d; typedef float64x1_t Packet1d; // fuctionally equivalent to _mm_shuffle_pd in SSE (i.e. shuffle(m, n, mask) equals _mm_shuffle_pd(m,n,mask)) // Currently used in LU/arch/InverseSize4.h to enable a shared implementation // for fast inversion of matrices of size 4. EIGEN_STRONG_INLINE Packet2d shuffle(const Packet2d& m, const Packet2d& n, int mask) { const double* a = reinterpret_cast(&m); const double* b = reinterpret_cast(&n); Packet2d res = {*(a + (mask & 1)), *(b + ((mask >> 1) & 1))}; return res; } EIGEN_STRONG_INLINE Packet2d vec2d_swizzle2(const Packet2d& a, const Packet2d& b, int mask) { return shuffle(a, b, mask); } EIGEN_STRONG_INLINE Packet2d vec2d_unpacklo(const Packet2d& a,const Packet2d& b) { return shuffle(a, b, 0); } EIGEN_STRONG_INLINE Packet2d vec2d_unpackhi(const Packet2d& a,const Packet2d& b) { return shuffle(a, b, 3); } #define vec2d_duplane(a, p) \ vdupq_laneq_f64(a, p) template<> struct packet_traits : default_packet_traits { typedef Packet2d type; typedef Packet2d half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 2, HasHalfPacket = 0, HasCmp = 1, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasNegate = 1, HasAbs = 1, HasArg = 0, HasAbs2 = 1, HasAbsDiff = 1, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasBlend = 0, HasDiv = 1, HasFloor = 1, HasCeil = 1, HasRint = 1, HasSin = 0, HasCos = 0, HasLog = 1, HasExp = 1, HasSqrt = 1, HasRsqrt = 1, HasTanh = 0, HasErf = 0 }; }; template<> struct unpacket_traits { typedef double type; typedef Packet2d half; typedef Packet2l integer_packet; enum { size = 2, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> EIGEN_STRONG_INLINE Packet2d pset1(const double& from) { return vdupq_n_f64(from); } template<> EIGEN_STRONG_INLINE Packet2d plset(const double& a) { const double c[] = {0.0,1.0}; return vaddq_f64(pset1(a), vld1q_f64(c)); } template<> EIGEN_STRONG_INLINE Packet2d padd(const Packet2d& a, const Packet2d& b) { return vaddq_f64(a,b); } template<> EIGEN_STRONG_INLINE Packet2d psub(const Packet2d& a, const Packet2d& b) { return vsubq_f64(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pxor(const Packet2d& , const Packet2d& ); template<> EIGEN_STRONG_INLINE Packet2d paddsub(const Packet2d& a, const Packet2d& b){ const Packet2d mask = {numext::bit_cast(0x8000000000000000ull),0.0}; return padd(a, pxor(mask, b)); } template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) { return vnegq_f64(a); } template<> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { return a; } template<> EIGEN_STRONG_INLINE Packet2d pmul(const Packet2d& a, const Packet2d& b) { return vmulq_f64(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pdiv(const Packet2d& a, const Packet2d& b) { return vdivq_f64(a,b); } #ifdef __ARM_FEATURE_FMA // See bug 936. See above comment about FMA for float. template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return vfmaq_f64(c,a,b); } #else template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return vmlaq_f64(c,a,b); } #endif template<> EIGEN_STRONG_INLINE Packet2d pmin(const Packet2d& a, const Packet2d& b) { return vminq_f64(a,b); } #ifdef __ARM_FEATURE_NUMERIC_MAXMIN // numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8 systems). template<> EIGEN_STRONG_INLINE Packet2d pmin(const Packet2d& a, const Packet2d& b) { return vminnmq_f64(a, b); } template<> EIGEN_STRONG_INLINE Packet2d pmax(const Packet2d& a, const Packet2d& b) { return vmaxnmq_f64(a, b); } #endif template<> EIGEN_STRONG_INLINE Packet2d pmin(const Packet2d& a, const Packet2d& b) { return pmin(a, b); } template<> EIGEN_STRONG_INLINE Packet2d pmax(const Packet2d& a, const Packet2d& b) { return vmaxq_f64(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pmax(const Packet2d& a, const Packet2d& b) { return pmax(a, b); } // Logical Operations are not supported for float, so we have to reinterpret casts using NEON intrinsics template<> EIGEN_STRONG_INLINE Packet2d pand(const Packet2d& a, const Packet2d& b) { return vreinterpretq_f64_u64(vandq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); } template<> EIGEN_STRONG_INLINE Packet2d por(const Packet2d& a, const Packet2d& b) { return vreinterpretq_f64_u64(vorrq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); } template<> EIGEN_STRONG_INLINE Packet2d pxor(const Packet2d& a, const Packet2d& b) { return vreinterpretq_f64_u64(veorq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); } template<> EIGEN_STRONG_INLINE Packet2d pandnot(const Packet2d& a, const Packet2d& b) { return vreinterpretq_f64_u64(vbicq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); } template<> EIGEN_STRONG_INLINE Packet2d pcmp_le(const Packet2d& a, const Packet2d& b) { return vreinterpretq_f64_u64(vcleq_f64(a,b)); } template<> EIGEN_STRONG_INLINE Packet2d pcmp_lt(const Packet2d& a, const Packet2d& b) { return vreinterpretq_f64_u64(vcltq_f64(a,b)); } template<> EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan(const Packet2d& a, const Packet2d& b) { return vreinterpretq_f64_u32(vmvnq_u32(vreinterpretq_u32_u64(vcgeq_f64(a,b)))); } template<> EIGEN_STRONG_INLINE Packet2d pcmp_eq(const Packet2d& a, const Packet2d& b) { return vreinterpretq_f64_u64(vceqq_f64(a,b)); } template<> EIGEN_STRONG_INLINE Packet2d pload(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f64(from); } template<> EIGEN_STRONG_INLINE Packet2d ploadu(const double* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f64(from); } template<> EIGEN_STRONG_INLINE Packet2d ploaddup(const double* from) { return vld1q_dup_f64(from); } template<> EIGEN_STRONG_INLINE void pstore(double* to, const Packet2d& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_f64(to,from); } template<> EIGEN_STRONG_INLINE void pstoreu(double* to, const Packet2d& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_f64(to,from); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pgather(const double* from, Index stride) { Packet2d res = pset1(0.0); res = vld1q_lane_f64(from + 0*stride, res, 0); res = vld1q_lane_f64(from + 1*stride, res, 1); return res; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(double* to, const Packet2d& from, Index stride) { vst1q_lane_f64(to + stride*0, from, 0); vst1q_lane_f64(to + stride*1, from, 1); } template<> EIGEN_STRONG_INLINE void prefetch(const double* addr) { EIGEN_ARM_PREFETCH(addr); } // FIXME only store the 2 first elements ? template<> EIGEN_STRONG_INLINE double pfirst(const Packet2d& a) { return vgetq_lane_f64(a,0); } template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) { return vcombine_f64(vget_high_f64(a), vget_low_f64(a)); } template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { return vabsq_f64(a); } #if EIGEN_COMP_CLANG && defined(__apple_build_version__) // workaround ICE, see bug 907 template<> EIGEN_STRONG_INLINE double predux(const Packet2d& a) { return (vget_low_f64(a) + vget_high_f64(a))[0]; } #else template<> EIGEN_STRONG_INLINE double predux(const Packet2d& a) { return vget_lane_f64(vget_low_f64(a) + vget_high_f64(a), 0); } #endif // Other reduction functions: // mul #if EIGEN_COMP_CLANG && defined(__apple_build_version__) template<> EIGEN_STRONG_INLINE double predux_mul(const Packet2d& a) { return (vget_low_f64(a) * vget_high_f64(a))[0]; } #else template<> EIGEN_STRONG_INLINE double predux_mul(const Packet2d& a) { return vget_lane_f64(vget_low_f64(a) * vget_high_f64(a), 0); } #endif // min template<> EIGEN_STRONG_INLINE double predux_min(const Packet2d& a) { return vgetq_lane_f64(vpminq_f64(a,a), 0); } // max template<> EIGEN_STRONG_INLINE double predux_max(const Packet2d& a) { return vgetq_lane_f64(vpmaxq_f64(a,a), 0); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const float64x2_t tmp1 = vzip1q_f64(kernel.packet[0], kernel.packet[1]); const float64x2_t tmp2 = vzip2q_f64(kernel.packet[0], kernel.packet[1]); kernel.packet[0] = tmp1; kernel.packet[1] = tmp2; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pselect( const Packet2d& mask, const Packet2d& a, const Packet2d& b) { return vbslq_f64(vreinterpretq_u64_f64(mask), a, b); } template<> EIGEN_STRONG_INLINE Packet2d print(const Packet2d& a) { return vrndnq_f64(a); } template<> EIGEN_STRONG_INLINE Packet2d pfloor(const Packet2d& a) { return vrndmq_f64(a); } template<> EIGEN_STRONG_INLINE Packet2d pceil(const Packet2d& a) { return vrndpq_f64(a); } template<> EIGEN_STRONG_INLINE Packet2d pldexp(const Packet2d& a, const Packet2d& exponent) { return pldexp_generic(a, exponent); } template<> EIGEN_STRONG_INLINE Packet2d pfrexp(const Packet2d& a, Packet2d& exponent) { return pfrexp_generic(a,exponent); } template<> EIGEN_STRONG_INLINE Packet2d pset1frombits(uint64_t from) { return vreinterpretq_f64_u64(vdupq_n_u64(from)); } template<> EIGEN_STRONG_INLINE Packet2d prsqrt(const Packet2d& a) { // Compute approximate reciprocal sqrt. Packet2d x = vrsqrteq_f64(a); // Do Newton iterations for 1/sqrt(x). x = vmulq_f64(vrsqrtsq_f64(vmulq_f64(a, x), x), x); x = vmulq_f64(vrsqrtsq_f64(vmulq_f64(a, x), x), x); x = vmulq_f64(vrsqrtsq_f64(vmulq_f64(a, x), x), x); const Packet2d infinity = pset1(NumTraits::infinity()); return pselect(pcmp_eq(a, pzero(a)), infinity, x); } template<> EIGEN_STRONG_INLINE Packet2d psqrt(const Packet2d& _x){ return vsqrtq_f64(_x); } #endif // EIGEN_ARCH_ARM64 // Do we have an fp16 types and supporting Neon intrinsics? #if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC typedef float16x4_t Packet4hf; typedef float16x8_t Packet8hf; template <> struct packet_traits : default_packet_traits { typedef Packet8hf type; typedef Packet4hf half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 8, HasHalfPacket = 1, HasCmp = 1, HasCast = 1, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasNegate = 1, HasAbs = 1, HasArg = 0, HasAbs2 = 1, HasAbsDiff = 0, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasBlend = 0, HasInsert = 1, HasReduxp = 1, HasDiv = 1, HasFloor = 1, HasCeil = 1, HasRint = 1, HasSin = 0, HasCos = 0, HasLog = 0, HasExp = 0, HasSqrt = 1, HasRsqrt = 1, HasErf = EIGEN_FAST_MATH, HasBessel = 0, // Issues with accuracy. HasNdtri = 0 }; }; template <> struct unpacket_traits { typedef Eigen::half type; typedef Packet4hf half; enum { size = 4, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template <> struct unpacket_traits { typedef Eigen::half type; typedef Packet4hf half; enum { size = 8, alignment = Aligned16, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf predux_half_dowto4(const Packet8hf& a) { return vadd_f16(vget_low_f16(a), vget_high_f16(a)); } template <> EIGEN_STRONG_INLINE Packet8hf pset1(const Eigen::half& from) { return vdupq_n_f16(from.x); } template <> EIGEN_STRONG_INLINE Packet4hf pset1(const Eigen::half& from) { return vdup_n_f16(from.x); } template <> EIGEN_STRONG_INLINE Packet8hf plset(const Eigen::half& a) { const float16_t f[] = {0, 1, 2, 3, 4, 5, 6, 7}; Packet8hf countdown = vld1q_f16(f); return vaddq_f16(pset1(a), countdown); } template <> EIGEN_STRONG_INLINE Packet4hf plset(const Eigen::half& a) { const float16_t f[] = {0, 1, 2, 3}; Packet4hf countdown = vld1_f16(f); return vadd_f16(pset1(a), countdown); } template <> EIGEN_STRONG_INLINE Packet8hf padd(const Packet8hf& a, const Packet8hf& b) { return vaddq_f16(a, b); } template <> EIGEN_STRONG_INLINE Packet4hf padd(const Packet4hf& a, const Packet4hf& b) { return vadd_f16(a, b); } template <> EIGEN_STRONG_INLINE Packet8hf psub(const Packet8hf& a, const Packet8hf& b) { return vsubq_f16(a, b); } template <> EIGEN_STRONG_INLINE Packet4hf psub(const Packet4hf& a, const Packet4hf& b) { return vsub_f16(a, b); } template <> EIGEN_STRONG_INLINE Packet8hf pnegate(const Packet8hf& a) { return vnegq_f16(a); } template <> EIGEN_STRONG_INLINE Packet4hf pnegate(const Packet4hf& a) { return vneg_f16(a); } template <> EIGEN_STRONG_INLINE Packet8hf pconj(const Packet8hf& a) { return a; } template <> EIGEN_STRONG_INLINE Packet4hf pconj(const Packet4hf& a) { return a; } template <> EIGEN_STRONG_INLINE Packet8hf pmul(const Packet8hf& a, const Packet8hf& b) { return vmulq_f16(a, b); } template <> EIGEN_STRONG_INLINE Packet4hf pmul(const Packet4hf& a, const Packet4hf& b) { return vmul_f16(a, b); } template <> EIGEN_STRONG_INLINE Packet8hf pdiv(const Packet8hf& a, const Packet8hf& b) { return vdivq_f16(a, b); } template <> EIGEN_STRONG_INLINE Packet4hf pdiv(const Packet4hf& a, const Packet4hf& b) { return vdiv_f16(a, b); } template <> EIGEN_STRONG_INLINE Packet8hf pmadd(const Packet8hf& a, const Packet8hf& b, const Packet8hf& c) { return vfmaq_f16(c, a, b); } template <> EIGEN_STRONG_INLINE Packet4hf pmadd(const Packet4hf& a, const Packet4hf& b, const Packet4hf& c) { return vfma_f16(c, a, b); } template <> EIGEN_STRONG_INLINE Packet8hf pmin(const Packet8hf& a, const Packet8hf& b) { return vminq_f16(a, b); } template <> EIGEN_STRONG_INLINE Packet4hf pmin(const Packet4hf& a, const Packet4hf& b) { return vmin_f16(a, b); } #ifdef __ARM_FEATURE_NUMERIC_MAXMIN // numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8 systems). template<> EIGEN_STRONG_INLINE Packet4hf pmin(const Packet4hf& a, const Packet4hf& b) { return vminnm_f16(a, b); } template<> EIGEN_STRONG_INLINE Packet8hf pmin(const Packet8hf& a, const Packet8hf& b) { return vminnmq_f16(a, b); } #endif template<> EIGEN_STRONG_INLINE Packet4hf pmin(const Packet4hf& a, const Packet4hf& b) { return pmin(a, b); } template<> EIGEN_STRONG_INLINE Packet8hf pmin(const Packet8hf& a, const Packet8hf& b) { return pmin(a, b); } template <> EIGEN_STRONG_INLINE Packet8hf pmax(const Packet8hf& a, const Packet8hf& b) { return vmaxq_f16(a, b); } template <> EIGEN_STRONG_INLINE Packet4hf pmax(const Packet4hf& a, const Packet4hf& b) { return vmax_f16(a, b); } #ifdef __ARM_FEATURE_NUMERIC_MAXMIN // numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8 systems). template<> EIGEN_STRONG_INLINE Packet4hf pmax(const Packet4hf& a, const Packet4hf& b) { return vmaxnm_f16(a, b); } template<> EIGEN_STRONG_INLINE Packet8hf pmax(const Packet8hf& a, const Packet8hf& b) { return vmaxnmq_f16(a, b); } #endif template<> EIGEN_STRONG_INLINE Packet4hf pmax(const Packet4hf& a, const Packet4hf& b) { return pmax(a, b); } template<> EIGEN_STRONG_INLINE Packet8hf pmax(const Packet8hf& a, const Packet8hf& b) { return pmax(a, b); } #define EIGEN_MAKE_ARM_FP16_CMP_8(name) \ template <> \ EIGEN_STRONG_INLINE Packet8hf pcmp_##name(const Packet8hf& a, const Packet8hf& b) { \ return vreinterpretq_f16_u16(vc##name##q_f16(a, b)); \ } #define EIGEN_MAKE_ARM_FP16_CMP_4(name) \ template <> \ EIGEN_STRONG_INLINE Packet4hf pcmp_##name(const Packet4hf& a, const Packet4hf& b) { \ return vreinterpret_f16_u16(vc##name##_f16(a, b)); \ } EIGEN_MAKE_ARM_FP16_CMP_8(eq) EIGEN_MAKE_ARM_FP16_CMP_8(lt) EIGEN_MAKE_ARM_FP16_CMP_8(le) EIGEN_MAKE_ARM_FP16_CMP_4(eq) EIGEN_MAKE_ARM_FP16_CMP_4(lt) EIGEN_MAKE_ARM_FP16_CMP_4(le) #undef EIGEN_MAKE_ARM_FP16_CMP_8 #undef EIGEN_MAKE_ARM_FP16_CMP_4 template <> EIGEN_STRONG_INLINE Packet8hf pcmp_lt_or_nan(const Packet8hf& a, const Packet8hf& b) { return vreinterpretq_f16_u16(vmvnq_u16(vcgeq_f16(a, b))); } template <> EIGEN_STRONG_INLINE Packet4hf pcmp_lt_or_nan(const Packet4hf& a, const Packet4hf& b) { return vreinterpret_f16_u16(vmvn_u16(vcge_f16(a, b))); } template <> EIGEN_STRONG_INLINE Packet8hf print(const Packet8hf& a) { return vrndnq_f16(a); } template <> EIGEN_STRONG_INLINE Packet4hf print(const Packet4hf& a) { return vrndn_f16(a); } template <> EIGEN_STRONG_INLINE Packet8hf pfloor(const Packet8hf& a) { return vrndmq_f16(a); } template <> EIGEN_STRONG_INLINE Packet4hf pfloor(const Packet4hf& a) { return vrndm_f16(a); } template <> EIGEN_STRONG_INLINE Packet8hf pceil(const Packet8hf& a) { return vrndpq_f16(a); } template <> EIGEN_STRONG_INLINE Packet4hf pceil(const Packet4hf& a) { return vrndp_f16(a); } template <> EIGEN_STRONG_INLINE Packet8hf psqrt(const Packet8hf& a) { return vsqrtq_f16(a); } template <> EIGEN_STRONG_INLINE Packet4hf psqrt(const Packet4hf& a) { return vsqrt_f16(a); } template <> EIGEN_STRONG_INLINE Packet8hf pand(const Packet8hf& a, const Packet8hf& b) { return vreinterpretq_f16_u16(vandq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b))); } template <> EIGEN_STRONG_INLINE Packet4hf pand(const Packet4hf& a, const Packet4hf& b) { return vreinterpret_f16_u16(vand_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b))); } template <> EIGEN_STRONG_INLINE Packet8hf por(const Packet8hf& a, const Packet8hf& b) { return vreinterpretq_f16_u16(vorrq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b))); } template <> EIGEN_STRONG_INLINE Packet4hf por(const Packet4hf& a, const Packet4hf& b) { return vreinterpret_f16_u16(vorr_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b))); } template <> EIGEN_STRONG_INLINE Packet8hf pxor(const Packet8hf& a, const Packet8hf& b) { return vreinterpretq_f16_u16(veorq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b))); } template <> EIGEN_STRONG_INLINE Packet4hf pxor(const Packet4hf& a, const Packet4hf& b) { return vreinterpret_f16_u16(veor_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b))); } template <> EIGEN_STRONG_INLINE Packet8hf pandnot(const Packet8hf& a, const Packet8hf& b) { return vreinterpretq_f16_u16(vbicq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b))); } template <> EIGEN_STRONG_INLINE Packet4hf pandnot(const Packet4hf& a, const Packet4hf& b) { return vreinterpret_f16_u16(vbic_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b))); } template <> EIGEN_STRONG_INLINE Packet8hf pload(const Eigen::half* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f16(reinterpret_cast(from)); } template <> EIGEN_STRONG_INLINE Packet4hf pload(const Eigen::half* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1_f16(reinterpret_cast(from)); } template <> EIGEN_STRONG_INLINE Packet8hf ploadu(const Eigen::half* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f16(reinterpret_cast(from)); } template <> EIGEN_STRONG_INLINE Packet4hf ploadu(const Eigen::half* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1_f16(reinterpret_cast(from)); } template <> EIGEN_STRONG_INLINE Packet8hf ploaddup(const Eigen::half* from) { Packet8hf packet; packet[0] = from[0].x; packet[1] = from[0].x; packet[2] = from[1].x; packet[3] = from[1].x; packet[4] = from[2].x; packet[5] = from[2].x; packet[6] = from[3].x; packet[7] = from[3].x; return packet; } template <> EIGEN_STRONG_INLINE Packet4hf ploaddup(const Eigen::half* from) { float16x4_t packet; float16_t* tmp; tmp = (float16_t*)&packet; tmp[0] = from[0].x; tmp[1] = from[0].x; tmp[2] = from[1].x; tmp[3] = from[1].x; return packet; } template <> EIGEN_STRONG_INLINE Packet8hf ploadquad(const Eigen::half* from) { Packet4hf lo, hi; lo = vld1_dup_f16(reinterpret_cast(from)); hi = vld1_dup_f16(reinterpret_cast(from+1)); return vcombine_f16(lo, hi); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pinsertfirst(const Packet8hf& a, Eigen::half b) { return vsetq_lane_f16(b.x, a, 0); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pinsertfirst(const Packet4hf& a, Eigen::half b) { return vset_lane_f16(b.x, a, 0); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pselect(const Packet8hf& mask, const Packet8hf& a, const Packet8hf& b) { return vbslq_f16(vreinterpretq_u16_f16(mask), a, b); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pselect(const Packet4hf& mask, const Packet4hf& a, const Packet4hf& b) { return vbsl_f16(vreinterpret_u16_f16(mask), a, b); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pinsertlast(const Packet8hf& a, Eigen::half b) { return vsetq_lane_f16(b.x, a, 7); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pinsertlast(const Packet4hf& a, Eigen::half b) { return vset_lane_f16(b.x, a, 3); } template <> EIGEN_STRONG_INLINE void pstore(Eigen::half* to, const Packet8hf& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_f16(reinterpret_cast(to), from); } template <> EIGEN_STRONG_INLINE void pstore(Eigen::half* to, const Packet4hf& from) { EIGEN_DEBUG_ALIGNED_STORE vst1_f16(reinterpret_cast(to), from); } template <> EIGEN_STRONG_INLINE void pstoreu(Eigen::half* to, const Packet8hf& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_f16(reinterpret_cast(to), from); } template <> EIGEN_STRONG_INLINE void pstoreu(Eigen::half* to, const Packet4hf& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1_f16(reinterpret_cast(to), from); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pgather(const Eigen::half* from, Index stride) { Packet8hf res = pset1(Eigen::half(0.f)); res = vsetq_lane_f16(from[0 * stride].x, res, 0); res = vsetq_lane_f16(from[1 * stride].x, res, 1); res = vsetq_lane_f16(from[2 * stride].x, res, 2); res = vsetq_lane_f16(from[3 * stride].x, res, 3); res = vsetq_lane_f16(from[4 * stride].x, res, 4); res = vsetq_lane_f16(from[5 * stride].x, res, 5); res = vsetq_lane_f16(from[6 * stride].x, res, 6); res = vsetq_lane_f16(from[7 * stride].x, res, 7); return res; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pgather(const Eigen::half* from, Index stride) { Packet4hf res = pset1(Eigen::half(0.f)); res = vset_lane_f16(from[0 * stride].x, res, 0); res = vset_lane_f16(from[1 * stride].x, res, 1); res = vset_lane_f16(from[2 * stride].x, res, 2); res = vset_lane_f16(from[3 * stride].x, res, 3); return res; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(Eigen::half* to, const Packet8hf& from, Index stride) { to[stride * 0].x = vgetq_lane_f16(from, 0); to[stride * 1].x = vgetq_lane_f16(from, 1); to[stride * 2].x = vgetq_lane_f16(from, 2); to[stride * 3].x = vgetq_lane_f16(from, 3); to[stride * 4].x = vgetq_lane_f16(from, 4); to[stride * 5].x = vgetq_lane_f16(from, 5); to[stride * 6].x = vgetq_lane_f16(from, 6); to[stride * 7].x = vgetq_lane_f16(from, 7); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(Eigen::half* to, const Packet4hf& from, Index stride) { to[stride * 0].x = vget_lane_f16(from, 0); to[stride * 1].x = vget_lane_f16(from, 1); to[stride * 2].x = vget_lane_f16(from, 2); to[stride * 3].x = vget_lane_f16(from, 3); } template <> EIGEN_STRONG_INLINE void prefetch(const Eigen::half* addr) { EIGEN_ARM_PREFETCH(addr); } template <> EIGEN_STRONG_INLINE Eigen::half pfirst(const Packet8hf& a) { float16_t x[8]; vst1q_f16(x, a); Eigen::half h; h.x = x[0]; return h; } template <> EIGEN_STRONG_INLINE Eigen::half pfirst(const Packet4hf& a) { float16_t x[4]; vst1_f16(x, a); Eigen::half h; h.x = x[0]; return h; } template<> EIGEN_STRONG_INLINE Packet8hf preverse(const Packet8hf& a) { float16x4_t a_lo, a_hi; Packet8hf a_r64; a_r64 = vrev64q_f16(a); a_lo = vget_low_f16(a_r64); a_hi = vget_high_f16(a_r64); return vcombine_f16(a_hi, a_lo); } template <> EIGEN_STRONG_INLINE Packet4hf preverse(const Packet4hf& a) { return vrev64_f16(a); } template <> EIGEN_STRONG_INLINE Packet8hf pabs(const Packet8hf& a) { return vabsq_f16(a); } template <> EIGEN_STRONG_INLINE Packet4hf pabs(const Packet4hf& a) { return vabs_f16(a); } template <> EIGEN_STRONG_INLINE Eigen::half predux(const Packet8hf& a) { float16x4_t a_lo, a_hi, sum; a_lo = vget_low_f16(a); a_hi = vget_high_f16(a); sum = vpadd_f16(a_lo, a_hi); sum = vpadd_f16(sum, sum); sum = vpadd_f16(sum, sum); Eigen::half h; h.x = vget_lane_f16(sum, 0); return h; } template <> EIGEN_STRONG_INLINE Eigen::half predux(const Packet4hf& a) { float16x4_t sum; sum = vpadd_f16(a, a); sum = vpadd_f16(sum, sum); Eigen::half h; h.x = vget_lane_f16(sum, 0); return h; } template <> EIGEN_STRONG_INLINE Eigen::half predux_mul(const Packet8hf& a) { float16x4_t a_lo, a_hi, prod; a_lo = vget_low_f16(a); a_hi = vget_high_f16(a); prod = vmul_f16(a_lo, a_hi); prod = vmul_f16(prod, vrev64_f16(prod)); Eigen::half h; h.x = vmulh_f16(vget_lane_f16(prod, 0), vget_lane_f16(prod, 1)); return h; } template <> EIGEN_STRONG_INLINE Eigen::half predux_mul(const Packet4hf& a) { float16x4_t prod; prod = vmul_f16(a, vrev64_f16(a)); Eigen::half h; h.x = vmulh_f16(vget_lane_f16(prod, 0), vget_lane_f16(prod, 1)); return h; } template <> EIGEN_STRONG_INLINE Eigen::half predux_min(const Packet8hf& a) { float16x4_t a_lo, a_hi, min; a_lo = vget_low_f16(a); a_hi = vget_high_f16(a); min = vpmin_f16(a_lo, a_hi); min = vpmin_f16(min, min); min = vpmin_f16(min, min); Eigen::half h; h.x = vget_lane_f16(min, 0); return h; } template <> EIGEN_STRONG_INLINE Eigen::half predux_min(const Packet4hf& a) { Packet4hf tmp; tmp = vpmin_f16(a, a); tmp = vpmin_f16(tmp, tmp); Eigen::half h; h.x = vget_lane_f16(tmp, 0); return h; } template <> EIGEN_STRONG_INLINE Eigen::half predux_max(const Packet8hf& a) { float16x4_t a_lo, a_hi, max; a_lo = vget_low_f16(a); a_hi = vget_high_f16(a); max = vpmax_f16(a_lo, a_hi); max = vpmax_f16(max, max); max = vpmax_f16(max, max); Eigen::half h; h.x = vget_lane_f16(max, 0); return h; } template <> EIGEN_STRONG_INLINE Eigen::half predux_max(const Packet4hf& a) { Packet4hf tmp; tmp = vpmax_f16(a, a); tmp = vpmax_f16(tmp, tmp); Eigen::half h; h.x = vget_lane_f16(tmp, 0); return h; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const float16x8x2_t zip16_1 = vzipq_f16(kernel.packet[0], kernel.packet[1]); const float16x8x2_t zip16_2 = vzipq_f16(kernel.packet[2], kernel.packet[3]); const float32x4x2_t zip32_1 = vzipq_f32(vreinterpretq_f32_f16(zip16_1.val[0]), vreinterpretq_f32_f16(zip16_2.val[0])); const float32x4x2_t zip32_2 = vzipq_f32(vreinterpretq_f32_f16(zip16_1.val[1]), vreinterpretq_f32_f16(zip16_2.val[1])); kernel.packet[0] = vreinterpretq_f16_f32(zip32_1.val[0]); kernel.packet[1] = vreinterpretq_f16_f32(zip32_1.val[1]); kernel.packet[2] = vreinterpretq_f16_f32(zip32_2.val[0]); kernel.packet[3] = vreinterpretq_f16_f32(zip32_2.val[1]); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { EIGEN_ALIGN16 float16x4x4_t tmp_x4; float16_t* tmp = (float16_t*)&kernel; tmp_x4 = vld4_f16(tmp); kernel.packet[0] = tmp_x4.val[0]; kernel.packet[1] = tmp_x4.val[1]; kernel.packet[2] = tmp_x4.val[2]; kernel.packet[3] = tmp_x4.val[3]; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { float16x8x2_t T_1[4]; T_1[0] = vuzpq_f16(kernel.packet[0], kernel.packet[1]); T_1[1] = vuzpq_f16(kernel.packet[2], kernel.packet[3]); T_1[2] = vuzpq_f16(kernel.packet[4], kernel.packet[5]); T_1[3] = vuzpq_f16(kernel.packet[6], kernel.packet[7]); float16x8x2_t T_2[4]; T_2[0] = vuzpq_f16(T_1[0].val[0], T_1[1].val[0]); T_2[1] = vuzpq_f16(T_1[0].val[1], T_1[1].val[1]); T_2[2] = vuzpq_f16(T_1[2].val[0], T_1[3].val[0]); T_2[3] = vuzpq_f16(T_1[2].val[1], T_1[3].val[1]); float16x8x2_t T_3[4]; T_3[0] = vuzpq_f16(T_2[0].val[0], T_2[2].val[0]); T_3[1] = vuzpq_f16(T_2[0].val[1], T_2[2].val[1]); T_3[2] = vuzpq_f16(T_2[1].val[0], T_2[3].val[0]); T_3[3] = vuzpq_f16(T_2[1].val[1], T_2[3].val[1]); kernel.packet[0] = T_3[0].val[0]; kernel.packet[1] = T_3[2].val[0]; kernel.packet[2] = T_3[1].val[0]; kernel.packet[3] = T_3[3].val[0]; kernel.packet[4] = T_3[0].val[1]; kernel.packet[5] = T_3[2].val[1]; kernel.packet[6] = T_3[1].val[1]; kernel.packet[7] = T_3[3].val[1]; } #endif // end EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC } // end namespace internal } // end namespace Eigen #endif // EIGEN_PACKET_MATH_NEON_H RcppEigen/inst/include/Eigen/src/Core/arch/NEON/MathFunctions.h0000644000176200001440000000601314567757725023674 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_MATH_FUNCTIONS_NEON_H #define EIGEN_MATH_FUNCTIONS_NEON_H namespace Eigen { namespace internal { template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2f pexp(const Packet2f& x) { return pexp_float(x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pexp(const Packet4f& x) { return pexp_float(x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2f plog(const Packet2f& x) { return plog_float(x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f plog(const Packet4f& x) { return plog_float(x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2f psin(const Packet2f& x) { return psin_float(x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f psin(const Packet4f& x) { return psin_float(x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2f pcos(const Packet2f& x) { return pcos_float(x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pcos(const Packet4f& x) { return pcos_float(x); } // Hyperbolic Tangent function. template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2f ptanh(const Packet2f& x) { return internal::generic_fast_tanh_float(x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f ptanh(const Packet4f& x) { return internal::generic_fast_tanh_float(x); } BF16_PACKET_FUNCTION(Packet4f, Packet4bf, psin) BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pcos) BF16_PACKET_FUNCTION(Packet4f, Packet4bf, plog) BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pexp) BF16_PACKET_FUNCTION(Packet4f, Packet4bf, ptanh) template <> EIGEN_STRONG_INLINE Packet4bf pfrexp(const Packet4bf& a, Packet4bf& exponent) { Packet4f fexponent; const Packet4bf out = F32ToBf16(pfrexp(Bf16ToF32(a), fexponent)); exponent = F32ToBf16(fexponent); return out; } template <> EIGEN_STRONG_INLINE Packet4bf pldexp(const Packet4bf& a, const Packet4bf& exponent) { return F32ToBf16(pldexp(Bf16ToF32(a), Bf16ToF32(exponent))); } //---------- double ---------- #if EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d pexp(const Packet2d& x) { return pexp_double(x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d plog(const Packet2d& x) { return plog_double(x); } #endif } // end namespace internal } // end namespace Eigen #endif // EIGEN_MATH_FUNCTIONS_NEON_H RcppEigen/inst/include/Eigen/src/Core/arch/MSA/0000755000176200001440000000000014567757725020622 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/arch/MSA/Complex.h0000644000176200001440000004220514567757725022405 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2018 Wave Computing, Inc. // Written by: // Chris Larsen // Alexey Frunze (afrunze@wavecomp.com) // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_COMPLEX_MSA_H #define EIGEN_COMPLEX_MSA_H #include namespace Eigen { namespace internal { //---------- float ---------- struct Packet2cf { EIGEN_STRONG_INLINE Packet2cf() { } EIGEN_STRONG_INLINE explicit Packet2cf(const std::complex& a, const std::complex& b) { Packet4f t = { std::real(a), std::imag(a), std::real(b), std::imag(b) }; v = t; } EIGEN_STRONG_INLINE explicit Packet2cf(const Packet4f& a) : v(a) { } EIGEN_STRONG_INLINE Packet2cf(const Packet2cf& a) : v(a.v) { } EIGEN_STRONG_INLINE Packet2cf& operator=(const Packet2cf& b) { v = b.v; return *this; } EIGEN_STRONG_INLINE Packet2cf conjugate(void) const { return Packet2cf((Packet4f)__builtin_msa_bnegi_d((v2u64)v, 63)); } EIGEN_STRONG_INLINE Packet2cf& operator*=(const Packet2cf& b) { Packet4f v1, v2; // Get the real values of a | a1_re | a1_re | a2_re | a2_re | v1 = (Packet4f)__builtin_msa_ilvev_w((v4i32)v, (v4i32)v); // Get the imag values of a | a1_im | a1_im | a2_im | a2_im | v2 = (Packet4f)__builtin_msa_ilvod_w((v4i32)v, (v4i32)v); // Multiply the real a with b v1 = pmul(v1, b.v); // Multiply the imag a with b v2 = pmul(v2, b.v); // Conjugate v2 v2 = Packet2cf(v2).conjugate().v; // Swap real/imag elements in v2. v2 = (Packet4f)__builtin_msa_shf_w((v4i32)v2, EIGEN_MSA_SHF_I8(1, 0, 3, 2)); // Add and return the result v = padd(v1, v2); return *this; } EIGEN_STRONG_INLINE Packet2cf operator*(const Packet2cf& b) const { return Packet2cf(*this) *= b; } EIGEN_STRONG_INLINE Packet2cf& operator+=(const Packet2cf& b) { v = padd(v, b.v); return *this; } EIGEN_STRONG_INLINE Packet2cf operator+(const Packet2cf& b) const { return Packet2cf(*this) += b; } EIGEN_STRONG_INLINE Packet2cf& operator-=(const Packet2cf& b) { v = psub(v, b.v); return *this; } EIGEN_STRONG_INLINE Packet2cf operator-(const Packet2cf& b) const { return Packet2cf(*this) -= b; } EIGEN_STRONG_INLINE Packet2cf& operator/=(const Packet2cf& b) { *this *= b.conjugate(); Packet4f s = pmul(b.v, b.v); s = padd(s, (Packet4f)__builtin_msa_shf_w((v4i32)s, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); v = pdiv(v, s); return *this; } EIGEN_STRONG_INLINE Packet2cf operator/(const Packet2cf& b) const { return Packet2cf(*this) /= b; } EIGEN_STRONG_INLINE Packet2cf operator-(void) const { return Packet2cf(pnegate(v)); } Packet4f v; }; inline std::ostream& operator<<(std::ostream& os, const Packet2cf& value) { os << "[ (" << value.v[0] << ", " << value.v[1] << "i)," " (" << value.v[2] << ", " << value.v[3] << "i) ]"; return os; } template <> struct packet_traits > : default_packet_traits { typedef Packet2cf type; typedef Packet2cf half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 2, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, HasMax = 0, HasSetLinear = 0, HasBlend = 1 }; }; template <> struct unpacket_traits { typedef std::complex type; enum { size = 2, alignment = Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false }; typedef Packet2cf half; }; template <> EIGEN_STRONG_INLINE Packet2cf pset1(const std::complex& from) { EIGEN_MSA_DEBUG; float f0 = from.real(), f1 = from.imag(); Packet4f v0 = { f0, f0, f0, f0 }; Packet4f v1 = { f1, f1, f1, f1 }; return Packet2cf((Packet4f)__builtin_msa_ilvr_w((Packet4i)v1, (Packet4i)v0)); } template <> EIGEN_STRONG_INLINE Packet2cf padd(const Packet2cf& a, const Packet2cf& b) { EIGEN_MSA_DEBUG; return a + b; } template <> EIGEN_STRONG_INLINE Packet2cf psub(const Packet2cf& a, const Packet2cf& b) { EIGEN_MSA_DEBUG; return a - b; } template <> EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf& a) { EIGEN_MSA_DEBUG; return -a; } template <> EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) { EIGEN_MSA_DEBUG; return a.conjugate(); } template <> EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) { EIGEN_MSA_DEBUG; return a * b; } template <> EIGEN_STRONG_INLINE Packet2cf pand(const Packet2cf& a, const Packet2cf& b) { EIGEN_MSA_DEBUG; return Packet2cf(pand(a.v, b.v)); } template <> EIGEN_STRONG_INLINE Packet2cf por(const Packet2cf& a, const Packet2cf& b) { EIGEN_MSA_DEBUG; return Packet2cf(por(a.v, b.v)); } template <> EIGEN_STRONG_INLINE Packet2cf pxor(const Packet2cf& a, const Packet2cf& b) { EIGEN_MSA_DEBUG; return Packet2cf(pxor(a.v, b.v)); } template <> EIGEN_STRONG_INLINE Packet2cf pandnot(const Packet2cf& a, const Packet2cf& b) { EIGEN_MSA_DEBUG; return Packet2cf(pandnot(a.v, b.v)); } template <> EIGEN_STRONG_INLINE Packet2cf pload(const std::complex* from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_ALIGNED_LOAD return Packet2cf(pload((const float*)from)); } template <> EIGEN_STRONG_INLINE Packet2cf ploadu(const std::complex* from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cf(ploadu((const float*)from)); } template <> EIGEN_STRONG_INLINE Packet2cf ploaddup(const std::complex* from) { EIGEN_MSA_DEBUG; return pset1(*from); } template <> EIGEN_STRONG_INLINE void pstore >(std::complex* to, const Packet2cf& from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_ALIGNED_STORE pstore((float*)to, from.v); } template <> EIGEN_STRONG_INLINE void pstoreu >(std::complex* to, const Packet2cf& from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_UNALIGNED_STORE pstoreu((float*)to, from.v); } template <> EIGEN_DEVICE_FUNC inline Packet2cf pgather, Packet2cf>( const std::complex* from, Index stride) { EIGEN_MSA_DEBUG; return Packet2cf(from[0 * stride], from[1 * stride]); } template <> EIGEN_DEVICE_FUNC inline void pscatter, Packet2cf>(std::complex* to, const Packet2cf& from, Index stride) { EIGEN_MSA_DEBUG; *to = std::complex(from.v[0], from.v[1]); to += stride; *to = std::complex(from.v[2], from.v[3]); } template <> EIGEN_STRONG_INLINE void prefetch >(const std::complex* addr) { EIGEN_MSA_DEBUG; prefetch(reinterpret_cast(addr)); } template <> EIGEN_STRONG_INLINE std::complex pfirst(const Packet2cf& a) { EIGEN_MSA_DEBUG; return std::complex(a.v[0], a.v[1]); } template <> EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) { EIGEN_MSA_DEBUG; return Packet2cf((Packet4f)__builtin_msa_shf_w((v4i32)a.v, EIGEN_MSA_SHF_I8(2, 3, 0, 1))); } template <> EIGEN_STRONG_INLINE Packet2cf pcplxflip(const Packet2cf& a) { EIGEN_MSA_DEBUG; return Packet2cf((Packet4f)__builtin_msa_shf_w((v4i32)a.v, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); } template <> EIGEN_STRONG_INLINE std::complex predux(const Packet2cf& a) { EIGEN_MSA_DEBUG; Packet4f value = (Packet4f)preverse((Packet2d)a.v); value += a.v; return std::complex(value[0], value[1]); } template <> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet2cf& a) { EIGEN_MSA_DEBUG; return std::complex((a.v[0] * a.v[2]) - (a.v[1] * a.v[3]), (a.v[0] * a.v[3]) + (a.v[1] * a.v[2])); } EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf, Packet4f) template <> EIGEN_STRONG_INLINE Packet2cf pdiv(const Packet2cf& a, const Packet2cf& b) { EIGEN_MSA_DEBUG; return a / b; } inline std::ostream& operator<<(std::ostream& os, const PacketBlock& value) { os << "[ " << value.packet[0] << ", " << std::endl << " " << value.packet[1] << " ]"; return os; } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { EIGEN_MSA_DEBUG; Packet4f tmp = (Packet4f)__builtin_msa_ilvl_d((v2i64)kernel.packet[1].v, (v2i64)kernel.packet[0].v); kernel.packet[0].v = (Packet4f)__builtin_msa_ilvr_d((v2i64)kernel.packet[1].v, (v2i64)kernel.packet[0].v); kernel.packet[1].v = tmp; } template <> EIGEN_STRONG_INLINE Packet2cf pblend(const Selector<2>& ifPacket, const Packet2cf& thenPacket, const Packet2cf& elsePacket) { return (Packet2cf)(Packet4f)pblend(ifPacket, (Packet2d)thenPacket.v, (Packet2d)elsePacket.v); } //---------- double ---------- struct Packet1cd { EIGEN_STRONG_INLINE Packet1cd() { } EIGEN_STRONG_INLINE explicit Packet1cd(const std::complex& a) { v[0] = std::real(a); v[1] = std::imag(a); } EIGEN_STRONG_INLINE explicit Packet1cd(const Packet2d& a) : v(a) { } EIGEN_STRONG_INLINE Packet1cd(const Packet1cd& a) : v(a.v) { } EIGEN_STRONG_INLINE Packet1cd& operator=(const Packet1cd& b) { v = b.v; return *this; } EIGEN_STRONG_INLINE Packet1cd conjugate(void) const { static const v2u64 p2ul_CONJ_XOR = { 0x0, 0x8000000000000000 }; return (Packet1cd)pxor(v, (Packet2d)p2ul_CONJ_XOR); } EIGEN_STRONG_INLINE Packet1cd& operator*=(const Packet1cd& b) { Packet2d v1, v2; // Get the real values of a | a1_re | a1_re v1 = (Packet2d)__builtin_msa_ilvev_d((v2i64)v, (v2i64)v); // Get the imag values of a | a1_im | a1_im v2 = (Packet2d)__builtin_msa_ilvod_d((v2i64)v, (v2i64)v); // Multiply the real a with b v1 = pmul(v1, b.v); // Multiply the imag a with b v2 = pmul(v2, b.v); // Conjugate v2 v2 = Packet1cd(v2).conjugate().v; // Swap real/imag elements in v2. v2 = (Packet2d)__builtin_msa_shf_w((v4i32)v2, EIGEN_MSA_SHF_I8(2, 3, 0, 1)); // Add and return the result v = padd(v1, v2); return *this; } EIGEN_STRONG_INLINE Packet1cd operator*(const Packet1cd& b) const { return Packet1cd(*this) *= b; } EIGEN_STRONG_INLINE Packet1cd& operator+=(const Packet1cd& b) { v = padd(v, b.v); return *this; } EIGEN_STRONG_INLINE Packet1cd operator+(const Packet1cd& b) const { return Packet1cd(*this) += b; } EIGEN_STRONG_INLINE Packet1cd& operator-=(const Packet1cd& b) { v = psub(v, b.v); return *this; } EIGEN_STRONG_INLINE Packet1cd operator-(const Packet1cd& b) const { return Packet1cd(*this) -= b; } EIGEN_STRONG_INLINE Packet1cd& operator/=(const Packet1cd& b) { *this *= b.conjugate(); Packet2d s = pmul(b.v, b.v); s = padd(s, preverse(s)); v = pdiv(v, s); return *this; } EIGEN_STRONG_INLINE Packet1cd operator/(const Packet1cd& b) const { return Packet1cd(*this) /= b; } EIGEN_STRONG_INLINE Packet1cd operator-(void) const { return Packet1cd(pnegate(v)); } Packet2d v; }; inline std::ostream& operator<<(std::ostream& os, const Packet1cd& value) { os << "[ (" << value.v[0] << ", " << value.v[1] << "i) ]"; return os; } template <> struct packet_traits > : default_packet_traits { typedef Packet1cd type; typedef Packet1cd half; enum { Vectorizable = 1, AlignedOnScalar = 0, size = 1, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, HasMax = 0, HasSetLinear = 0 }; }; template <> struct unpacket_traits { typedef std::complex type; enum { size = 1, alignment = Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false }; typedef Packet1cd half; }; template <> EIGEN_STRONG_INLINE Packet1cd pload(const std::complex* from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_ALIGNED_LOAD return Packet1cd(pload((const double*)from)); } template <> EIGEN_STRONG_INLINE Packet1cd ploadu(const std::complex* from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_UNALIGNED_LOAD return Packet1cd(ploadu((const double*)from)); } template <> EIGEN_STRONG_INLINE Packet1cd pset1(const std::complex& from) { EIGEN_MSA_DEBUG; return Packet1cd(from); } template <> EIGEN_STRONG_INLINE Packet1cd padd(const Packet1cd& a, const Packet1cd& b) { EIGEN_MSA_DEBUG; return a + b; } template <> EIGEN_STRONG_INLINE Packet1cd psub(const Packet1cd& a, const Packet1cd& b) { EIGEN_MSA_DEBUG; return a - b; } template <> EIGEN_STRONG_INLINE Packet1cd pnegate(const Packet1cd& a) { EIGEN_MSA_DEBUG; return -a; } template <> EIGEN_STRONG_INLINE Packet1cd pconj(const Packet1cd& a) { EIGEN_MSA_DEBUG; return a.conjugate(); } template <> EIGEN_STRONG_INLINE Packet1cd pmul(const Packet1cd& a, const Packet1cd& b) { EIGEN_MSA_DEBUG; return a * b; } template <> EIGEN_STRONG_INLINE Packet1cd pand(const Packet1cd& a, const Packet1cd& b) { EIGEN_MSA_DEBUG; return Packet1cd(pand(a.v, b.v)); } template <> EIGEN_STRONG_INLINE Packet1cd por(const Packet1cd& a, const Packet1cd& b) { EIGEN_MSA_DEBUG; return Packet1cd(por(a.v, b.v)); } template <> EIGEN_STRONG_INLINE Packet1cd pxor(const Packet1cd& a, const Packet1cd& b) { EIGEN_MSA_DEBUG; return Packet1cd(pxor(a.v, b.v)); } template <> EIGEN_STRONG_INLINE Packet1cd pandnot(const Packet1cd& a, const Packet1cd& b) { EIGEN_MSA_DEBUG; return Packet1cd(pandnot(a.v, b.v)); } template <> EIGEN_STRONG_INLINE Packet1cd ploaddup(const std::complex* from) { EIGEN_MSA_DEBUG; return pset1(*from); } template <> EIGEN_STRONG_INLINE void pstore >(std::complex* to, const Packet1cd& from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v); } template <> EIGEN_STRONG_INLINE void pstoreu >(std::complex* to, const Packet1cd& from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, from.v); } template <> EIGEN_STRONG_INLINE void prefetch >(const std::complex* addr) { EIGEN_MSA_DEBUG; prefetch(reinterpret_cast(addr)); } template <> EIGEN_DEVICE_FUNC inline Packet1cd pgather, Packet1cd>( const std::complex* from, Index stride __attribute__((unused))) { EIGEN_MSA_DEBUG; Packet1cd res; res.v[0] = std::real(from[0]); res.v[1] = std::imag(from[0]); return res; } template <> EIGEN_DEVICE_FUNC inline void pscatter, Packet1cd>(std::complex* to, const Packet1cd& from, Index stride __attribute__((unused))) { EIGEN_MSA_DEBUG; pstore(to, from); } template <> EIGEN_STRONG_INLINE std::complex pfirst(const Packet1cd& a) { EIGEN_MSA_DEBUG; return std::complex(a.v[0], a.v[1]); } template <> EIGEN_STRONG_INLINE Packet1cd preverse(const Packet1cd& a) { EIGEN_MSA_DEBUG; return a; } template <> EIGEN_STRONG_INLINE std::complex predux(const Packet1cd& a) { EIGEN_MSA_DEBUG; return pfirst(a); } template <> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet1cd& a) { EIGEN_MSA_DEBUG; return pfirst(a); } EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd, Packet2d) template <> EIGEN_STRONG_INLINE Packet1cd pdiv(const Packet1cd& a, const Packet1cd& b) { EIGEN_MSA_DEBUG; return a / b; } EIGEN_STRONG_INLINE Packet1cd pcplxflip /**/ (const Packet1cd& x) { EIGEN_MSA_DEBUG; return Packet1cd(preverse(Packet2d(x.v))); } inline std::ostream& operator<<(std::ostream& os, const PacketBlock& value) { os << "[ " << value.packet[0] << ", " << std::endl << " " << value.packet[1] << " ]"; return os; } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { EIGEN_MSA_DEBUG; Packet2d v1, v2; v1 = (Packet2d)__builtin_msa_ilvev_d((v2i64)kernel.packet[0].v, (v2i64)kernel.packet[1].v); // Get the imag values of a v2 = (Packet2d)__builtin_msa_ilvod_d((v2i64)kernel.packet[0].v, (v2i64)kernel.packet[1].v); kernel.packet[0].v = v1; kernel.packet[1].v = v2; } } // end namespace internal } // end namespace Eigen #endif // EIGEN_COMPLEX_MSA_H RcppEigen/inst/include/Eigen/src/Core/arch/MSA/PacketMath.h0000644000176200001440000010151714567757725023021 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2018 Wave Computing, Inc. // Written by: // Chris Larsen // Alexey Frunze (afrunze@wavecomp.com) // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_PACKET_MATH_MSA_H #define EIGEN_PACKET_MATH_MSA_H #include #include namespace Eigen { namespace internal { #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 #endif #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD #endif #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32 #endif #if 0 #define EIGEN_MSA_DEBUG \ static bool firstTime = true; \ do { \ if (firstTime) { \ std::cout << __FILE__ << ':' << __LINE__ << ':' << __FUNCTION__ << std::endl; \ firstTime = false; \ } \ } while (0) #else #define EIGEN_MSA_DEBUG #endif #define EIGEN_MSA_SHF_I8(a, b, c, d) (((d) << 6) | ((c) << 4) | ((b) << 2) | (a)) typedef v4f32 Packet4f; typedef v4i32 Packet4i; typedef v4u32 Packet4ui; #define _EIGEN_DECLARE_CONST_Packet4f(NAME, X) const Packet4f p4f_##NAME = { X, X, X, X } #define _EIGEN_DECLARE_CONST_Packet4i(NAME, X) const Packet4i p4i_##NAME = { X, X, X, X } #define _EIGEN_DECLARE_CONST_Packet4ui(NAME, X) const Packet4ui p4ui_##NAME = { X, X, X, X } inline std::ostream& operator<<(std::ostream& os, const Packet4f& value) { os << "[ " << value[0] << ", " << value[1] << ", " << value[2] << ", " << value[3] << " ]"; return os; } inline std::ostream& operator<<(std::ostream& os, const Packet4i& value) { os << "[ " << value[0] << ", " << value[1] << ", " << value[2] << ", " << value[3] << " ]"; return os; } inline std::ostream& operator<<(std::ostream& os, const Packet4ui& value) { os << "[ " << value[0] << ", " << value[1] << ", " << value[2] << ", " << value[3] << " ]"; return os; } template <> struct packet_traits : default_packet_traits { typedef Packet4f type; typedef Packet4f half; // Packet2f intrinsics not implemented yet enum { Vectorizable = 1, AlignedOnScalar = 1, size = 4, HasHalfPacket = 0, // Packet2f intrinsics not implemented yet // FIXME check the Has* HasDiv = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, HasLog = 1, HasExp = 1, HasSqrt = 1, HasRsqrt = 1, HasRound = 1, HasFloor = 1, HasCeil = 1, HasBlend = 1 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet4i type; typedef Packet4i half; // Packet2i intrinsics not implemented yet enum { Vectorizable = 1, AlignedOnScalar = 1, size = 4, HasHalfPacket = 0, // Packet2i intrinsics not implemented yet // FIXME check the Has* HasDiv = 1, HasBlend = 1 }; }; template <> struct unpacket_traits { typedef float type; enum { size = 4, alignment = Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false }; typedef Packet4f half; }; template <> struct unpacket_traits { typedef int32_t type; enum { size = 4, alignment = Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false }; typedef Packet4i half; }; template <> EIGEN_STRONG_INLINE Packet4f pset1(const float& from) { EIGEN_MSA_DEBUG; Packet4f v = { from, from, from, from }; return v; } template <> EIGEN_STRONG_INLINE Packet4i pset1(const int32_t& from) { EIGEN_MSA_DEBUG; return __builtin_msa_fill_w(from); } template <> EIGEN_STRONG_INLINE Packet4f pload1(const float* from) { EIGEN_MSA_DEBUG; float f = *from; Packet4f v = { f, f, f, f }; return v; } template <> EIGEN_STRONG_INLINE Packet4i pload1(const int32_t* from) { EIGEN_MSA_DEBUG; return __builtin_msa_fill_w(*from); } template <> EIGEN_STRONG_INLINE Packet4f padd(const Packet4f& a, const Packet4f& b) { EIGEN_MSA_DEBUG; return __builtin_msa_fadd_w(a, b); } template <> EIGEN_STRONG_INLINE Packet4i padd(const Packet4i& a, const Packet4i& b) { EIGEN_MSA_DEBUG; return __builtin_msa_addv_w(a, b); } template <> EIGEN_STRONG_INLINE Packet4f plset(const float& a) { EIGEN_MSA_DEBUG; static const Packet4f countdown = { 0.0f, 1.0f, 2.0f, 3.0f }; return padd(pset1(a), countdown); } template <> EIGEN_STRONG_INLINE Packet4i plset(const int32_t& a) { EIGEN_MSA_DEBUG; static const Packet4i countdown = { 0, 1, 2, 3 }; return padd(pset1(a), countdown); } template <> EIGEN_STRONG_INLINE Packet4f psub(const Packet4f& a, const Packet4f& b) { EIGEN_MSA_DEBUG; return __builtin_msa_fsub_w(a, b); } template <> EIGEN_STRONG_INLINE Packet4i psub(const Packet4i& a, const Packet4i& b) { EIGEN_MSA_DEBUG; return __builtin_msa_subv_w(a, b); } template <> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { EIGEN_MSA_DEBUG; return (Packet4f)__builtin_msa_bnegi_w((v4u32)a, 31); } template <> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { EIGEN_MSA_DEBUG; return __builtin_msa_addvi_w((v4i32)__builtin_msa_nori_b((v16u8)a, 0), 1); } template <> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { EIGEN_MSA_DEBUG; return a; } template <> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { EIGEN_MSA_DEBUG; return a; } template <> EIGEN_STRONG_INLINE Packet4f pmul(const Packet4f& a, const Packet4f& b) { EIGEN_MSA_DEBUG; return __builtin_msa_fmul_w(a, b); } template <> EIGEN_STRONG_INLINE Packet4i pmul(const Packet4i& a, const Packet4i& b) { EIGEN_MSA_DEBUG; return __builtin_msa_mulv_w(a, b); } template <> EIGEN_STRONG_INLINE Packet4f pdiv(const Packet4f& a, const Packet4f& b) { EIGEN_MSA_DEBUG; return __builtin_msa_fdiv_w(a, b); } template <> EIGEN_STRONG_INLINE Packet4i pdiv(const Packet4i& a, const Packet4i& b) { EIGEN_MSA_DEBUG; return __builtin_msa_div_s_w(a, b); } template <> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { EIGEN_MSA_DEBUG; return __builtin_msa_fmadd_w(c, a, b); } template <> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { EIGEN_MSA_DEBUG; // Use "asm" construct to avoid __builtin_msa_maddv_w GNU C bug. Packet4i value = c; __asm__("maddv.w %w[value], %w[a], %w[b]\n" // Outputs : [value] "+f"(value) // Inputs : [a] "f"(a), [b] "f"(b)); return value; } template <> EIGEN_STRONG_INLINE Packet4f pand(const Packet4f& a, const Packet4f& b) { EIGEN_MSA_DEBUG; return (Packet4f)__builtin_msa_and_v((v16u8)a, (v16u8)b); } template <> EIGEN_STRONG_INLINE Packet4i pand(const Packet4i& a, const Packet4i& b) { EIGEN_MSA_DEBUG; return (Packet4i)__builtin_msa_and_v((v16u8)a, (v16u8)b); } template <> EIGEN_STRONG_INLINE Packet4f por(const Packet4f& a, const Packet4f& b) { EIGEN_MSA_DEBUG; return (Packet4f)__builtin_msa_or_v((v16u8)a, (v16u8)b); } template <> EIGEN_STRONG_INLINE Packet4i por(const Packet4i& a, const Packet4i& b) { EIGEN_MSA_DEBUG; return (Packet4i)__builtin_msa_or_v((v16u8)a, (v16u8)b); } template <> EIGEN_STRONG_INLINE Packet4f pxor(const Packet4f& a, const Packet4f& b) { EIGEN_MSA_DEBUG; return (Packet4f)__builtin_msa_xor_v((v16u8)a, (v16u8)b); } template <> EIGEN_STRONG_INLINE Packet4i pxor(const Packet4i& a, const Packet4i& b) { EIGEN_MSA_DEBUG; return (Packet4i)__builtin_msa_xor_v((v16u8)a, (v16u8)b); } template <> EIGEN_STRONG_INLINE Packet4f pandnot(const Packet4f& a, const Packet4f& b) { EIGEN_MSA_DEBUG; return pand(a, (Packet4f)__builtin_msa_xori_b((v16u8)b, 255)); } template <> EIGEN_STRONG_INLINE Packet4i pandnot(const Packet4i& a, const Packet4i& b) { EIGEN_MSA_DEBUG; return pand(a, (Packet4i)__builtin_msa_xori_b((v16u8)b, 255)); } template <> EIGEN_STRONG_INLINE Packet4f pmin(const Packet4f& a, const Packet4f& b) { EIGEN_MSA_DEBUG; #if EIGEN_FAST_MATH // This prefers numbers to NaNs. return __builtin_msa_fmin_w(a, b); #else // This prefers NaNs to numbers. Packet4i aNaN = __builtin_msa_fcun_w(a, a); Packet4i aMinOrNaN = por(__builtin_msa_fclt_w(a, b), aNaN); return (Packet4f)__builtin_msa_bsel_v((v16u8)aMinOrNaN, (v16u8)b, (v16u8)a); #endif } template <> EIGEN_STRONG_INLINE Packet4i pmin(const Packet4i& a, const Packet4i& b) { EIGEN_MSA_DEBUG; return __builtin_msa_min_s_w(a, b); } template <> EIGEN_STRONG_INLINE Packet4f pmax(const Packet4f& a, const Packet4f& b) { EIGEN_MSA_DEBUG; #if EIGEN_FAST_MATH // This prefers numbers to NaNs. return __builtin_msa_fmax_w(a, b); #else // This prefers NaNs to numbers. Packet4i aNaN = __builtin_msa_fcun_w(a, a); Packet4i aMaxOrNaN = por(__builtin_msa_fclt_w(b, a), aNaN); return (Packet4f)__builtin_msa_bsel_v((v16u8)aMaxOrNaN, (v16u8)b, (v16u8)a); #endif } template <> EIGEN_STRONG_INLINE Packet4i pmax(const Packet4i& a, const Packet4i& b) { EIGEN_MSA_DEBUG; return __builtin_msa_max_s_w(a, b); } template <> EIGEN_STRONG_INLINE Packet4f pload(const float* from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_ALIGNED_LOAD return (Packet4f)__builtin_msa_ld_w(const_cast(from), 0); } template <> EIGEN_STRONG_INLINE Packet4i pload(const int32_t* from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_ALIGNED_LOAD return __builtin_msa_ld_w(const_cast(from), 0); } template <> EIGEN_STRONG_INLINE Packet4f ploadu(const float* from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_UNALIGNED_LOAD return (Packet4f)__builtin_msa_ld_w(const_cast(from), 0); } template <> EIGEN_STRONG_INLINE Packet4i ploadu(const int32_t* from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_UNALIGNED_LOAD return (Packet4i)__builtin_msa_ld_w(const_cast(from), 0); } template <> EIGEN_STRONG_INLINE Packet4f ploaddup(const float* from) { EIGEN_MSA_DEBUG; float f0 = from[0], f1 = from[1]; Packet4f v0 = { f0, f0, f0, f0 }; Packet4f v1 = { f1, f1, f1, f1 }; return (Packet4f)__builtin_msa_ilvr_d((v2i64)v1, (v2i64)v0); } template <> EIGEN_STRONG_INLINE Packet4i ploaddup(const int32_t* from) { EIGEN_MSA_DEBUG; int32_t i0 = from[0], i1 = from[1]; Packet4i v0 = { i0, i0, i0, i0 }; Packet4i v1 = { i1, i1, i1, i1 }; return (Packet4i)__builtin_msa_ilvr_d((v2i64)v1, (v2i64)v0); } template <> EIGEN_STRONG_INLINE void pstore(float* to, const Packet4f& from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_ALIGNED_STORE __builtin_msa_st_w((Packet4i)from, to, 0); } template <> EIGEN_STRONG_INLINE void pstore(int32_t* to, const Packet4i& from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_ALIGNED_STORE __builtin_msa_st_w(from, to, 0); } template <> EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet4f& from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_UNALIGNED_STORE __builtin_msa_st_w((Packet4i)from, to, 0); } template <> EIGEN_STRONG_INLINE void pstoreu(int32_t* to, const Packet4i& from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_UNALIGNED_STORE __builtin_msa_st_w(from, to, 0); } template <> EIGEN_DEVICE_FUNC inline Packet4f pgather(const float* from, Index stride) { EIGEN_MSA_DEBUG; float f = *from; Packet4f v = { f, f, f, f }; v[1] = from[stride]; v[2] = from[2 * stride]; v[3] = from[3 * stride]; return v; } template <> EIGEN_DEVICE_FUNC inline Packet4i pgather(const int32_t* from, Index stride) { EIGEN_MSA_DEBUG; int32_t i = *from; Packet4i v = { i, i, i, i }; v[1] = from[stride]; v[2] = from[2 * stride]; v[3] = from[3 * stride]; return v; } template <> EIGEN_DEVICE_FUNC inline void pscatter(float* to, const Packet4f& from, Index stride) { EIGEN_MSA_DEBUG; *to = from[0]; to += stride; *to = from[1]; to += stride; *to = from[2]; to += stride; *to = from[3]; } template <> EIGEN_DEVICE_FUNC inline void pscatter(int32_t* to, const Packet4i& from, Index stride) { EIGEN_MSA_DEBUG; *to = from[0]; to += stride; *to = from[1]; to += stride; *to = from[2]; to += stride; *to = from[3]; } template <> EIGEN_STRONG_INLINE void prefetch(const float* addr) { EIGEN_MSA_DEBUG; __builtin_prefetch(addr); } template <> EIGEN_STRONG_INLINE void prefetch(const int32_t* addr) { EIGEN_MSA_DEBUG; __builtin_prefetch(addr); } template <> EIGEN_STRONG_INLINE float pfirst(const Packet4f& a) { EIGEN_MSA_DEBUG; return a[0]; } template <> EIGEN_STRONG_INLINE int32_t pfirst(const Packet4i& a) { EIGEN_MSA_DEBUG; return a[0]; } template <> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) { EIGEN_MSA_DEBUG; return (Packet4f)__builtin_msa_shf_w((v4i32)a, EIGEN_MSA_SHF_I8(3, 2, 1, 0)); } template <> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) { EIGEN_MSA_DEBUG; return __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(3, 2, 1, 0)); } template <> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { EIGEN_MSA_DEBUG; return (Packet4f)__builtin_msa_bclri_w((v4u32)a, 31); } template <> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { EIGEN_MSA_DEBUG; Packet4i zero = __builtin_msa_ldi_w(0); return __builtin_msa_add_a_w(zero, a); } template <> EIGEN_STRONG_INLINE float predux(const Packet4f& a) { EIGEN_MSA_DEBUG; Packet4f s = padd(a, (Packet4f)__builtin_msa_shf_w((v4i32)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1))); s = padd(s, (Packet4f)__builtin_msa_shf_w((v4i32)s, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); return s[0]; } template <> EIGEN_STRONG_INLINE int32_t predux(const Packet4i& a) { EIGEN_MSA_DEBUG; Packet4i s = padd(a, __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(2, 3, 0, 1))); s = padd(s, __builtin_msa_shf_w(s, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); return s[0]; } // Other reduction functions: // mul template <> EIGEN_STRONG_INLINE float predux_mul(const Packet4f& a) { EIGEN_MSA_DEBUG; Packet4f p = pmul(a, (Packet4f)__builtin_msa_shf_w((v4i32)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1))); p = pmul(p, (Packet4f)__builtin_msa_shf_w((v4i32)p, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); return p[0]; } template <> EIGEN_STRONG_INLINE int32_t predux_mul(const Packet4i& a) { EIGEN_MSA_DEBUG; Packet4i p = pmul(a, __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(2, 3, 0, 1))); p = pmul(p, __builtin_msa_shf_w(p, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); return p[0]; } // min template <> EIGEN_STRONG_INLINE float predux_min(const Packet4f& a) { EIGEN_MSA_DEBUG; // Swap 64-bit halves of a. Packet4f swapped = (Packet4f)__builtin_msa_shf_w((Packet4i)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)); #if !EIGEN_FAST_MATH // Detect presence of NaNs from pairs a[0]-a[2] and a[1]-a[3] as two 32-bit // masks of all zeroes/ones in low 64 bits. v16u8 unord = (v16u8)__builtin_msa_fcun_w(a, swapped); // Combine the two masks into one: 64 ones if no NaNs, otherwise 64 zeroes. unord = (v16u8)__builtin_msa_ceqi_d((v2i64)unord, 0); #endif // Continue with min computation. Packet4f v = __builtin_msa_fmin_w(a, swapped); v = __builtin_msa_fmin_w( v, (Packet4f)__builtin_msa_shf_w((Packet4i)v, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); #if !EIGEN_FAST_MATH // Based on the mask select between v and 4 qNaNs. v16u8 qnans = (v16u8)__builtin_msa_fill_w(0x7FC00000); v = (Packet4f)__builtin_msa_bsel_v(unord, qnans, (v16u8)v); #endif return v[0]; } template <> EIGEN_STRONG_INLINE int32_t predux_min(const Packet4i& a) { EIGEN_MSA_DEBUG; Packet4i m = pmin(a, __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(2, 3, 0, 1))); m = pmin(m, __builtin_msa_shf_w(m, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); return m[0]; } // max template <> EIGEN_STRONG_INLINE float predux_max(const Packet4f& a) { EIGEN_MSA_DEBUG; // Swap 64-bit halves of a. Packet4f swapped = (Packet4f)__builtin_msa_shf_w((Packet4i)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)); #if !EIGEN_FAST_MATH // Detect presence of NaNs from pairs a[0]-a[2] and a[1]-a[3] as two 32-bit // masks of all zeroes/ones in low 64 bits. v16u8 unord = (v16u8)__builtin_msa_fcun_w(a, swapped); // Combine the two masks into one: 64 ones if no NaNs, otherwise 64 zeroes. unord = (v16u8)__builtin_msa_ceqi_d((v2i64)unord, 0); #endif // Continue with max computation. Packet4f v = __builtin_msa_fmax_w(a, swapped); v = __builtin_msa_fmax_w( v, (Packet4f)__builtin_msa_shf_w((Packet4i)v, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); #if !EIGEN_FAST_MATH // Based on the mask select between v and 4 qNaNs. v16u8 qnans = (v16u8)__builtin_msa_fill_w(0x7FC00000); v = (Packet4f)__builtin_msa_bsel_v(unord, qnans, (v16u8)v); #endif return v[0]; } template <> EIGEN_STRONG_INLINE int32_t predux_max(const Packet4i& a) { EIGEN_MSA_DEBUG; Packet4i m = pmax(a, __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(2, 3, 0, 1))); m = pmax(m, __builtin_msa_shf_w(m, EIGEN_MSA_SHF_I8(1, 0, 3, 2))); return m[0]; } inline std::ostream& operator<<(std::ostream& os, const PacketBlock& value) { os << "[ " << value.packet[0] << "," << std::endl << " " << value.packet[1] << "," << std::endl << " " << value.packet[2] << "," << std::endl << " " << value.packet[3] << " ]"; return os; } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { EIGEN_MSA_DEBUG; v4i32 tmp1, tmp2, tmp3, tmp4; tmp1 = __builtin_msa_ilvr_w((v4i32)kernel.packet[1], (v4i32)kernel.packet[0]); tmp2 = __builtin_msa_ilvr_w((v4i32)kernel.packet[3], (v4i32)kernel.packet[2]); tmp3 = __builtin_msa_ilvl_w((v4i32)kernel.packet[1], (v4i32)kernel.packet[0]); tmp4 = __builtin_msa_ilvl_w((v4i32)kernel.packet[3], (v4i32)kernel.packet[2]); kernel.packet[0] = (Packet4f)__builtin_msa_ilvr_d((v2i64)tmp2, (v2i64)tmp1); kernel.packet[1] = (Packet4f)__builtin_msa_ilvod_d((v2i64)tmp2, (v2i64)tmp1); kernel.packet[2] = (Packet4f)__builtin_msa_ilvr_d((v2i64)tmp4, (v2i64)tmp3); kernel.packet[3] = (Packet4f)__builtin_msa_ilvod_d((v2i64)tmp4, (v2i64)tmp3); } inline std::ostream& operator<<(std::ostream& os, const PacketBlock& value) { os << "[ " << value.packet[0] << "," << std::endl << " " << value.packet[1] << "," << std::endl << " " << value.packet[2] << "," << std::endl << " " << value.packet[3] << " ]"; return os; } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { EIGEN_MSA_DEBUG; v4i32 tmp1, tmp2, tmp3, tmp4; tmp1 = __builtin_msa_ilvr_w(kernel.packet[1], kernel.packet[0]); tmp2 = __builtin_msa_ilvr_w(kernel.packet[3], kernel.packet[2]); tmp3 = __builtin_msa_ilvl_w(kernel.packet[1], kernel.packet[0]); tmp4 = __builtin_msa_ilvl_w(kernel.packet[3], kernel.packet[2]); kernel.packet[0] = (Packet4i)__builtin_msa_ilvr_d((v2i64)tmp2, (v2i64)tmp1); kernel.packet[1] = (Packet4i)__builtin_msa_ilvod_d((v2i64)tmp2, (v2i64)tmp1); kernel.packet[2] = (Packet4i)__builtin_msa_ilvr_d((v2i64)tmp4, (v2i64)tmp3); kernel.packet[3] = (Packet4i)__builtin_msa_ilvod_d((v2i64)tmp4, (v2i64)tmp3); } template <> EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& a) { EIGEN_MSA_DEBUG; return __builtin_msa_fsqrt_w(a); } template <> EIGEN_STRONG_INLINE Packet4f prsqrt(const Packet4f& a) { EIGEN_MSA_DEBUG; #if EIGEN_FAST_MATH return __builtin_msa_frsqrt_w(a); #else Packet4f ones = __builtin_msa_ffint_s_w(__builtin_msa_ldi_w(1)); return pdiv(ones, psqrt(a)); #endif } template <> EIGEN_STRONG_INLINE Packet4f pfloor(const Packet4f& a) { Packet4f v = a; int32_t old_mode, new_mode; asm volatile( "cfcmsa %[old_mode], $1\n" "ori %[new_mode], %[old_mode], 3\n" // 3 = round towards -INFINITY. "ctcmsa $1, %[new_mode]\n" "frint.w %w[v], %w[v]\n" "ctcmsa $1, %[old_mode]\n" : // outputs [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode), [v] "+f"(v) : // inputs : // clobbers ); return v; } template <> EIGEN_STRONG_INLINE Packet4f pceil(const Packet4f& a) { Packet4f v = a; int32_t old_mode, new_mode; asm volatile( "cfcmsa %[old_mode], $1\n" "ori %[new_mode], %[old_mode], 3\n" "xori %[new_mode], %[new_mode], 1\n" // 2 = round towards +INFINITY. "ctcmsa $1, %[new_mode]\n" "frint.w %w[v], %w[v]\n" "ctcmsa $1, %[old_mode]\n" : // outputs [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode), [v] "+f"(v) : // inputs : // clobbers ); return v; } template <> EIGEN_STRONG_INLINE Packet4f pround(const Packet4f& a) { Packet4f v = a; int32_t old_mode, new_mode; asm volatile( "cfcmsa %[old_mode], $1\n" "ori %[new_mode], %[old_mode], 3\n" "xori %[new_mode], %[new_mode], 3\n" // 0 = round to nearest, ties to even. "ctcmsa $1, %[new_mode]\n" "frint.w %w[v], %w[v]\n" "ctcmsa $1, %[old_mode]\n" : // outputs [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode), [v] "+f"(v) : // inputs : // clobbers ); return v; } template <> EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket, const Packet4f& elsePacket) { Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] }; Packet4i mask = __builtin_msa_ceqi_w((Packet4i)select, 0); return (Packet4f)__builtin_msa_bsel_v((v16u8)mask, (v16u8)thenPacket, (v16u8)elsePacket); } template <> EIGEN_STRONG_INLINE Packet4i pblend(const Selector<4>& ifPacket, const Packet4i& thenPacket, const Packet4i& elsePacket) { Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] }; Packet4i mask = __builtin_msa_ceqi_w((Packet4i)select, 0); return (Packet4i)__builtin_msa_bsel_v((v16u8)mask, (v16u8)thenPacket, (v16u8)elsePacket); } //---------- double ---------- typedef v2f64 Packet2d; typedef v2i64 Packet2l; typedef v2u64 Packet2ul; #define _EIGEN_DECLARE_CONST_Packet2d(NAME, X) const Packet2d p2d_##NAME = { X, X } #define _EIGEN_DECLARE_CONST_Packet2l(NAME, X) const Packet2l p2l_##NAME = { X, X } #define _EIGEN_DECLARE_CONST_Packet2ul(NAME, X) const Packet2ul p2ul_##NAME = { X, X } inline std::ostream& operator<<(std::ostream& os, const Packet2d& value) { os << "[ " << value[0] << ", " << value[1] << " ]"; return os; } inline std::ostream& operator<<(std::ostream& os, const Packet2l& value) { os << "[ " << value[0] << ", " << value[1] << " ]"; return os; } inline std::ostream& operator<<(std::ostream& os, const Packet2ul& value) { os << "[ " << value[0] << ", " << value[1] << " ]"; return os; } template <> struct packet_traits : default_packet_traits { typedef Packet2d type; typedef Packet2d half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 2, HasHalfPacket = 0, // FIXME check the Has* HasDiv = 1, HasExp = 1, HasSqrt = 1, HasRsqrt = 1, HasRound = 1, HasFloor = 1, HasCeil = 1, HasBlend = 1 }; }; template <> struct unpacket_traits { typedef double type; enum { size = 2, alignment = Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false }; typedef Packet2d half; }; template <> EIGEN_STRONG_INLINE Packet2d pset1(const double& from) { EIGEN_MSA_DEBUG; Packet2d value = { from, from }; return value; } template <> EIGEN_STRONG_INLINE Packet2d padd(const Packet2d& a, const Packet2d& b) { EIGEN_MSA_DEBUG; return __builtin_msa_fadd_d(a, b); } template <> EIGEN_STRONG_INLINE Packet2d plset(const double& a) { EIGEN_MSA_DEBUG; static const Packet2d countdown = { 0.0, 1.0 }; return padd(pset1(a), countdown); } template <> EIGEN_STRONG_INLINE Packet2d psub(const Packet2d& a, const Packet2d& b) { EIGEN_MSA_DEBUG; return __builtin_msa_fsub_d(a, b); } template <> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) { EIGEN_MSA_DEBUG; return (Packet2d)__builtin_msa_bnegi_d((v2u64)a, 63); } template <> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { EIGEN_MSA_DEBUG; return a; } template <> EIGEN_STRONG_INLINE Packet2d pmul(const Packet2d& a, const Packet2d& b) { EIGEN_MSA_DEBUG; return __builtin_msa_fmul_d(a, b); } template <> EIGEN_STRONG_INLINE Packet2d pdiv(const Packet2d& a, const Packet2d& b) { EIGEN_MSA_DEBUG; return __builtin_msa_fdiv_d(a, b); } template <> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { EIGEN_MSA_DEBUG; return __builtin_msa_fmadd_d(c, a, b); } // Logical Operations are not supported for float, so we have to reinterpret casts using MSA // intrinsics template <> EIGEN_STRONG_INLINE Packet2d pand(const Packet2d& a, const Packet2d& b) { EIGEN_MSA_DEBUG; return (Packet2d)__builtin_msa_and_v((v16u8)a, (v16u8)b); } template <> EIGEN_STRONG_INLINE Packet2d por(const Packet2d& a, const Packet2d& b) { EIGEN_MSA_DEBUG; return (Packet2d)__builtin_msa_or_v((v16u8)a, (v16u8)b); } template <> EIGEN_STRONG_INLINE Packet2d pxor(const Packet2d& a, const Packet2d& b) { EIGEN_MSA_DEBUG; return (Packet2d)__builtin_msa_xor_v((v16u8)a, (v16u8)b); } template <> EIGEN_STRONG_INLINE Packet2d pandnot(const Packet2d& a, const Packet2d& b) { EIGEN_MSA_DEBUG; return pand(a, (Packet2d)__builtin_msa_xori_b((v16u8)b, 255)); } template <> EIGEN_STRONG_INLINE Packet2d pload(const double* from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_UNALIGNED_LOAD return (Packet2d)__builtin_msa_ld_d(const_cast(from), 0); } template <> EIGEN_STRONG_INLINE Packet2d pmin(const Packet2d& a, const Packet2d& b) { EIGEN_MSA_DEBUG; #if EIGEN_FAST_MATH // This prefers numbers to NaNs. return __builtin_msa_fmin_d(a, b); #else // This prefers NaNs to numbers. v2i64 aNaN = __builtin_msa_fcun_d(a, a); v2i64 aMinOrNaN = por(__builtin_msa_fclt_d(a, b), aNaN); return (Packet2d)__builtin_msa_bsel_v((v16u8)aMinOrNaN, (v16u8)b, (v16u8)a); #endif } template <> EIGEN_STRONG_INLINE Packet2d pmax(const Packet2d& a, const Packet2d& b) { EIGEN_MSA_DEBUG; #if EIGEN_FAST_MATH // This prefers numbers to NaNs. return __builtin_msa_fmax_d(a, b); #else // This prefers NaNs to numbers. v2i64 aNaN = __builtin_msa_fcun_d(a, a); v2i64 aMaxOrNaN = por(__builtin_msa_fclt_d(b, a), aNaN); return (Packet2d)__builtin_msa_bsel_v((v16u8)aMaxOrNaN, (v16u8)b, (v16u8)a); #endif } template <> EIGEN_STRONG_INLINE Packet2d ploadu(const double* from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_UNALIGNED_LOAD return (Packet2d)__builtin_msa_ld_d(const_cast(from), 0); } template <> EIGEN_STRONG_INLINE Packet2d ploaddup(const double* from) { EIGEN_MSA_DEBUG; Packet2d value = { *from, *from }; return value; } template <> EIGEN_STRONG_INLINE void pstore(double* to, const Packet2d& from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_ALIGNED_STORE __builtin_msa_st_d((v2i64)from, to, 0); } template <> EIGEN_STRONG_INLINE void pstoreu(double* to, const Packet2d& from) { EIGEN_MSA_DEBUG; EIGEN_DEBUG_UNALIGNED_STORE __builtin_msa_st_d((v2i64)from, to, 0); } template <> EIGEN_DEVICE_FUNC inline Packet2d pgather(const double* from, Index stride) { EIGEN_MSA_DEBUG; Packet2d value; value[0] = *from; from += stride; value[1] = *from; return value; } template <> EIGEN_DEVICE_FUNC inline void pscatter(double* to, const Packet2d& from, Index stride) { EIGEN_MSA_DEBUG; *to = from[0]; to += stride; *to = from[1]; } template <> EIGEN_STRONG_INLINE void prefetch(const double* addr) { EIGEN_MSA_DEBUG; __builtin_prefetch(addr); } template <> EIGEN_STRONG_INLINE double pfirst(const Packet2d& a) { EIGEN_MSA_DEBUG; return a[0]; } template <> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) { EIGEN_MSA_DEBUG; return (Packet2d)__builtin_msa_shf_w((v4i32)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)); } template <> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { EIGEN_MSA_DEBUG; return (Packet2d)__builtin_msa_bclri_d((v2u64)a, 63); } template <> EIGEN_STRONG_INLINE double predux(const Packet2d& a) { EIGEN_MSA_DEBUG; Packet2d s = padd(a, preverse(a)); return s[0]; } // Other reduction functions: // mul template <> EIGEN_STRONG_INLINE double predux_mul(const Packet2d& a) { EIGEN_MSA_DEBUG; Packet2d p = pmul(a, preverse(a)); return p[0]; } // min template <> EIGEN_STRONG_INLINE double predux_min(const Packet2d& a) { EIGEN_MSA_DEBUG; #if EIGEN_FAST_MATH Packet2d swapped = (Packet2d)__builtin_msa_shf_w((Packet4i)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)); Packet2d v = __builtin_msa_fmin_d(a, swapped); return v[0]; #else double a0 = a[0], a1 = a[1]; return ((numext::isnan)(a0) || a0 < a1) ? a0 : a1; #endif } // max template <> EIGEN_STRONG_INLINE double predux_max(const Packet2d& a) { EIGEN_MSA_DEBUG; #if EIGEN_FAST_MATH Packet2d swapped = (Packet2d)__builtin_msa_shf_w((Packet4i)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)); Packet2d v = __builtin_msa_fmax_d(a, swapped); return v[0]; #else double a0 = a[0], a1 = a[1]; return ((numext::isnan)(a0) || a0 > a1) ? a0 : a1; #endif } template <> EIGEN_STRONG_INLINE Packet2d psqrt(const Packet2d& a) { EIGEN_MSA_DEBUG; return __builtin_msa_fsqrt_d(a); } template <> EIGEN_STRONG_INLINE Packet2d prsqrt(const Packet2d& a) { EIGEN_MSA_DEBUG; #if EIGEN_FAST_MATH return __builtin_msa_frsqrt_d(a); #else Packet2d ones = __builtin_msa_ffint_s_d(__builtin_msa_ldi_d(1)); return pdiv(ones, psqrt(a)); #endif } inline std::ostream& operator<<(std::ostream& os, const PacketBlock& value) { os << "[ " << value.packet[0] << "," << std::endl << " " << value.packet[1] << " ]"; return os; } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { EIGEN_MSA_DEBUG; Packet2d trn1 = (Packet2d)__builtin_msa_ilvev_d((v2i64)kernel.packet[1], (v2i64)kernel.packet[0]); Packet2d trn2 = (Packet2d)__builtin_msa_ilvod_d((v2i64)kernel.packet[1], (v2i64)kernel.packet[0]); kernel.packet[0] = trn1; kernel.packet[1] = trn2; } template <> EIGEN_STRONG_INLINE Packet2d pfloor(const Packet2d& a) { Packet2d v = a; int32_t old_mode, new_mode; asm volatile( "cfcmsa %[old_mode], $1\n" "ori %[new_mode], %[old_mode], 3\n" // 3 = round towards -INFINITY. "ctcmsa $1, %[new_mode]\n" "frint.d %w[v], %w[v]\n" "ctcmsa $1, %[old_mode]\n" : // outputs [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode), [v] "+f"(v) : // inputs : // clobbers ); return v; } template <> EIGEN_STRONG_INLINE Packet2d pceil(const Packet2d& a) { Packet2d v = a; int32_t old_mode, new_mode; asm volatile( "cfcmsa %[old_mode], $1\n" "ori %[new_mode], %[old_mode], 3\n" "xori %[new_mode], %[new_mode], 1\n" // 2 = round towards +INFINITY. "ctcmsa $1, %[new_mode]\n" "frint.d %w[v], %w[v]\n" "ctcmsa $1, %[old_mode]\n" : // outputs [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode), [v] "+f"(v) : // inputs : // clobbers ); return v; } template <> EIGEN_STRONG_INLINE Packet2d pround(const Packet2d& a) { Packet2d v = a; int32_t old_mode, new_mode; asm volatile( "cfcmsa %[old_mode], $1\n" "ori %[new_mode], %[old_mode], 3\n" "xori %[new_mode], %[new_mode], 3\n" // 0 = round to nearest, ties to even. "ctcmsa $1, %[new_mode]\n" "frint.d %w[v], %w[v]\n" "ctcmsa $1, %[old_mode]\n" : // outputs [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode), [v] "+f"(v) : // inputs : // clobbers ); return v; } template <> EIGEN_STRONG_INLINE Packet2d pblend(const Selector<2>& ifPacket, const Packet2d& thenPacket, const Packet2d& elsePacket) { Packet2ul select = { ifPacket.select[0], ifPacket.select[1] }; Packet2l mask = __builtin_msa_ceqi_d((Packet2l)select, 0); return (Packet2d)__builtin_msa_bsel_v((v16u8)mask, (v16u8)thenPacket, (v16u8)elsePacket); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_PACKET_MATH_MSA_H RcppEigen/inst/include/Eigen/src/Core/arch/MSA/MathFunctions.h0000644000176200001440000003743714567757725023573 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2007 Julien Pommier // Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com) // Copyright (C) 2016 Gael Guennebaud // // Copyright (C) 2018 Wave Computing, Inc. // Written by: // Chris Larsen // Alexey Frunze (afrunze@wavecomp.com) // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. /* The sin, cos, exp, and log functions of this file come from * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/ */ /* The tanh function of this file is an adaptation of * template T generic_fast_tanh_float(const T&) * from MathFunctionsImpl.h. */ #ifndef EIGEN_MATH_FUNCTIONS_MSA_H #define EIGEN_MATH_FUNCTIONS_MSA_H namespace Eigen { namespace internal { template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f plog(const Packet4f& _x) { static _EIGEN_DECLARE_CONST_Packet4f(cephes_SQRTHF, 0.707106781186547524f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p0, 7.0376836292e-2f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p1, -1.1514610310e-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p2, 1.1676998740e-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p3, -1.2420140846e-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p4, +1.4249322787e-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p5, -1.6668057665e-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p6, +2.0000714765e-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p7, -2.4999993993e-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p8, +3.3333331174e-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_q1, -2.12194440e-4f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_q2, 0.693359375f); static _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); static _EIGEN_DECLARE_CONST_Packet4f(1, 1.0f); // Convert negative argument into NAN (quiet negative, to be specific). Packet4f zero = (Packet4f)__builtin_msa_ldi_w(0); Packet4i neg_mask = __builtin_msa_fclt_w(_x, zero); Packet4i zero_mask = __builtin_msa_fceq_w(_x, zero); Packet4f non_neg_x_or_nan = padd(_x, (Packet4f)neg_mask); // Add 0.0 or NAN. Packet4f x = non_neg_x_or_nan; // Extract exponent from x = mantissa * 2**exponent, where 1.0 <= mantissa < 2.0. // N.B. the exponent is one less of what frexpf() would return. Packet4i e_int = __builtin_msa_ftint_s_w(__builtin_msa_flog2_w(x)); // Multiply x by 2**(-exponent-1) to get 0.5 <= x < 1.0 as from frexpf(). x = __builtin_msa_fexp2_w(x, (Packet4i)__builtin_msa_nori_b((v16u8)e_int, 0)); /* if (x < SQRTHF) { x = x + x - 1.0; } else { e += 1; x = x - 1.0; } */ Packet4f xx = padd(x, x); Packet4i ge_mask = __builtin_msa_fcle_w(p4f_cephes_SQRTHF, x); e_int = psub(e_int, ge_mask); x = (Packet4f)__builtin_msa_bsel_v((v16u8)ge_mask, (v16u8)xx, (v16u8)x); x = psub(x, p4f_1); Packet4f e = __builtin_msa_ffint_s_w(e_int); Packet4f x2 = pmul(x, x); Packet4f x3 = pmul(x2, x); Packet4f y, y1, y2; y = pmadd(p4f_cephes_log_p0, x, p4f_cephes_log_p1); y1 = pmadd(p4f_cephes_log_p3, x, p4f_cephes_log_p4); y2 = pmadd(p4f_cephes_log_p6, x, p4f_cephes_log_p7); y = pmadd(y, x, p4f_cephes_log_p2); y1 = pmadd(y1, x, p4f_cephes_log_p5); y2 = pmadd(y2, x, p4f_cephes_log_p8); y = pmadd(y, x3, y1); y = pmadd(y, x3, y2); y = pmul(y, x3); y = pmadd(e, p4f_cephes_log_q1, y); x = __builtin_msa_fmsub_w(x, x2, p4f_half); x = padd(x, y); x = pmadd(e, p4f_cephes_log_q2, x); // x is now the logarithm result candidate. We still need to handle the // extreme arguments of zero and positive infinity, though. // N.B. if the argument is +INFINITY, x is NAN because the polynomial terms // contain infinities of both signs (see the coefficients and code above). // INFINITY - INFINITY is NAN. // If the argument is +INFINITY, make it the new result candidate. // To achieve that we choose the smaller of the result candidate and the // argument. // This is correct for all finite pairs of values (the logarithm is smaller // than the argument). // This is also correct in the special case when the argument is +INFINITY // and the result candidate is NAN. This is because the fmin.df instruction // prefers non-NANs to NANs. x = __builtin_msa_fmin_w(x, non_neg_x_or_nan); // If the argument is zero (including -0.0), the result becomes -INFINITY. Packet4i neg_infs = __builtin_msa_slli_w(zero_mask, 23); x = (Packet4f)__builtin_msa_bsel_v((v16u8)zero_mask, (v16u8)x, (v16u8)neg_infs); return x; } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pexp(const Packet4f& _x) { // Limiting single-precision pexp's argument to [-128, +128] lets pexp // reach 0 and INFINITY naturally. static _EIGEN_DECLARE_CONST_Packet4f(exp_lo, -128.0f); static _EIGEN_DECLARE_CONST_Packet4f(exp_hi, +128.0f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_LOG2EF, 1.44269504088896341f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C1, 0.693359375f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C2, -2.12194440e-4f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p0, 1.9875691500e-4f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p1, 1.3981999507e-3f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p2, 8.3334519073e-3f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p3, 4.1665795894e-2f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p4, 1.6666665459e-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p5, 5.0000001201e-1f); static _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); static _EIGEN_DECLARE_CONST_Packet4f(1, 1.0f); Packet4f x = _x; // Clamp x. x = (Packet4f)__builtin_msa_bsel_v((v16u8)__builtin_msa_fclt_w(x, p4f_exp_lo), (v16u8)x, (v16u8)p4f_exp_lo); x = (Packet4f)__builtin_msa_bsel_v((v16u8)__builtin_msa_fclt_w(p4f_exp_hi, x), (v16u8)x, (v16u8)p4f_exp_hi); // Round to nearest integer by adding 0.5 (with x's sign) and truncating. Packet4f x2_add = (Packet4f)__builtin_msa_binsli_w((v4u32)p4f_half, (v4u32)x, 0); Packet4f x2 = pmadd(x, p4f_cephes_LOG2EF, x2_add); Packet4i x2_int = __builtin_msa_ftrunc_s_w(x2); Packet4f x2_int_f = __builtin_msa_ffint_s_w(x2_int); x = __builtin_msa_fmsub_w(x, x2_int_f, p4f_cephes_exp_C1); x = __builtin_msa_fmsub_w(x, x2_int_f, p4f_cephes_exp_C2); Packet4f z = pmul(x, x); Packet4f y = p4f_cephes_exp_p0; y = pmadd(y, x, p4f_cephes_exp_p1); y = pmadd(y, x, p4f_cephes_exp_p2); y = pmadd(y, x, p4f_cephes_exp_p3); y = pmadd(y, x, p4f_cephes_exp_p4); y = pmadd(y, x, p4f_cephes_exp_p5); y = pmadd(y, z, x); y = padd(y, p4f_1); // y *= 2**exponent. y = __builtin_msa_fexp2_w(y, x2_int); return y; } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f ptanh(const Packet4f& _x) { static _EIGEN_DECLARE_CONST_Packet4f(tanh_tiny, 1e-4f); static _EIGEN_DECLARE_CONST_Packet4f(tanh_hi, 9.0f); // The monomial coefficients of the numerator polynomial (odd). static _EIGEN_DECLARE_CONST_Packet4f(alpha_1, 4.89352455891786e-3f); static _EIGEN_DECLARE_CONST_Packet4f(alpha_3, 6.37261928875436e-4f); static _EIGEN_DECLARE_CONST_Packet4f(alpha_5, 1.48572235717979e-5f); static _EIGEN_DECLARE_CONST_Packet4f(alpha_7, 5.12229709037114e-8f); static _EIGEN_DECLARE_CONST_Packet4f(alpha_9, -8.60467152213735e-11f); static _EIGEN_DECLARE_CONST_Packet4f(alpha_11, 2.00018790482477e-13f); static _EIGEN_DECLARE_CONST_Packet4f(alpha_13, -2.76076847742355e-16f); // The monomial coefficients of the denominator polynomial (even). static _EIGEN_DECLARE_CONST_Packet4f(beta_0, 4.89352518554385e-3f); static _EIGEN_DECLARE_CONST_Packet4f(beta_2, 2.26843463243900e-3f); static _EIGEN_DECLARE_CONST_Packet4f(beta_4, 1.18534705686654e-4f); static _EIGEN_DECLARE_CONST_Packet4f(beta_6, 1.19825839466702e-6f); Packet4f x = pabs(_x); Packet4i tiny_mask = __builtin_msa_fclt_w(x, p4f_tanh_tiny); // Clamp the inputs to the range [-9, 9] since anything outside // this range is -/+1.0f in single-precision. x = (Packet4f)__builtin_msa_bsel_v((v16u8)__builtin_msa_fclt_w(p4f_tanh_hi, x), (v16u8)x, (v16u8)p4f_tanh_hi); // Since the polynomials are odd/even, we need x**2. Packet4f x2 = pmul(x, x); // Evaluate the numerator polynomial p. Packet4f p = pmadd(x2, p4f_alpha_13, p4f_alpha_11); p = pmadd(x2, p, p4f_alpha_9); p = pmadd(x2, p, p4f_alpha_7); p = pmadd(x2, p, p4f_alpha_5); p = pmadd(x2, p, p4f_alpha_3); p = pmadd(x2, p, p4f_alpha_1); p = pmul(x, p); // Evaluate the denominator polynomial q. Packet4f q = pmadd(x2, p4f_beta_6, p4f_beta_4); q = pmadd(x2, q, p4f_beta_2); q = pmadd(x2, q, p4f_beta_0); // Divide the numerator by the denominator. p = pdiv(p, q); // Reinstate the sign. p = (Packet4f)__builtin_msa_binsli_w((v4u32)p, (v4u32)_x, 0); // When the argument is very small in magnitude it's more accurate to just return it. p = (Packet4f)__builtin_msa_bsel_v((v16u8)tiny_mask, (v16u8)p, (v16u8)_x); return p; } template Packet4f psincos_inner_msa_float(const Packet4f& _x) { static _EIGEN_DECLARE_CONST_Packet4f(sincos_max_arg, 13176795.0f); // Approx. (2**24) / (4/Pi). static _EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP1, -0.78515625f); static _EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP2, -2.4187564849853515625e-4f); static _EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP3, -3.77489497744594108e-8f); static _EIGEN_DECLARE_CONST_Packet4f(sincof_p0, -1.9515295891e-4f); static _EIGEN_DECLARE_CONST_Packet4f(sincof_p1, 8.3321608736e-3f); static _EIGEN_DECLARE_CONST_Packet4f(sincof_p2, -1.6666654611e-1f); static _EIGEN_DECLARE_CONST_Packet4f(coscof_p0, 2.443315711809948e-5f); static _EIGEN_DECLARE_CONST_Packet4f(coscof_p1, -1.388731625493765e-3f); static _EIGEN_DECLARE_CONST_Packet4f(coscof_p2, 4.166664568298827e-2f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_FOPI, 1.27323954473516f); // 4/Pi. static _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); static _EIGEN_DECLARE_CONST_Packet4f(1, 1.0f); Packet4f x = pabs(_x); // Translate infinite arguments into NANs. Packet4f zero_or_nan_if_inf = psub(_x, _x); x = padd(x, zero_or_nan_if_inf); // Prevent sin/cos from generating values larger than 1.0 in magnitude // for very large arguments by setting x to 0.0. Packet4i small_or_nan_mask = __builtin_msa_fcult_w(x, p4f_sincos_max_arg); x = pand(x, (Packet4f)small_or_nan_mask); // Scale x by 4/Pi to find x's octant. Packet4f y = pmul(x, p4f_cephes_FOPI); // Get the octant. We'll reduce x by this number of octants or by one more than it. Packet4i y_int = __builtin_msa_ftrunc_s_w(y); // x's from even-numbered octants will translate to octant 0: [0, +Pi/4]. // x's from odd-numbered octants will translate to octant -1: [-Pi/4, 0]. // Adjustment for odd-numbered octants: octant = (octant + 1) & (~1). Packet4i y_int1 = __builtin_msa_addvi_w(y_int, 1); Packet4i y_int2 = (Packet4i)__builtin_msa_bclri_w((Packet4ui)y_int1, 0); // bclri = bit-clear y = __builtin_msa_ffint_s_w(y_int2); // Compute the sign to apply to the polynomial. Packet4i sign_mask = sine ? pxor(__builtin_msa_slli_w(y_int1, 29), (Packet4i)_x) : __builtin_msa_slli_w(__builtin_msa_addvi_w(y_int, 3), 29); // Get the polynomial selection mask. // We'll calculate both (sin and cos) polynomials and then select from the two. Packet4i poly_mask = __builtin_msa_ceqi_w(__builtin_msa_slli_w(y_int2, 30), 0); // Reduce x by y octants to get: -Pi/4 <= x <= +Pi/4. // The magic pass: "Extended precision modular arithmetic" // x = ((x - y * DP1) - y * DP2) - y * DP3 Packet4f tmp1 = pmul(y, p4f_minus_cephes_DP1); Packet4f tmp2 = pmul(y, p4f_minus_cephes_DP2); Packet4f tmp3 = pmul(y, p4f_minus_cephes_DP3); x = padd(x, tmp1); x = padd(x, tmp2); x = padd(x, tmp3); // Evaluate the cos(x) polynomial. y = p4f_coscof_p0; Packet4f z = pmul(x, x); y = pmadd(y, z, p4f_coscof_p1); y = pmadd(y, z, p4f_coscof_p2); y = pmul(y, z); y = pmul(y, z); y = __builtin_msa_fmsub_w(y, z, p4f_half); y = padd(y, p4f_1); // Evaluate the sin(x) polynomial. Packet4f y2 = p4f_sincof_p0; y2 = pmadd(y2, z, p4f_sincof_p1); y2 = pmadd(y2, z, p4f_sincof_p2); y2 = pmul(y2, z); y2 = pmadd(y2, x, x); // Select the correct result from the two polynomials. y = sine ? (Packet4f)__builtin_msa_bsel_v((v16u8)poly_mask, (v16u8)y, (v16u8)y2) : (Packet4f)__builtin_msa_bsel_v((v16u8)poly_mask, (v16u8)y2, (v16u8)y); // Update the sign. sign_mask = pxor(sign_mask, (Packet4i)y); y = (Packet4f)__builtin_msa_binsli_w((v4u32)y, (v4u32)sign_mask, 0); // binsli = bit-insert-left return y; } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f psin(const Packet4f& x) { return psincos_inner_msa_float(x); } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pcos(const Packet4f& x) { return psincos_inner_msa_float(x); } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d pexp(const Packet2d& _x) { // Limiting double-precision pexp's argument to [-1024, +1024] lets pexp // reach 0 and INFINITY naturally. static _EIGEN_DECLARE_CONST_Packet2d(exp_lo, -1024.0); static _EIGEN_DECLARE_CONST_Packet2d(exp_hi, +1024.0); static _EIGEN_DECLARE_CONST_Packet2d(cephes_LOG2EF, 1.4426950408889634073599); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C1, 0.693145751953125); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C2, 1.42860682030941723212e-6); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p0, 1.26177193074810590878e-4); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p1, 3.02994407707441961300e-2); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p2, 9.99999999999999999910e-1); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q0, 3.00198505138664455042e-6); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q1, 2.52448340349684104192e-3); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q2, 2.27265548208155028766e-1); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q3, 2.00000000000000000009e0); static _EIGEN_DECLARE_CONST_Packet2d(half, 0.5); static _EIGEN_DECLARE_CONST_Packet2d(1, 1.0); static _EIGEN_DECLARE_CONST_Packet2d(2, 2.0); Packet2d x = _x; // Clamp x. x = (Packet2d)__builtin_msa_bsel_v((v16u8)__builtin_msa_fclt_d(x, p2d_exp_lo), (v16u8)x, (v16u8)p2d_exp_lo); x = (Packet2d)__builtin_msa_bsel_v((v16u8)__builtin_msa_fclt_d(p2d_exp_hi, x), (v16u8)x, (v16u8)p2d_exp_hi); // Round to nearest integer by adding 0.5 (with x's sign) and truncating. Packet2d x2_add = (Packet2d)__builtin_msa_binsli_d((v2u64)p2d_half, (v2u64)x, 0); Packet2d x2 = pmadd(x, p2d_cephes_LOG2EF, x2_add); Packet2l x2_long = __builtin_msa_ftrunc_s_d(x2); Packet2d x2_long_d = __builtin_msa_ffint_s_d(x2_long); x = __builtin_msa_fmsub_d(x, x2_long_d, p2d_cephes_exp_C1); x = __builtin_msa_fmsub_d(x, x2_long_d, p2d_cephes_exp_C2); x2 = pmul(x, x); Packet2d px = p2d_cephes_exp_p0; px = pmadd(px, x2, p2d_cephes_exp_p1); px = pmadd(px, x2, p2d_cephes_exp_p2); px = pmul(px, x); Packet2d qx = p2d_cephes_exp_q0; qx = pmadd(qx, x2, p2d_cephes_exp_q1); qx = pmadd(qx, x2, p2d_cephes_exp_q2); qx = pmadd(qx, x2, p2d_cephes_exp_q3); x = pdiv(px, psub(qx, px)); x = pmadd(p2d_2, x, p2d_1); // x *= 2**exponent. x = __builtin_msa_fexp2_d(x, x2_long); return x; } } // end namespace internal } // end namespace Eigen #endif // EIGEN_MATH_FUNCTIONS_MSA_H RcppEigen/inst/include/Eigen/src/Core/arch/ZVector/0000755000176200001440000000000014567757725021576 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/arch/ZVector/Complex.h0000644000176200001440000004053014567757725023360 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2010 Gael Guennebaud // Copyright (C) 2016 Konstantinos Margaritis // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_COMPLEX32_ALTIVEC_H #define EIGEN_COMPLEX32_ALTIVEC_H namespace Eigen { namespace internal { #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) static Packet4ui p4ui_CONJ_XOR = { 0x00000000, 0x80000000, 0x00000000, 0x80000000 }; //vec_mergeh((Packet4ui)p4i_ZERO, (Packet4ui)p4f_MZERO); #endif static Packet2ul p2ul_CONJ_XOR1 = (Packet2ul) vec_sld((Packet4ui) p2d_ZERO_, (Packet4ui) p2l_ZERO, 8);//{ 0x8000000000000000, 0x0000000000000000 }; static Packet2ul p2ul_CONJ_XOR2 = (Packet2ul) vec_sld((Packet4ui) p2l_ZERO, (Packet4ui) p2d_ZERO_, 8);//{ 0x8000000000000000, 0x0000000000000000 }; struct Packet1cd { EIGEN_STRONG_INLINE Packet1cd() {} EIGEN_STRONG_INLINE explicit Packet1cd(const Packet2d& a) : v(a) {} Packet2d v; }; struct Packet2cf { EIGEN_STRONG_INLINE Packet2cf() {} EIGEN_STRONG_INLINE explicit Packet2cf(const Packet4f& a) : v(a) {} #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ < 12) union { Packet4f v; Packet1cd cd[2]; }; #else Packet4f v; #endif }; template<> struct packet_traits > : default_packet_traits { typedef Packet2cf type; typedef Packet2cf half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 2, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, HasMax = 0, HasBlend = 1, HasSetLinear = 0 }; }; template<> struct packet_traits > : default_packet_traits { typedef Packet1cd type; typedef Packet1cd half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 1, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, HasMax = 0, HasSetLinear = 0 }; }; template<> struct unpacket_traits { typedef std::complex type; enum {size=2, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet2cf half; }; template<> struct unpacket_traits { typedef std::complex type; enum {size=1, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet1cd half; }; /* Forward declaration */ EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel); /* complex first */ template<> EIGEN_STRONG_INLINE Packet1cd pload (const std::complex* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet1cd(pload((const double*)from)); } template<> EIGEN_STRONG_INLINE Packet1cd ploadu(const std::complex* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet1cd(ploadu((const double*)from)); } template<> EIGEN_STRONG_INLINE void pstore >(std::complex * to, const Packet1cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v); } template<> EIGEN_STRONG_INLINE void pstoreu >(std::complex * to, const Packet1cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, from.v); } template<> EIGEN_STRONG_INLINE Packet1cd pset1(const std::complex& from) { /* here we really have to use unaligned loads :( */ return ploadu(&from); } template<> EIGEN_DEVICE_FUNC inline Packet1cd pgather, Packet1cd>(const std::complex* from, Index stride EIGEN_UNUSED) { return pload(from); } template<> EIGEN_DEVICE_FUNC inline void pscatter, Packet1cd>(std::complex* to, const Packet1cd& from, Index stride EIGEN_UNUSED) { pstore >(to, from); } template<> EIGEN_STRONG_INLINE Packet1cd padd(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(a.v + b.v); } template<> EIGEN_STRONG_INLINE Packet1cd psub(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(a.v - b.v); } template<> EIGEN_STRONG_INLINE Packet1cd pnegate(const Packet1cd& a) { return Packet1cd(pnegate(Packet2d(a.v))); } template<> EIGEN_STRONG_INLINE Packet1cd pconj(const Packet1cd& a) { return Packet1cd((Packet2d)vec_xor((Packet2d)a.v, (Packet2d)p2ul_CONJ_XOR2)); } template<> EIGEN_STRONG_INLINE Packet1cd pmul(const Packet1cd& a, const Packet1cd& b) { Packet2d a_re, a_im, v1, v2; // Permute and multiply the real parts of a and b a_re = vec_perm(a.v, a.v, p16uc_PSET64_HI); // Get the imaginary parts of a a_im = vec_perm(a.v, a.v, p16uc_PSET64_LO); // multiply a_re * b v1 = vec_madd(a_re, b.v, p2d_ZERO); // multiply a_im * b and get the conjugate result v2 = vec_madd(a_im, b.v, p2d_ZERO); v2 = (Packet2d) vec_sld((Packet4ui)v2, (Packet4ui)v2, 8); v2 = (Packet2d) vec_xor((Packet2d)v2, (Packet2d) p2ul_CONJ_XOR1); return Packet1cd(v1 + v2); } template<> EIGEN_STRONG_INLINE Packet1cd pand (const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_and(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd por (const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_or(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd pxor (const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_xor(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet1cd pandnot (const Packet1cd& a, const Packet1cd& b) { return Packet1cd(vec_and(a.v, vec_nor(b.v,b.v))); } template<> EIGEN_STRONG_INLINE Packet1cd ploaddup(const std::complex* from) { return pset1(*from); } template<> EIGEN_STRONG_INLINE Packet1cd pcmp_eq(const Packet1cd& a, const Packet1cd& b) { Packet2d eq = vec_cmpeq (a.v, b.v); Packet2d tmp = { eq[1], eq[0] }; return (Packet1cd)pand(eq, tmp); } template<> EIGEN_STRONG_INLINE void prefetch >(const std::complex * addr) { EIGEN_ZVECTOR_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE std::complex pfirst(const Packet1cd& a) { std::complex EIGEN_ALIGN16 res; pstore >(&res, a); return res; } template<> EIGEN_STRONG_INLINE Packet1cd preverse(const Packet1cd& a) { return a; } template<> EIGEN_STRONG_INLINE std::complex predux(const Packet1cd& a) { return pfirst(a); } template<> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet1cd& a) { return pfirst(a); } EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd,Packet2d) template<> EIGEN_STRONG_INLINE Packet1cd pdiv(const Packet1cd& a, const Packet1cd& b) { // TODO optimize it for AltiVec Packet1cd res = pmul(a,pconj(b)); Packet2d s = vec_madd(b.v, b.v, p2d_ZERO_); return Packet1cd(pdiv(res.v, s + vec_perm(s, s, p16uc_REVERSE64))); } EIGEN_STRONG_INLINE Packet1cd pcplxflip/**/(const Packet1cd& x) { return Packet1cd(preverse(Packet2d(x.v))); } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { Packet2d tmp = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_HI); kernel.packet[1].v = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_LO); kernel.packet[0].v = tmp; } /* complex follows */ template<> EIGEN_STRONG_INLINE Packet2cf pload (const std::complex* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet2cf(pload((const float*)from)); } template<> EIGEN_STRONG_INLINE Packet2cf ploadu(const std::complex* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cf(ploadu((const float*)from)); } template<> EIGEN_STRONG_INLINE void pstore >(std::complex * to, const Packet2cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((float*)to, from.v); } template<> EIGEN_STRONG_INLINE void pstoreu >(std::complex * to, const Packet2cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((float*)to, from.v); } template<> EIGEN_STRONG_INLINE std::complex pfirst(const Packet2cf& a) { std::complex EIGEN_ALIGN16 res[2]; pstore >(res, a); return res[0]; } #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ < 12) template<> EIGEN_STRONG_INLINE Packet2cf pset1(const std::complex& from) { Packet2cf res; res.cd[0] = Packet1cd(vec_ld2f((const float *)&from)); res.cd[1] = res.cd[0]; return res; } #else template<> EIGEN_STRONG_INLINE Packet2cf pset1(const std::complex& from) { Packet2cf res; if((std::ptrdiff_t(&from) % 16) == 0) res.v = pload((const float *)&from); else res.v = ploadu((const float *)&from); res.v = vec_perm(res.v, res.v, p16uc_PSET64_HI); return res; } #endif template<> EIGEN_DEVICE_FUNC inline Packet2cf pgather, Packet2cf>(const std::complex* from, Index stride) { std::complex EIGEN_ALIGN16 af[2]; af[0] = from[0*stride]; af[1] = from[1*stride]; return pload(af); } template<> EIGEN_DEVICE_FUNC inline void pscatter, Packet2cf>(std::complex* to, const Packet2cf& from, Index stride) { std::complex EIGEN_ALIGN16 af[2]; pstore >((std::complex *) af, from); to[0*stride] = af[0]; to[1*stride] = af[1]; } template<> EIGEN_STRONG_INLINE Packet2cf padd(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(padd(a.v, b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf psub(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(psub(a.v, b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf& a) { return Packet2cf(pnegate(Packet4f(a.v))); } template<> EIGEN_STRONG_INLINE Packet2cf pand (const Packet2cf& a, const Packet2cf& b) { return Packet2cf(pand(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf por (const Packet2cf& a, const Packet2cf& b) { return Packet2cf(por(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf pxor (const Packet2cf& a, const Packet2cf& b) { return Packet2cf(pxor(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf pandnot(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(pandnot(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf ploaddup(const std::complex* from) { return pset1(*from); } template<> EIGEN_STRONG_INLINE void prefetch >(const std::complex * addr) { EIGEN_ZVECTOR_PREFETCH(addr); } #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ < 12) template<> EIGEN_STRONG_INLINE Packet2cf pcmp_eq(const Packet2cf& a, const Packet2cf& b) { Packet4f eq = pcmp_eq (a.v, b.v); Packet2cf res; Packet2d tmp1 = { eq.v4f[0][1], eq.v4f[0][0] }; Packet2d tmp2 = { eq.v4f[1][1], eq.v4f[1][0] }; res.v.v4f[0] = pand(eq.v4f[0], tmp1); res.v.v4f[1] = pand(eq.v4f[1], tmp2); return res; } template<> EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) { Packet2cf res; res.v.v4f[0] = pconj(Packet1cd(reinterpret_cast(a.v.v4f[0]))).v; res.v.v4f[1] = pconj(Packet1cd(reinterpret_cast(a.v.v4f[1]))).v; return res; } template<> EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) { Packet2cf res; res.v.v4f[0] = pmul(Packet1cd(reinterpret_cast(a.v.v4f[0])), Packet1cd(reinterpret_cast(b.v.v4f[0]))).v; res.v.v4f[1] = pmul(Packet1cd(reinterpret_cast(a.v.v4f[1])), Packet1cd(reinterpret_cast(b.v.v4f[1]))).v; return res; } template<> EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) { Packet2cf res; res.cd[0] = a.cd[1]; res.cd[1] = a.cd[0]; return res; } template<> EIGEN_STRONG_INLINE std::complex predux(const Packet2cf& a) { std::complex res; Packet1cd b = padd(a.cd[0], a.cd[1]); vec_st2f(b.v, (float*)&res); return res; } template<> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet2cf& a) { std::complex res; Packet1cd b = pmul(a.cd[0], a.cd[1]); vec_st2f(b.v, (float*)&res); return res; } EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf,Packet4f) template<> EIGEN_STRONG_INLINE Packet2cf pdiv(const Packet2cf& a, const Packet2cf& b) { // TODO optimize it for AltiVec Packet2cf res; res.cd[0] = pdiv(a.cd[0], b.cd[0]); res.cd[1] = pdiv(a.cd[1], b.cd[1]); return res; } EIGEN_STRONG_INLINE Packet2cf pcplxflip/**/(const Packet2cf& x) { Packet2cf res; res.cd[0] = pcplxflip(x.cd[0]); res.cd[1] = pcplxflip(x.cd[1]); return res; } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { Packet1cd tmp = kernel.packet[0].cd[1]; kernel.packet[0].cd[1] = kernel.packet[1].cd[0]; kernel.packet[1].cd[0] = tmp; } template<> EIGEN_STRONG_INLINE Packet2cf pblend(const Selector<2>& ifPacket, const Packet2cf& thenPacket, const Packet2cf& elsePacket) { Packet2cf result; const Selector<4> ifPacket4 = { ifPacket.select[0], ifPacket.select[0], ifPacket.select[1], ifPacket.select[1] }; result.v = pblend(ifPacket4, thenPacket.v, elsePacket.v); return result; } #else template<> EIGEN_STRONG_INLINE Packet2cf pcmp_eq(const Packet2cf& a, const Packet2cf& b) { Packet4f eq = vec_cmpeq (a.v, b.v); Packet4f tmp = { eq[1], eq[0], eq[3], eq[2] }; return (Packet2cf)pand(eq, tmp); } template<> EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) { return Packet2cf(pxor(a.v, reinterpret_cast(p4ui_CONJ_XOR))); } template<> EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) { Packet4f a_re, a_im, prod, prod_im; // Permute and multiply the real parts of a and b a_re = vec_perm(a.v, a.v, p16uc_PSET32_WODD); // Get the imaginary parts of a a_im = vec_perm(a.v, a.v, p16uc_PSET32_WEVEN); // multiply a_im * b and get the conjugate result prod_im = a_im * b.v; prod_im = pxor(prod_im, reinterpret_cast(p4ui_CONJ_XOR)); // permute back to a proper order prod_im = vec_perm(prod_im, prod_im, p16uc_COMPLEX32_REV); // multiply a_re * b, add prod_im prod = pmadd(a_re, b.v, prod_im); return Packet2cf(prod); } template<> EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) { Packet4f rev_a; rev_a = vec_perm(a.v, a.v, p16uc_COMPLEX32_REV2); return Packet2cf(rev_a); } template<> EIGEN_STRONG_INLINE std::complex predux(const Packet2cf& a) { Packet4f b; b = vec_sld(a.v, a.v, 8); b = padd(a.v, b); return pfirst(Packet2cf(b)); } template<> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet2cf& a) { Packet4f b; Packet2cf prod; b = vec_sld(a.v, a.v, 8); prod = pmul(a, Packet2cf(b)); return pfirst(prod); } EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf,Packet4f) template<> EIGEN_STRONG_INLINE Packet2cf pdiv(const Packet2cf& a, const Packet2cf& b) { // TODO optimize it for AltiVec Packet2cf res = pmul(a, pconj(b)); Packet4f s = pmul(b.v, b.v); return Packet2cf(pdiv(res.v, padd(s, vec_perm(s, s, p16uc_COMPLEX32_REV)))); } template<> EIGEN_STRONG_INLINE Packet2cf pcplxflip(const Packet2cf& x) { return Packet2cf(vec_perm(x.v, x.v, p16uc_COMPLEX32_REV)); } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { Packet4f tmp = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_HI); kernel.packet[1].v = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_LO); kernel.packet[0].v = tmp; } template<> EIGEN_STRONG_INLINE Packet2cf pblend(const Selector<2>& ifPacket, const Packet2cf& thenPacket, const Packet2cf& elsePacket) { Packet2cf result; result.v = reinterpret_cast(pblend(ifPacket, reinterpret_cast(thenPacket.v), reinterpret_cast(elsePacket.v))); return result; } #endif } // end namespace internal } // end namespace Eigen #endif // EIGEN_COMPLEX32_ALTIVEC_H RcppEigen/inst/include/Eigen/src/Core/arch/ZVector/PacketMath.h0000755000176200001440000011003614567757725023774 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2016 Konstantinos Margaritis // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_PACKET_MATH_ZVECTOR_H #define EIGEN_PACKET_MATH_ZVECTOR_H namespace Eigen { namespace internal { #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 16 #endif #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD #endif #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32 #endif typedef __vector int Packet4i; typedef __vector unsigned int Packet4ui; typedef __vector __bool int Packet4bi; typedef __vector short int Packet8i; typedef __vector unsigned char Packet16uc; typedef __vector double Packet2d; typedef __vector unsigned long long Packet2ul; typedef __vector long long Packet2l; // Z14 has builtin support for float vectors #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) typedef __vector float Packet4f; #else typedef struct { Packet2d v4f[2]; } Packet4f; #endif typedef union { numext::int32_t i[4]; numext::uint32_t ui[4]; numext::int64_t l[2]; numext::uint64_t ul[2]; double d[2]; float f[4]; Packet4i v4i; Packet4ui v4ui; Packet2l v2l; Packet2ul v2ul; Packet2d v2d; #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) Packet4f v4f; #endif } Packet; // We don't want to write the same code all the time, but we need to reuse the constants // and it doesn't really work to declare them global, so we define macros instead #define _EIGEN_DECLARE_CONST_FAST_Packet4i(NAME,X) \ Packet4i p4i_##NAME = reinterpret_cast(vec_splat_s32(X)) #define _EIGEN_DECLARE_CONST_FAST_Packet2d(NAME,X) \ Packet2d p2d_##NAME = reinterpret_cast(vec_splat_s64(X)) #define _EIGEN_DECLARE_CONST_FAST_Packet2l(NAME,X) \ Packet2l p2l_##NAME = reinterpret_cast(vec_splat_s64(X)) #define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ Packet4i p4i_##NAME = pset1(X) #define _EIGEN_DECLARE_CONST_Packet2d(NAME,X) \ Packet2d p2d_##NAME = pset1(X) #define _EIGEN_DECLARE_CONST_Packet2l(NAME,X) \ Packet2l p2l_##NAME = pset1(X) // These constants are endian-agnostic static _EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0); //{ 0, 0, 0, 0,} static _EIGEN_DECLARE_CONST_FAST_Packet4i(ONE, 1); //{ 1, 1, 1, 1} static _EIGEN_DECLARE_CONST_FAST_Packet2d(ZERO, 0); static _EIGEN_DECLARE_CONST_FAST_Packet2l(ZERO, 0); static _EIGEN_DECLARE_CONST_FAST_Packet2l(ONE, 1); static Packet2d p2d_ONE = { 1.0, 1.0 }; static Packet2d p2d_ZERO_ = { numext::bit_cast0x8000000000000000ull), numext::bit_cast0x8000000000000000ull) }; #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) #define _EIGEN_DECLARE_CONST_FAST_Packet4f(NAME,X) \ Packet4f p4f_##NAME = reinterpret_cast(vec_splat_s32(X)) #define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ Packet4f p4f_##NAME = pset1(X) #define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ const Packet4f p4f_##NAME = reinterpret_cast(pset1(X)) static _EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0); //{ 0.0, 0.0, 0.0, 0.0} static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1,-1); //{ -1, -1, -1, -1} static Packet4f p4f_MZERO = { 0x80000000, 0x80000000, 0x80000000, 0x80000000}; #endif static Packet4i p4i_COUNTDOWN = { 0, 1, 2, 3 }; static Packet4f p4f_COUNTDOWN = { 0.0, 1.0, 2.0, 3.0 }; static Packet2d p2d_COUNTDOWN = reinterpret_cast(vec_sld(reinterpret_cast(p2d_ZERO), reinterpret_cast(p2d_ONE), 8)); static Packet16uc p16uc_PSET64_HI = { 0,1,2,3, 4,5,6,7, 0,1,2,3, 4,5,6,7 }; static Packet16uc p16uc_DUPLICATE32_HI = { 0,1,2,3, 0,1,2,3, 4,5,6,7, 4,5,6,7 }; // Mask alignment #define _EIGEN_MASK_ALIGNMENT 0xfffffffffffffff0 #define _EIGEN_ALIGNED_PTR(x) ((std::ptrdiff_t)(x) & _EIGEN_MASK_ALIGNMENT) // Handle endianness properly while loading constants // Define global static constants: static Packet16uc p16uc_FORWARD = { 0,1,2,3, 4,5,6,7, 8,9,10,11, 12,13,14,15 }; static Packet16uc p16uc_REVERSE32 = { 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3 }; static Packet16uc p16uc_REVERSE64 = { 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 }; static Packet16uc p16uc_PSET32_WODD = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);//{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 }; static Packet16uc p16uc_PSET32_WEVEN = vec_sld(p16uc_DUPLICATE32_HI, (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);//{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 }; /*static Packet16uc p16uc_HALF64_0_16 = vec_sld((Packet16uc)p4i_ZERO, vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 3), 8); //{ 0,0,0,0, 0,0,0,0, 16,16,16,16, 16,16,16,16}; static Packet16uc p16uc_PSET64_HI = (Packet16uc) vec_mergeh((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN); //{ 0,1,2,3, 4,5,6,7, 0,1,2,3, 4,5,6,7 };*/ static Packet16uc p16uc_PSET64_LO = (Packet16uc) vec_mergel((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN); //{ 8,9,10,11, 12,13,14,15, 8,9,10,11, 12,13,14,15 }; /*static Packet16uc p16uc_TRANSPOSE64_HI = vec_add(p16uc_PSET64_HI, p16uc_HALF64_0_16); //{ 0,1,2,3, 4,5,6,7, 16,17,18,19, 20,21,22,23}; static Packet16uc p16uc_TRANSPOSE64_LO = vec_add(p16uc_PSET64_LO, p16uc_HALF64_0_16); //{ 8,9,10,11, 12,13,14,15, 24,25,26,27, 28,29,30,31};*/ static Packet16uc p16uc_TRANSPOSE64_HI = { 0,1,2,3, 4,5,6,7, 16,17,18,19, 20,21,22,23}; static Packet16uc p16uc_TRANSPOSE64_LO = { 8,9,10,11, 12,13,14,15, 24,25,26,27, 28,29,30,31}; static Packet16uc p16uc_COMPLEX32_REV = vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8); //{ 4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11 }; static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_FORWARD, p16uc_FORWARD, 8); //{ 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 }; #if EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC #define EIGEN_ZVECTOR_PREFETCH(ADDR) __builtin_prefetch(ADDR); #else #define EIGEN_ZVECTOR_PREFETCH(ADDR) asm( " pfd [%[addr]]\n" :: [addr] "r" (ADDR) : "cc" ); #endif template<> struct packet_traits : default_packet_traits { typedef Packet4i type; typedef Packet4i half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 4, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasBlend = 1 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet4f type; typedef Packet4f half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 4, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasMin = 1, HasMax = 1, HasAbs = 1, HasSin = 0, HasCos = 0, HasLog = 0, HasExp = 1, HasSqrt = 1, HasRsqrt = 1, HasTanh = 1, HasErf = 1, HasRound = 1, HasFloor = 1, HasCeil = 1, HasNegate = 1, HasBlend = 1 }; }; template<> struct packet_traits : default_packet_traits { typedef Packet2d type; typedef Packet2d half; enum { Vectorizable = 1, AlignedOnScalar = 1, size=2, HasHalfPacket = 1, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasMin = 1, HasMax = 1, HasAbs = 1, HasSin = 0, HasCos = 0, HasLog = 0, HasExp = 1, HasSqrt = 1, HasRsqrt = 1, HasRound = 1, HasFloor = 1, HasCeil = 1, HasNegate = 1, HasBlend = 1 }; }; template<> struct unpacket_traits { typedef int type; enum {size=4, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet4i half; }; template<> struct unpacket_traits { typedef float type; enum {size=4, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet4f half; }; template<> struct unpacket_traits { typedef double type; enum {size=2, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet2d half; }; /* Forward declaration */ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel); inline std::ostream & operator <<(std::ostream & s, const Packet4i & v) { Packet vt; vt.v4i = v; s << vt.i[0] << ", " << vt.i[1] << ", " << vt.i[2] << ", " << vt.i[3]; return s; } inline std::ostream & operator <<(std::ostream & s, const Packet4ui & v) { Packet vt; vt.v4ui = v; s << vt.ui[0] << ", " << vt.ui[1] << ", " << vt.ui[2] << ", " << vt.ui[3]; return s; } inline std::ostream & operator <<(std::ostream & s, const Packet2l & v) { Packet vt; vt.v2l = v; s << vt.l[0] << ", " << vt.l[1]; return s; } inline std::ostream & operator <<(std::ostream & s, const Packet2ul & v) { Packet vt; vt.v2ul = v; s << vt.ul[0] << ", " << vt.ul[1] ; return s; } inline std::ostream & operator <<(std::ostream & s, const Packet2d & v) { Packet vt; vt.v2d = v; s << vt.d[0] << ", " << vt.d[1]; return s; } #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) inline std::ostream & operator <<(std::ostream & s, const Packet4f & v) { Packet vt; vt.v4f = v; s << vt.f[0] << ", " << vt.f[1] << ", " << vt.f[2] << ", " << vt.f[3]; return s; } #endif template<> EIGEN_STRONG_INLINE Packet4i pload(const int* from) { // FIXME: No intrinsic yet EIGEN_DEBUG_ALIGNED_LOAD Packet *vfrom; vfrom = (Packet *) from; return vfrom->v4i; } template<> EIGEN_STRONG_INLINE Packet2d pload(const double* from) { // FIXME: No intrinsic yet EIGEN_DEBUG_ALIGNED_LOAD Packet *vfrom; vfrom = (Packet *) from; return vfrom->v2d; } template<> EIGEN_STRONG_INLINE void pstore(int* to, const Packet4i& from) { // FIXME: No intrinsic yet EIGEN_DEBUG_ALIGNED_STORE Packet *vto; vto = (Packet *) to; vto->v4i = from; } template<> EIGEN_STRONG_INLINE void pstore(double* to, const Packet2d& from) { // FIXME: No intrinsic yet EIGEN_DEBUG_ALIGNED_STORE Packet *vto; vto = (Packet *) to; vto->v2d = from; } template<> EIGEN_STRONG_INLINE Packet4i pset1(const int& from) { return vec_splats(from); } template<> EIGEN_STRONG_INLINE Packet2d pset1(const double& from) { return vec_splats(from); } template<> EIGEN_STRONG_INLINE void pbroadcast4(const int *a, Packet4i& a0, Packet4i& a1, Packet4i& a2, Packet4i& a3) { a3 = pload(a); a0 = vec_splat(a3, 0); a1 = vec_splat(a3, 1); a2 = vec_splat(a3, 2); a3 = vec_splat(a3, 3); } template<> EIGEN_STRONG_INLINE void pbroadcast4(const double *a, Packet2d& a0, Packet2d& a1, Packet2d& a2, Packet2d& a3) { a1 = pload(a); a0 = vec_splat(a1, 0); a1 = vec_splat(a1, 1); a3 = pload(a+2); a2 = vec_splat(a3, 0); a3 = vec_splat(a3, 1); } template<> EIGEN_DEVICE_FUNC inline Packet4i pgather(const int* from, Index stride) { int EIGEN_ALIGN16 ai[4]; ai[0] = from[0*stride]; ai[1] = from[1*stride]; ai[2] = from[2*stride]; ai[3] = from[3*stride]; return pload(ai); } template<> EIGEN_DEVICE_FUNC inline Packet2d pgather(const double* from, Index stride) { double EIGEN_ALIGN16 af[2]; af[0] = from[0*stride]; af[1] = from[1*stride]; return pload(af); } template<> EIGEN_DEVICE_FUNC inline void pscatter(int* to, const Packet4i& from, Index stride) { int EIGEN_ALIGN16 ai[4]; pstore((int *)ai, from); to[0*stride] = ai[0]; to[1*stride] = ai[1]; to[2*stride] = ai[2]; to[3*stride] = ai[3]; } template<> EIGEN_DEVICE_FUNC inline void pscatter(double* to, const Packet2d& from, Index stride) { double EIGEN_ALIGN16 af[2]; pstore(af, from); to[0*stride] = af[0]; to[1*stride] = af[1]; } template<> EIGEN_STRONG_INLINE Packet4i padd(const Packet4i& a, const Packet4i& b) { return (a + b); } template<> EIGEN_STRONG_INLINE Packet2d padd(const Packet2d& a, const Packet2d& b) { return (a + b); } template<> EIGEN_STRONG_INLINE Packet4i psub(const Packet4i& a, const Packet4i& b) { return (a - b); } template<> EIGEN_STRONG_INLINE Packet2d psub(const Packet2d& a, const Packet2d& b) { return (a - b); } template<> EIGEN_STRONG_INLINE Packet4i pmul(const Packet4i& a, const Packet4i& b) { return (a * b); } template<> EIGEN_STRONG_INLINE Packet2d pmul(const Packet2d& a, const Packet2d& b) { return (a * b); } template<> EIGEN_STRONG_INLINE Packet4i pdiv(const Packet4i& a, const Packet4i& b) { return (a / b); } template<> EIGEN_STRONG_INLINE Packet2d pdiv(const Packet2d& a, const Packet2d& b) { return (a / b); } template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return (-a); } template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) { return (-a); } template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; } template<> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return padd(pmul(a, b), c); } template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return vec_madd(a, b, c); } template<> EIGEN_STRONG_INLINE Packet4i plset(const int& a) { return padd(pset1(a), p4i_COUNTDOWN); } template<> EIGEN_STRONG_INLINE Packet2d plset(const double& a) { return padd(pset1(a), p2d_COUNTDOWN); } template<> EIGEN_STRONG_INLINE Packet4i pmin(const Packet4i& a, const Packet4i& b) { return vec_min(a, b); } template<> EIGEN_STRONG_INLINE Packet2d pmin(const Packet2d& a, const Packet2d& b) { return vec_min(a, b); } template<> EIGEN_STRONG_INLINE Packet4i pmax(const Packet4i& a, const Packet4i& b) { return vec_max(a, b); } template<> EIGEN_STRONG_INLINE Packet2d pmax(const Packet2d& a, const Packet2d& b) { return vec_max(a, b); } template<> EIGEN_STRONG_INLINE Packet4i pand(const Packet4i& a, const Packet4i& b) { return vec_and(a, b); } template<> EIGEN_STRONG_INLINE Packet2d pand(const Packet2d& a, const Packet2d& b) { return vec_and(a, b); } template<> EIGEN_STRONG_INLINE Packet4i por(const Packet4i& a, const Packet4i& b) { return vec_or(a, b); } template<> EIGEN_STRONG_INLINE Packet2d por(const Packet2d& a, const Packet2d& b) { return vec_or(a, b); } template<> EIGEN_STRONG_INLINE Packet4i pxor(const Packet4i& a, const Packet4i& b) { return vec_xor(a, b); } template<> EIGEN_STRONG_INLINE Packet2d pxor(const Packet2d& a, const Packet2d& b) { return vec_xor(a, b); } template<> EIGEN_STRONG_INLINE Packet4i pandnot(const Packet4i& a, const Packet4i& b) { return pand(a, vec_nor(b, b)); } template<> EIGEN_STRONG_INLINE Packet2d pandnot(const Packet2d& a, const Packet2d& b) { return vec_and(a, vec_nor(b, b)); } template<> EIGEN_STRONG_INLINE Packet2d pround(const Packet2d& a) { return vec_round(a); } template<> EIGEN_STRONG_INLINE Packet2d pceil(const Packet2d& a) { return vec_ceil(a); } template<> EIGEN_STRONG_INLINE Packet2d pfloor(const Packet2d& a) { return vec_floor(a); } template<> EIGEN_STRONG_INLINE Packet4i ploadu(const int* from) { return pload(from); } template<> EIGEN_STRONG_INLINE Packet2d ploadu(const double* from) { return pload(from); } template<> EIGEN_STRONG_INLINE Packet4i ploaddup(const int* from) { Packet4i p = pload(from); return vec_perm(p, p, p16uc_DUPLICATE32_HI); } template<> EIGEN_STRONG_INLINE Packet2d ploaddup(const double* from) { Packet2d p = pload(from); return vec_perm(p, p, p16uc_PSET64_HI); } template<> EIGEN_STRONG_INLINE void pstoreu(int* to, const Packet4i& from) { pstore(to, from); } template<> EIGEN_STRONG_INLINE void pstoreu(double* to, const Packet2d& from) { pstore(to, from); } template<> EIGEN_STRONG_INLINE void prefetch(const int* addr) { EIGEN_ZVECTOR_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE void prefetch(const double* addr) { EIGEN_ZVECTOR_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE int pfirst(const Packet4i& a) { int EIGEN_ALIGN16 x[4]; pstore(x, a); return x[0]; } template<> EIGEN_STRONG_INLINE double pfirst(const Packet2d& a) { double EIGEN_ALIGN16 x[2]; pstore(x, a); return x[0]; } template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) { return reinterpret_cast(vec_perm(reinterpret_cast(a), reinterpret_cast(a), p16uc_REVERSE32)); } template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) { return reinterpret_cast(vec_perm(reinterpret_cast(a), reinterpret_cast(a), p16uc_REVERSE64)); } template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { return vec_abs(a); } template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { return vec_abs(a); } template<> EIGEN_STRONG_INLINE int predux(const Packet4i& a) { Packet4i b, sum; b = vec_sld(a, a, 8); sum = padd(a, b); b = vec_sld(sum, sum, 4); sum = padd(sum, b); return pfirst(sum); } template<> EIGEN_STRONG_INLINE double predux(const Packet2d& a) { Packet2d b, sum; b = reinterpret_cast(vec_sld(reinterpret_cast(a), reinterpret_cast(a), 8)); sum = padd(a, b); return pfirst(sum); } // Other reduction functions: // mul template<> EIGEN_STRONG_INLINE int predux_mul(const Packet4i& a) { EIGEN_ALIGN16 int aux[4]; pstore(aux, a); return aux[0] * aux[1] * aux[2] * aux[3]; } template<> EIGEN_STRONG_INLINE double predux_mul(const Packet2d& a) { return pfirst(pmul(a, reinterpret_cast(vec_sld(reinterpret_cast(a), reinterpret_cast(a), 8)))); } // min template<> EIGEN_STRONG_INLINE int predux_min(const Packet4i& a) { Packet4i b, res; b = pmin(a, vec_sld(a, a, 8)); res = pmin(b, vec_sld(b, b, 4)); return pfirst(res); } template<> EIGEN_STRONG_INLINE double predux_min(const Packet2d& a) { return pfirst(pmin(a, reinterpret_cast(vec_sld(reinterpret_cast(a), reinterpret_cast(a), 8)))); } // max template<> EIGEN_STRONG_INLINE int predux_max(const Packet4i& a) { Packet4i b, res; b = pmax(a, vec_sld(a, a, 8)); res = pmax(b, vec_sld(b, b, 4)); return pfirst(res); } // max template<> EIGEN_STRONG_INLINE double predux_max(const Packet2d& a) { return pfirst(pmax(a, reinterpret_cast(vec_sld(reinterpret_cast(a), reinterpret_cast(a), 8)))); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { Packet4i t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]); Packet4i t1 = vec_mergel(kernel.packet[0], kernel.packet[2]); Packet4i t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]); Packet4i t3 = vec_mergel(kernel.packet[1], kernel.packet[3]); kernel.packet[0] = vec_mergeh(t0, t2); kernel.packet[1] = vec_mergel(t0, t2); kernel.packet[2] = vec_mergeh(t1, t3); kernel.packet[3] = vec_mergel(t1, t3); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { Packet2d t0 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_HI); Packet2d t1 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_LO); kernel.packet[0] = t0; kernel.packet[1] = t1; } template<> EIGEN_STRONG_INLINE Packet4i pblend(const Selector<4>& ifPacket, const Packet4i& thenPacket, const Packet4i& elsePacket) { Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] }; Packet4ui mask = vec_cmpeq(select, reinterpret_cast(p4i_ONE)); return vec_sel(elsePacket, thenPacket, mask); } template<> EIGEN_STRONG_INLINE Packet2d pblend(const Selector<2>& ifPacket, const Packet2d& thenPacket, const Packet2d& elsePacket) { Packet2ul select = { ifPacket.select[0], ifPacket.select[1] }; Packet2ul mask = vec_cmpeq(select, reinterpret_cast(p2l_ONE)); return vec_sel(elsePacket, thenPacket, mask); } /* z13 has no vector float support so we emulate that with double z14 has proper vector float support. */ #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ < 12) /* Helper function to simulate a vec_splat_packet4f */ template EIGEN_STRONG_INLINE Packet4f vec_splat_packet4f(const Packet4f& from) { Packet4f splat; switch (element) { case 0: splat.v4f[0] = vec_splat(from.v4f[0], 0); splat.v4f[1] = splat.v4f[0]; break; case 1: splat.v4f[0] = vec_splat(from.v4f[0], 1); splat.v4f[1] = splat.v4f[0]; break; case 2: splat.v4f[0] = vec_splat(from.v4f[1], 0); splat.v4f[1] = splat.v4f[0]; break; case 3: splat.v4f[0] = vec_splat(from.v4f[1], 1); splat.v4f[1] = splat.v4f[0]; break; } return splat; } template<> EIGEN_STRONG_INLINE Packet4f pload(const float* from) { // FIXME: No intrinsic yet EIGEN_DEBUG_ALIGNED_LOAD Packet4f vfrom; vfrom.v4f[0] = vec_ld2f(&from[0]); vfrom.v4f[1] = vec_ld2f(&from[2]); return vfrom; } template<> EIGEN_STRONG_INLINE void pstore(float* to, const Packet4f& from) { // FIXME: No intrinsic yet EIGEN_DEBUG_ALIGNED_STORE vec_st2f(from.v4f[0], &to[0]); vec_st2f(from.v4f[1], &to[2]); } template<> EIGEN_STRONG_INLINE Packet4f pset1(const float& from) { Packet4f to; to.v4f[0] = pset1(static_cast(from)); to.v4f[1] = to.v4f[0]; return to; } template<> EIGEN_STRONG_INLINE void pbroadcast4(const float *a, Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) { a3 = pload(a); a0 = vec_splat_packet4f<0>(a3); a1 = vec_splat_packet4f<1>(a3); a2 = vec_splat_packet4f<2>(a3); a3 = vec_splat_packet4f<3>(a3); } template<> EIGEN_DEVICE_FUNC inline Packet4f pgather(const float* from, Index stride) { float EIGEN_ALIGN16 ai[4]; ai[0] = from[0*stride]; ai[1] = from[1*stride]; ai[2] = from[2*stride]; ai[3] = from[3*stride]; return pload(ai); } template<> EIGEN_DEVICE_FUNC inline void pscatter(float* to, const Packet4f& from, Index stride) { float EIGEN_ALIGN16 ai[4]; pstore((float *)ai, from); to[0*stride] = ai[0]; to[1*stride] = ai[1]; to[2*stride] = ai[2]; to[3*stride] = ai[3]; } template<> EIGEN_STRONG_INLINE Packet4f padd(const Packet4f& a, const Packet4f& b) { Packet4f c; c.v4f[0] = a.v4f[0] + b.v4f[0]; c.v4f[1] = a.v4f[1] + b.v4f[1]; return c; } template<> EIGEN_STRONG_INLINE Packet4f psub(const Packet4f& a, const Packet4f& b) { Packet4f c; c.v4f[0] = a.v4f[0] - b.v4f[0]; c.v4f[1] = a.v4f[1] - b.v4f[1]; return c; } template<> EIGEN_STRONG_INLINE Packet4f pmul(const Packet4f& a, const Packet4f& b) { Packet4f c; c.v4f[0] = a.v4f[0] * b.v4f[0]; c.v4f[1] = a.v4f[1] * b.v4f[1]; return c; } template<> EIGEN_STRONG_INLINE Packet4f pdiv(const Packet4f& a, const Packet4f& b) { Packet4f c; c.v4f[0] = a.v4f[0] / b.v4f[0]; c.v4f[1] = a.v4f[1] / b.v4f[1]; return c; } template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { Packet4f c; c.v4f[0] = -a.v4f[0]; c.v4f[1] = -a.v4f[1]; return c; } template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { Packet4f res; res.v4f[0] = vec_madd(a.v4f[0], b.v4f[0], c.v4f[0]); res.v4f[1] = vec_madd(a.v4f[1], b.v4f[1], c.v4f[1]); return res; } template<> EIGEN_STRONG_INLINE Packet4f pmin(const Packet4f& a, const Packet4f& b) { Packet4f res; res.v4f[0] = pmin(a.v4f[0], b.v4f[0]); res.v4f[1] = pmin(a.v4f[1], b.v4f[1]); return res; } template<> EIGEN_STRONG_INLINE Packet4f pmax(const Packet4f& a, const Packet4f& b) { Packet4f res; res.v4f[0] = pmax(a.v4f[0], b.v4f[0]); res.v4f[1] = pmax(a.v4f[1], b.v4f[1]); return res; } template<> EIGEN_STRONG_INLINE Packet4f pand(const Packet4f& a, const Packet4f& b) { Packet4f res; res.v4f[0] = pand(a.v4f[0], b.v4f[0]); res.v4f[1] = pand(a.v4f[1], b.v4f[1]); return res; } template<> EIGEN_STRONG_INLINE Packet4f por(const Packet4f& a, const Packet4f& b) { Packet4f res; res.v4f[0] = por(a.v4f[0], b.v4f[0]); res.v4f[1] = por(a.v4f[1], b.v4f[1]); return res; } template<> EIGEN_STRONG_INLINE Packet4f pxor(const Packet4f& a, const Packet4f& b) { Packet4f res; res.v4f[0] = pxor(a.v4f[0], b.v4f[0]); res.v4f[1] = pxor(a.v4f[1], b.v4f[1]); return res; } template<> EIGEN_STRONG_INLINE Packet4f pandnot(const Packet4f& a, const Packet4f& b) { Packet4f res; res.v4f[0] = pandnot(a.v4f[0], b.v4f[0]); res.v4f[1] = pandnot(a.v4f[1], b.v4f[1]); return res; } template<> EIGEN_STRONG_INLINE Packet4f pround(const Packet4f& a) { Packet4f res; res.v4f[0] = vec_round(a.v4f[0]); res.v4f[1] = vec_round(a.v4f[1]); return res; } template<> EIGEN_STRONG_INLINE Packet4f pceil(const Packet4f& a) { Packet4f res; res.v4f[0] = vec_ceil(a.v4f[0]); res.v4f[1] = vec_ceil(a.v4f[1]); return res; } template<> EIGEN_STRONG_INLINE Packet4f pfloor(const Packet4f& a) { Packet4f res; res.v4f[0] = vec_floor(a.v4f[0]); res.v4f[1] = vec_floor(a.v4f[1]); return res; } template<> EIGEN_STRONG_INLINE Packet4f ploaddup(const float* from) { Packet4f p = pload(from); p.v4f[1] = vec_splat(p.v4f[0], 1); p.v4f[0] = vec_splat(p.v4f[0], 0); return p; } template<> EIGEN_STRONG_INLINE float pfirst(const Packet4f& a) { float EIGEN_ALIGN16 x[2]; vec_st2f(a.v4f[0], &x[0]); return x[0]; } template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) { Packet4f rev; rev.v4f[0] = preverse(a.v4f[1]); rev.v4f[1] = preverse(a.v4f[0]); return rev; } template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { Packet4f res; res.v4f[0] = pabs(a.v4f[0]); res.v4f[1] = pabs(a.v4f[1]); return res; } template<> EIGEN_STRONG_INLINE float predux(const Packet4f& a) { Packet2d sum; sum = padd(a.v4f[0], a.v4f[1]); double first = predux(sum); return static_cast(first); } template<> EIGEN_STRONG_INLINE float predux_mul(const Packet4f& a) { // Return predux_mul of the subvectors product return static_cast(pfirst(predux_mul(pmul(a.v4f[0], a.v4f[1])))); } template<> EIGEN_STRONG_INLINE float predux_min(const Packet4f& a) { Packet2d b, res; b = pmin(a.v4f[0], a.v4f[1]); res = pmin(b, reinterpret_cast(vec_sld(reinterpret_cast(b), reinterpret_cast(b), 8))); return static_cast(pfirst(res)); } template<> EIGEN_STRONG_INLINE float predux_max(const Packet4f& a) { Packet2d b, res; b = pmax(a.v4f[0], a.v4f[1]); res = pmax(b, reinterpret_cast(vec_sld(reinterpret_cast(b), reinterpret_cast(b), 8))); return static_cast(pfirst(res)); } /* Split the Packet4f PacketBlock into 4 Packet2d PacketBlocks and transpose each one */ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { PacketBlock t0,t1,t2,t3; // copy top-left 2x2 Packet2d block t0.packet[0] = kernel.packet[0].v4f[0]; t0.packet[1] = kernel.packet[1].v4f[0]; // copy top-right 2x2 Packet2d block t1.packet[0] = kernel.packet[0].v4f[1]; t1.packet[1] = kernel.packet[1].v4f[1]; // copy bottom-left 2x2 Packet2d block t2.packet[0] = kernel.packet[2].v4f[0]; t2.packet[1] = kernel.packet[3].v4f[0]; // copy bottom-right 2x2 Packet2d block t3.packet[0] = kernel.packet[2].v4f[1]; t3.packet[1] = kernel.packet[3].v4f[1]; // Transpose all 2x2 blocks ptranspose(t0); ptranspose(t1); ptranspose(t2); ptranspose(t3); // Copy back transposed blocks, but exchange t1 and t2 due to transposition kernel.packet[0].v4f[0] = t0.packet[0]; kernel.packet[0].v4f[1] = t2.packet[0]; kernel.packet[1].v4f[0] = t0.packet[1]; kernel.packet[1].v4f[1] = t2.packet[1]; kernel.packet[2].v4f[0] = t1.packet[0]; kernel.packet[2].v4f[1] = t3.packet[0]; kernel.packet[3].v4f[0] = t1.packet[1]; kernel.packet[3].v4f[1] = t3.packet[1]; } template<> EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket, const Packet4f& elsePacket) { Packet2ul select_hi = { ifPacket.select[0], ifPacket.select[1] }; Packet2ul select_lo = { ifPacket.select[2], ifPacket.select[3] }; Packet2ul mask_hi = vec_cmpeq(select_hi, reinterpret_cast(p2l_ONE)); Packet2ul mask_lo = vec_cmpeq(select_lo, reinterpret_cast(p2l_ONE)); Packet4f result; result.v4f[0] = vec_sel(elsePacket.v4f[0], thenPacket.v4f[0], mask_hi); result.v4f[1] = vec_sel(elsePacket.v4f[1], thenPacket.v4f[1], mask_lo); return result; } template<> Packet4f EIGEN_STRONG_INLINE pcmp_le(const Packet4f& a, const Packet4f& b) { Packet4f res; res.v4f[0] = pcmp_le(a.v4f[0], b.v4f[0]); res.v4f[1] = pcmp_le(a.v4f[1], b.v4f[1]); return res; } template<> Packet4f EIGEN_STRONG_INLINE pcmp_lt(const Packet4f& a, const Packet4f& b) { Packet4f res; res.v4f[0] = pcmp_lt(a.v4f[0], b.v4f[0]); res.v4f[1] = pcmp_lt(a.v4f[1], b.v4f[1]); return res; } template<> Packet4f EIGEN_STRONG_INLINE pcmp_eq(const Packet4f& a, const Packet4f& b) { Packet4f res; res.v4f[0] = pcmp_eq(a.v4f[0], b.v4f[0]); res.v4f[1] = pcmp_eq(a.v4f[1], b.v4f[1]); return res; } #else template<> EIGEN_STRONG_INLINE Packet4f pload(const float* from) { // FIXME: No intrinsic yet EIGEN_DEBUG_ALIGNED_LOAD Packet *vfrom; vfrom = (Packet *) from; return vfrom->v4f; } template<> EIGEN_STRONG_INLINE void pstore(float* to, const Packet4f& from) { // FIXME: No intrinsic yet EIGEN_DEBUG_ALIGNED_STORE Packet *vto; vto = (Packet *) to; vto->v4f = from; } template<> EIGEN_STRONG_INLINE Packet4f pset1(const float& from) { return vec_splats(from); } template<> EIGEN_STRONG_INLINE void pbroadcast4(const float *a, Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) { a3 = pload(a); a0 = vec_splat(a3, 0); a1 = vec_splat(a3, 1); a2 = vec_splat(a3, 2); a3 = vec_splat(a3, 3); } template<> EIGEN_DEVICE_FUNC inline Packet4f pgather(const float* from, Index stride) { float EIGEN_ALIGN16 af[4]; af[0] = from[0*stride]; af[1] = from[1*stride]; af[2] = from[2*stride]; af[3] = from[3*stride]; return pload(af); } template<> EIGEN_DEVICE_FUNC inline void pscatter(float* to, const Packet4f& from, Index stride) { float EIGEN_ALIGN16 af[4]; pstore((float*)af, from); to[0*stride] = af[0]; to[1*stride] = af[1]; to[2*stride] = af[2]; to[3*stride] = af[3]; } template<> EIGEN_STRONG_INLINE Packet4f padd(const Packet4f& a, const Packet4f& b) { return (a + b); } template<> EIGEN_STRONG_INLINE Packet4f psub(const Packet4f& a, const Packet4f& b) { return (a - b); } template<> EIGEN_STRONG_INLINE Packet4f pmul(const Packet4f& a, const Packet4f& b) { return (a * b); } template<> EIGEN_STRONG_INLINE Packet4f pdiv(const Packet4f& a, const Packet4f& b) { return (a / b); } template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { return (-a); } template<> EIGEN_STRONG_INLINE Packet4f pconj (const Packet4f& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4f pmadd (const Packet4f& a, const Packet4f& b, const Packet4f& c) { return vec_madd(a, b, c); } template<> EIGEN_STRONG_INLINE Packet4f pmin (const Packet4f& a, const Packet4f& b) { return vec_min(a, b); } template<> EIGEN_STRONG_INLINE Packet4f pmax (const Packet4f& a, const Packet4f& b) { return vec_max(a, b); } template<> EIGEN_STRONG_INLINE Packet4f pand (const Packet4f& a, const Packet4f& b) { return vec_and(a, b); } template<> EIGEN_STRONG_INLINE Packet4f por (const Packet4f& a, const Packet4f& b) { return vec_or(a, b); } template<> EIGEN_STRONG_INLINE Packet4f pxor (const Packet4f& a, const Packet4f& b) { return vec_xor(a, b); } template<> EIGEN_STRONG_INLINE Packet4f pandnot(const Packet4f& a, const Packet4f& b) { return vec_and(a, vec_nor(b, b)); } template<> EIGEN_STRONG_INLINE Packet4f pround (const Packet4f& a) { return vec_round(a); } template<> EIGEN_STRONG_INLINE Packet4f pceil (const Packet4f& a) { return vec_ceil(a); } template<> EIGEN_STRONG_INLINE Packet4f pfloor (const Packet4f& a) { return vec_floor(a); } template<> EIGEN_STRONG_INLINE Packet4f pabs (const Packet4f& a) { return vec_abs(a); } template<> EIGEN_STRONG_INLINE float pfirst(const Packet4f& a) { float EIGEN_ALIGN16 x[4]; pstore(x, a); return x[0]; } template<> EIGEN_STRONG_INLINE Packet4f ploaddup(const float* from) { Packet4f p = pload(from); return vec_perm(p, p, p16uc_DUPLICATE32_HI); } template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) { return reinterpret_cast(vec_perm(reinterpret_cast(a), reinterpret_cast(a), p16uc_REVERSE32)); } template<> EIGEN_STRONG_INLINE float predux(const Packet4f& a) { Packet4f b, sum; b = vec_sld(a, a, 8); sum = padd(a, b); b = vec_sld(sum, sum, 4); sum = padd(sum, b); return pfirst(sum); } // Other reduction functions: // mul template<> EIGEN_STRONG_INLINE float predux_mul(const Packet4f& a) { Packet4f prod; prod = pmul(a, vec_sld(a, a, 8)); return pfirst(pmul(prod, vec_sld(prod, prod, 4))); } // min template<> EIGEN_STRONG_INLINE float predux_min(const Packet4f& a) { Packet4f b, res; b = pmin(a, vec_sld(a, a, 8)); res = pmin(b, vec_sld(b, b, 4)); return pfirst(res); } // max template<> EIGEN_STRONG_INLINE float predux_max(const Packet4f& a) { Packet4f b, res; b = pmax(a, vec_sld(a, a, 8)); res = pmax(b, vec_sld(b, b, 4)); return pfirst(res); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { Packet4f t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]); Packet4f t1 = vec_mergel(kernel.packet[0], kernel.packet[2]); Packet4f t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]); Packet4f t3 = vec_mergel(kernel.packet[1], kernel.packet[3]); kernel.packet[0] = vec_mergeh(t0, t2); kernel.packet[1] = vec_mergel(t0, t2); kernel.packet[2] = vec_mergeh(t1, t3); kernel.packet[3] = vec_mergel(t1, t3); } template<> EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket, const Packet4f& elsePacket) { Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] }; Packet4ui mask = vec_cmpeq(select, reinterpret_cast(p4i_ONE)); return vec_sel(elsePacket, thenPacket, mask); } #endif template<> EIGEN_STRONG_INLINE void prefetch(const float* addr) { EIGEN_ZVECTOR_PREFETCH(addr); } template<> EIGEN_STRONG_INLINE Packet4f ploadu (const float* from) { return pload(from); } template<> EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet4f& from) { pstore(to, from); } template<> EIGEN_STRONG_INLINE Packet4f plset (const float& a) { return padd(pset1(a), p4f_COUNTDOWN); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_PACKET_MATH_ZVECTOR_H RcppEigen/inst/include/Eigen/src/Core/arch/ZVector/MathFunctions.h0000644000176200001440000001753014567757725024537 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2007 Julien Pommier // Copyright (C) 2009 Gael Guennebaud // Copyright (C) 2016 Konstantinos Margaritis // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. /* The sin, cos, exp, and log functions of this file come from * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/ */ #ifndef EIGEN_MATH_FUNCTIONS_ALTIVEC_H #define EIGEN_MATH_FUNCTIONS_ALTIVEC_H namespace Eigen { namespace internal { #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) static _EIGEN_DECLARE_CONST_Packet4f(1 , 1.0f); static _EIGEN_DECLARE_CONST_Packet4f(half, 0.5f); static _EIGEN_DECLARE_CONST_Packet4i(0x7f, 0x7f); static _EIGEN_DECLARE_CONST_Packet4i(23, 23); static _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(inv_mant_mask, ~0x7f800000); /* the smallest non denormalized float number */ static _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(min_norm_pos, 0x00800000); static _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(minus_inf, 0xff800000); // -1.f/0.f static _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(minus_nan, 0xffffffff); /* natural logarithm computed for 4 simultaneous float return NaN for x <= 0 */ static _EIGEN_DECLARE_CONST_Packet4f(cephes_SQRTHF, 0.707106781186547524f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p0, 7.0376836292E-2f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p1, - 1.1514610310E-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p2, 1.1676998740E-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p3, - 1.2420140846E-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p4, + 1.4249322787E-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p5, - 1.6668057665E-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p6, + 2.0000714765E-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p7, - 2.4999993993E-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_p8, + 3.3333331174E-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_q1, -2.12194440e-4f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_log_q2, 0.693359375f); static _EIGEN_DECLARE_CONST_Packet4f(exp_hi, 88.3762626647950f); static _EIGEN_DECLARE_CONST_Packet4f(exp_lo, -88.3762626647949f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_LOG2EF, 1.44269504088896341f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C1, 0.693359375f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C2, -2.12194440e-4f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p0, 1.9875691500E-4f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p1, 1.3981999507E-3f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p2, 8.3334519073E-3f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p3, 4.1665795894E-2f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p4, 1.6666665459E-1f); static _EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p5, 5.0000001201E-1f); #endif static _EIGEN_DECLARE_CONST_Packet2d(1 , 1.0); static _EIGEN_DECLARE_CONST_Packet2d(2 , 2.0); static _EIGEN_DECLARE_CONST_Packet2d(half, 0.5); static _EIGEN_DECLARE_CONST_Packet2d(exp_hi, 709.437); static _EIGEN_DECLARE_CONST_Packet2d(exp_lo, -709.436139303); static _EIGEN_DECLARE_CONST_Packet2d(cephes_LOG2EF, 1.4426950408889634073599); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p0, 1.26177193074810590878e-4); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p1, 3.02994407707441961300e-2); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p2, 9.99999999999999999910e-1); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q0, 3.00198505138664455042e-6); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q1, 2.52448340349684104192e-3); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q2, 2.27265548208155028766e-1); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q3, 2.00000000000000000009e0); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C1, 0.693145751953125); static _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C2, 1.42860682030941723212e-6); template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d pexp(const Packet2d& _x) { Packet2d x = _x; Packet2d tmp, fx; Packet2l emm0; // clamp x x = pmax(pmin(x, p2d_exp_hi), p2d_exp_lo); /* express exp(x) as exp(g + n*log(2)) */ fx = pmadd(p2d_cephes_LOG2EF, x, p2d_half); fx = vec_floor(fx); tmp = pmul(fx, p2d_cephes_exp_C1); Packet2d z = pmul(fx, p2d_cephes_exp_C2); x = psub(x, tmp); x = psub(x, z); Packet2d x2 = pmul(x,x); Packet2d px = p2d_cephes_exp_p0; px = pmadd(px, x2, p2d_cephes_exp_p1); px = pmadd(px, x2, p2d_cephes_exp_p2); px = pmul (px, x); Packet2d qx = p2d_cephes_exp_q0; qx = pmadd(qx, x2, p2d_cephes_exp_q1); qx = pmadd(qx, x2, p2d_cephes_exp_q2); qx = pmadd(qx, x2, p2d_cephes_exp_q3); x = pdiv(px,psub(qx,px)); x = pmadd(p2d_2,x,p2d_1); // build 2^n emm0 = vec_ctsl(fx, 0); static const Packet2l p2l_1023 = { 1023, 1023 }; static const Packet2ul p2ul_52 = { 52, 52 }; emm0 = emm0 + p2l_1023; emm0 = emm0 << reinterpret_cast(p2ul_52); // Altivec's max & min operators just drop silent NaNs. Check NaNs in // inputs and return them unmodified. Packet2ul isnumber_mask = reinterpret_cast(vec_cmpeq(_x, _x)); return vec_sel(_x, pmax(pmul(x, reinterpret_cast(emm0)), _x), isnumber_mask); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pexp(const Packet4f& _x) { #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) Packet4f x = _x; Packet4f tmp, fx; Packet4i emm0; // clamp x x = pmax(pmin(x, p4f_exp_hi), p4f_exp_lo); // express exp(x) as exp(g + n*log(2)) fx = pmadd(x, p4f_cephes_LOG2EF, p4f_half); fx = pfloor(fx); tmp = pmul(fx, p4f_cephes_exp_C1); Packet4f z = pmul(fx, p4f_cephes_exp_C2); x = psub(x, tmp); x = psub(x, z); z = pmul(x,x); Packet4f y = p4f_cephes_exp_p0; y = pmadd(y, x, p4f_cephes_exp_p1); y = pmadd(y, x, p4f_cephes_exp_p2); y = pmadd(y, x, p4f_cephes_exp_p3); y = pmadd(y, x, p4f_cephes_exp_p4); y = pmadd(y, x, p4f_cephes_exp_p5); y = pmadd(y, z, x); y = padd(y, p4f_1); // build 2^n emm0 = (Packet4i){ (int)fx[0], (int)fx[1], (int)fx[2], (int)fx[3] }; emm0 = emm0 + p4i_0x7f; emm0 = emm0 << reinterpret_cast(p4i_23); return pmax(pmul(y, reinterpret_cast(emm0)), _x); #else Packet4f res; res.v4f[0] = pexp(_x.v4f[0]); res.v4f[1] = pexp(_x.v4f[1]); return res; #endif } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d psqrt(const Packet2d& x) { return vec_sqrt(x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f psqrt(const Packet4f& x) { Packet4f res; #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) res = vec_sqrt(x); #else res.v4f[0] = psqrt(x.v4f[0]); res.v4f[1] = psqrt(x.v4f[1]); #endif return res; } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d prsqrt(const Packet2d& x) { return pset1(1.0) / psqrt(x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f prsqrt(const Packet4f& x) { Packet4f res; #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12) res = pset1(1.0) / psqrt(x); #else res.v4f[0] = prsqrt(x.v4f[0]); res.v4f[1] = prsqrt(x.v4f[1]); #endif return res; } // Hyperbolic Tangent function. template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f ptanh(const Packet4f& x) { return internal::generic_fast_tanh_float(x); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_MATH_FUNCTIONS_ALTIVEC_H RcppEigen/inst/include/Eigen/src/Core/arch/AVX/0000755000176200001440000000000014567757725020640 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/arch/AVX/TypeCasting.h0000644000176200001440000000500414567757725023242 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2015 Benoit Steiner // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_TYPE_CASTING_AVX_H #define EIGEN_TYPE_CASTING_AVX_H namespace Eigen { namespace internal { // For now we use SSE to handle integers, so we can't use AVX instructions to cast // from int to float template <> struct type_casting_traits { enum { VectorizedCast = 0, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> struct type_casting_traits { enum { VectorizedCast = 0, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; #ifndef EIGEN_VECTORIZE_AVX512 template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; #endif // EIGEN_VECTORIZE_AVX512 template<> EIGEN_STRONG_INLINE Packet8i pcast(const Packet8f& a) { return _mm256_cvttps_epi32(a); } template<> EIGEN_STRONG_INLINE Packet8f pcast(const Packet8i& a) { return _mm256_cvtepi32_ps(a); } template<> EIGEN_STRONG_INLINE Packet8i preinterpret(const Packet8f& a) { return _mm256_castps_si256(a); } template<> EIGEN_STRONG_INLINE Packet8f preinterpret(const Packet8i& a) { return _mm256_castsi256_ps(a); } template<> EIGEN_STRONG_INLINE Packet8f pcast(const Packet8h& a) { return half2float(a); } template<> EIGEN_STRONG_INLINE Packet8f pcast(const Packet8bf& a) { return Bf16ToF32(a); } template<> EIGEN_STRONG_INLINE Packet8h pcast(const Packet8f& a) { return float2half(a); } template<> EIGEN_STRONG_INLINE Packet8bf pcast(const Packet8f& a) { return F32ToBf16(a); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_TYPE_CASTING_AVX_H RcppEigen/inst/include/Eigen/src/Core/arch/AVX/Complex.h0000644000176200001440000003556714567757725022440 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2014 Benoit Steiner (benoit.steiner.goog@gmail.com) // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_COMPLEX_AVX_H #define EIGEN_COMPLEX_AVX_H namespace Eigen { namespace internal { //---------- float ---------- struct Packet4cf { EIGEN_STRONG_INLINE Packet4cf() {} EIGEN_STRONG_INLINE explicit Packet4cf(const __m256& a) : v(a) {} __m256 v; }; #ifndef EIGEN_VECTORIZE_AVX512 template<> struct packet_traits > : default_packet_traits { typedef Packet4cf type; typedef Packet2cf half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 4, HasHalfPacket = 1, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, HasSqrt = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, HasMax = 0, HasSetLinear = 0 }; }; #endif template<> struct unpacket_traits { typedef std::complex type; typedef Packet2cf half; typedef Packet8f as_real; enum { size=4, alignment=Aligned32, vectorizable=true, masked_load_available=false, masked_store_available=false }; }; template<> EIGEN_STRONG_INLINE Packet4cf padd(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_add_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet4cf psub(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_sub_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet4cf pnegate(const Packet4cf& a) { return Packet4cf(pnegate(a.v)); } template<> EIGEN_STRONG_INLINE Packet4cf pconj(const Packet4cf& a) { const __m256 mask = _mm256_castsi256_ps(_mm256_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000)); return Packet4cf(_mm256_xor_ps(a.v,mask)); } template<> EIGEN_STRONG_INLINE Packet4cf pmul(const Packet4cf& a, const Packet4cf& b) { __m256 tmp1 = _mm256_mul_ps(_mm256_moveldup_ps(a.v), b.v); __m256 tmp2 = _mm256_mul_ps(_mm256_movehdup_ps(a.v), _mm256_permute_ps(b.v, _MM_SHUFFLE(2,3,0,1))); __m256 result = _mm256_addsub_ps(tmp1, tmp2); return Packet4cf(result); } template <> EIGEN_STRONG_INLINE Packet4cf pcmp_eq(const Packet4cf& a, const Packet4cf& b) { __m256 eq = _mm256_cmp_ps(a.v, b.v, _CMP_EQ_OQ); return Packet4cf(_mm256_and_ps(eq, _mm256_permute_ps(eq, 0xb1))); } template<> EIGEN_STRONG_INLINE Packet4cf ptrue(const Packet4cf& a) { return Packet4cf(ptrue(Packet8f(a.v))); } template<> EIGEN_STRONG_INLINE Packet4cf pand (const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_and_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet4cf por (const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_or_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet4cf pxor (const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_xor_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet4cf pandnot(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_andnot_ps(b.v,a.v)); } template<> EIGEN_STRONG_INLINE Packet4cf pload (const std::complex* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet4cf(pload(&numext::real_ref(*from))); } template<> EIGEN_STRONG_INLINE Packet4cf ploadu(const std::complex* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet4cf(ploadu(&numext::real_ref(*from))); } template<> EIGEN_STRONG_INLINE Packet4cf pset1(const std::complex& from) { return Packet4cf(_mm256_castpd_ps(_mm256_broadcast_sd((const double*)(const void*)&from))); } template<> EIGEN_STRONG_INLINE Packet4cf ploaddup(const std::complex* from) { // FIXME The following might be optimized using _mm256_movedup_pd Packet2cf a = ploaddup(from); Packet2cf b = ploaddup(from+1); return Packet4cf(_mm256_insertf128_ps(_mm256_castps128_ps256(a.v), b.v, 1)); } template<> EIGEN_STRONG_INLINE void pstore >(std::complex* to, const Packet4cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), from.v); } template<> EIGEN_STRONG_INLINE void pstoreu >(std::complex* to, const Packet4cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), from.v); } template<> EIGEN_DEVICE_FUNC inline Packet4cf pgather, Packet4cf>(const std::complex* from, Index stride) { return Packet4cf(_mm256_set_ps(std::imag(from[3*stride]), std::real(from[3*stride]), std::imag(from[2*stride]), std::real(from[2*stride]), std::imag(from[1*stride]), std::real(from[1*stride]), std::imag(from[0*stride]), std::real(from[0*stride]))); } template<> EIGEN_DEVICE_FUNC inline void pscatter, Packet4cf>(std::complex* to, const Packet4cf& from, Index stride) { __m128 low = _mm256_extractf128_ps(from.v, 0); to[stride*0] = std::complex(_mm_cvtss_f32(_mm_shuffle_ps(low, low, 0)), _mm_cvtss_f32(_mm_shuffle_ps(low, low, 1))); to[stride*1] = std::complex(_mm_cvtss_f32(_mm_shuffle_ps(low, low, 2)), _mm_cvtss_f32(_mm_shuffle_ps(low, low, 3))); __m128 high = _mm256_extractf128_ps(from.v, 1); to[stride*2] = std::complex(_mm_cvtss_f32(_mm_shuffle_ps(high, high, 0)), _mm_cvtss_f32(_mm_shuffle_ps(high, high, 1))); to[stride*3] = std::complex(_mm_cvtss_f32(_mm_shuffle_ps(high, high, 2)), _mm_cvtss_f32(_mm_shuffle_ps(high, high, 3))); } template<> EIGEN_STRONG_INLINE std::complex pfirst(const Packet4cf& a) { return pfirst(Packet2cf(_mm256_castps256_ps128(a.v))); } template<> EIGEN_STRONG_INLINE Packet4cf preverse(const Packet4cf& a) { __m128 low = _mm256_extractf128_ps(a.v, 0); __m128 high = _mm256_extractf128_ps(a.v, 1); __m128d lowd = _mm_castps_pd(low); __m128d highd = _mm_castps_pd(high); low = _mm_castpd_ps(_mm_shuffle_pd(lowd,lowd,0x1)); high = _mm_castpd_ps(_mm_shuffle_pd(highd,highd,0x1)); __m256 result = _mm256_setzero_ps(); result = _mm256_insertf128_ps(result, low, 1); result = _mm256_insertf128_ps(result, high, 0); return Packet4cf(result); } template<> EIGEN_STRONG_INLINE std::complex predux(const Packet4cf& a) { return predux(padd(Packet2cf(_mm256_extractf128_ps(a.v,0)), Packet2cf(_mm256_extractf128_ps(a.v,1)))); } template<> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet4cf& a) { return predux_mul(pmul(Packet2cf(_mm256_extractf128_ps(a.v, 0)), Packet2cf(_mm256_extractf128_ps(a.v, 1)))); } EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet4cf,Packet8f) template<> EIGEN_STRONG_INLINE Packet4cf pdiv(const Packet4cf& a, const Packet4cf& b) { Packet4cf num = pmul(a, pconj(b)); __m256 tmp = _mm256_mul_ps(b.v, b.v); __m256 tmp2 = _mm256_shuffle_ps(tmp,tmp,0xB1); __m256 denom = _mm256_add_ps(tmp, tmp2); return Packet4cf(_mm256_div_ps(num.v, denom)); } template<> EIGEN_STRONG_INLINE Packet4cf pcplxflip(const Packet4cf& x) { return Packet4cf(_mm256_shuffle_ps(x.v, x.v, _MM_SHUFFLE(2, 3, 0 ,1))); } //---------- double ---------- struct Packet2cd { EIGEN_STRONG_INLINE Packet2cd() {} EIGEN_STRONG_INLINE explicit Packet2cd(const __m256d& a) : v(a) {} __m256d v; }; #ifndef EIGEN_VECTORIZE_AVX512 template<> struct packet_traits > : default_packet_traits { typedef Packet2cd type; typedef Packet1cd half; enum { Vectorizable = 1, AlignedOnScalar = 0, size = 2, HasHalfPacket = 1, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, HasSqrt = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, HasMax = 0, HasSetLinear = 0 }; }; #endif template<> struct unpacket_traits { typedef std::complex type; typedef Packet1cd half; typedef Packet4d as_real; enum { size=2, alignment=Aligned32, vectorizable=true, masked_load_available=false, masked_store_available=false }; }; template<> EIGEN_STRONG_INLINE Packet2cd padd(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_add_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cd psub(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_sub_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cd pnegate(const Packet2cd& a) { return Packet2cd(pnegate(a.v)); } template<> EIGEN_STRONG_INLINE Packet2cd pconj(const Packet2cd& a) { const __m256d mask = _mm256_castsi256_pd(_mm256_set_epi32(0x80000000,0x0,0x0,0x0,0x80000000,0x0,0x0,0x0)); return Packet2cd(_mm256_xor_pd(a.v,mask)); } template<> EIGEN_STRONG_INLINE Packet2cd pmul(const Packet2cd& a, const Packet2cd& b) { __m256d tmp1 = _mm256_shuffle_pd(a.v,a.v,0x0); __m256d even = _mm256_mul_pd(tmp1, b.v); __m256d tmp2 = _mm256_shuffle_pd(a.v,a.v,0xF); __m256d tmp3 = _mm256_shuffle_pd(b.v,b.v,0x5); __m256d odd = _mm256_mul_pd(tmp2, tmp3); return Packet2cd(_mm256_addsub_pd(even, odd)); } template <> EIGEN_STRONG_INLINE Packet2cd pcmp_eq(const Packet2cd& a, const Packet2cd& b) { __m256d eq = _mm256_cmp_pd(a.v, b.v, _CMP_EQ_OQ); return Packet2cd(pand(eq, _mm256_permute_pd(eq, 0x5))); } template<> EIGEN_STRONG_INLINE Packet2cd ptrue(const Packet2cd& a) { return Packet2cd(ptrue(Packet4d(a.v))); } template<> EIGEN_STRONG_INLINE Packet2cd pand (const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_and_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cd por (const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_or_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cd pxor (const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_xor_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cd pandnot(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_andnot_pd(b.v,a.v)); } template<> EIGEN_STRONG_INLINE Packet2cd pload (const std::complex* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet2cd(pload((const double*)from)); } template<> EIGEN_STRONG_INLINE Packet2cd ploadu(const std::complex* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cd(ploadu((const double*)from)); } template<> EIGEN_STRONG_INLINE Packet2cd pset1(const std::complex& from) { // in case casting to a __m128d* is really not safe, then we can still fallback to this version: (much slower though) // return Packet2cd(_mm256_loadu2_m128d((const double*)&from,(const double*)&from)); return Packet2cd(_mm256_broadcast_pd((const __m128d*)(const void*)&from)); } template<> EIGEN_STRONG_INLINE Packet2cd ploaddup(const std::complex* from) { return pset1(*from); } template<> EIGEN_STRONG_INLINE void pstore >(std::complex * to, const Packet2cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v); } template<> EIGEN_STRONG_INLINE void pstoreu >(std::complex * to, const Packet2cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, from.v); } template<> EIGEN_DEVICE_FUNC inline Packet2cd pgather, Packet2cd>(const std::complex* from, Index stride) { return Packet2cd(_mm256_set_pd(std::imag(from[1*stride]), std::real(from[1*stride]), std::imag(from[0*stride]), std::real(from[0*stride]))); } template<> EIGEN_DEVICE_FUNC inline void pscatter, Packet2cd>(std::complex* to, const Packet2cd& from, Index stride) { __m128d low = _mm256_extractf128_pd(from.v, 0); to[stride*0] = std::complex(_mm_cvtsd_f64(low), _mm_cvtsd_f64(_mm_shuffle_pd(low, low, 1))); __m128d high = _mm256_extractf128_pd(from.v, 1); to[stride*1] = std::complex(_mm_cvtsd_f64(high), _mm_cvtsd_f64(_mm_shuffle_pd(high, high, 1))); } template<> EIGEN_STRONG_INLINE std::complex pfirst(const Packet2cd& a) { __m128d low = _mm256_extractf128_pd(a.v, 0); EIGEN_ALIGN16 double res[2]; _mm_store_pd(res, low); return std::complex(res[0],res[1]); } template<> EIGEN_STRONG_INLINE Packet2cd preverse(const Packet2cd& a) { __m256d result = _mm256_permute2f128_pd(a.v, a.v, 1); return Packet2cd(result); } template<> EIGEN_STRONG_INLINE std::complex predux(const Packet2cd& a) { return predux(padd(Packet1cd(_mm256_extractf128_pd(a.v,0)), Packet1cd(_mm256_extractf128_pd(a.v,1)))); } template<> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet2cd& a) { return predux(pmul(Packet1cd(_mm256_extractf128_pd(a.v,0)), Packet1cd(_mm256_extractf128_pd(a.v,1)))); } EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cd,Packet4d) template<> EIGEN_STRONG_INLINE Packet2cd pdiv(const Packet2cd& a, const Packet2cd& b) { Packet2cd num = pmul(a, pconj(b)); __m256d tmp = _mm256_mul_pd(b.v, b.v); __m256d denom = _mm256_hadd_pd(tmp, tmp); return Packet2cd(_mm256_div_pd(num.v, denom)); } template<> EIGEN_STRONG_INLINE Packet2cd pcplxflip(const Packet2cd& x) { return Packet2cd(_mm256_shuffle_pd(x.v, x.v, 0x5)); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { __m256d P0 = _mm256_castps_pd(kernel.packet[0].v); __m256d P1 = _mm256_castps_pd(kernel.packet[1].v); __m256d P2 = _mm256_castps_pd(kernel.packet[2].v); __m256d P3 = _mm256_castps_pd(kernel.packet[3].v); __m256d T0 = _mm256_shuffle_pd(P0, P1, 15); __m256d T1 = _mm256_shuffle_pd(P0, P1, 0); __m256d T2 = _mm256_shuffle_pd(P2, P3, 15); __m256d T3 = _mm256_shuffle_pd(P2, P3, 0); kernel.packet[1].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T0, T2, 32)); kernel.packet[3].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T0, T2, 49)); kernel.packet[0].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T1, T3, 32)); kernel.packet[2].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T1, T3, 49)); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { __m256d tmp = _mm256_permute2f128_pd(kernel.packet[0].v, kernel.packet[1].v, 0+(2<<4)); kernel.packet[1].v = _mm256_permute2f128_pd(kernel.packet[0].v, kernel.packet[1].v, 1+(3<<4)); kernel.packet[0].v = tmp; } template<> EIGEN_STRONG_INLINE Packet2cd psqrt(const Packet2cd& a) { return psqrt_complex(a); } template<> EIGEN_STRONG_INLINE Packet4cf psqrt(const Packet4cf& a) { return psqrt_complex(a); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_COMPLEX_AVX_H RcppEigen/inst/include/Eigen/src/Core/arch/AVX/PacketMath.h0000644000176200001440000017614014567757725023043 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2014 Benoit Steiner (benoit.steiner.goog@gmail.com) // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_PACKET_MATH_AVX_H #define EIGEN_PACKET_MATH_AVX_H namespace Eigen { namespace internal { #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 #endif #if !defined(EIGEN_VECTORIZE_AVX512) && !defined(EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS) #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 16 #endif #ifdef EIGEN_VECTORIZE_FMA #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD #endif #endif typedef __m256 Packet8f; typedef __m256i Packet8i; typedef __m256d Packet4d; typedef eigen_packet_wrapper<__m128i, 2> Packet8h; typedef eigen_packet_wrapper<__m128i, 3> Packet8bf; template<> struct is_arithmetic<__m256> { enum { value = true }; }; template<> struct is_arithmetic<__m256i> { enum { value = true }; }; template<> struct is_arithmetic<__m256d> { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; #define _EIGEN_DECLARE_CONST_Packet8f(NAME,X) \ const Packet8f p8f_##NAME = pset1(X) #define _EIGEN_DECLARE_CONST_Packet4d(NAME,X) \ const Packet4d p4d_##NAME = pset1(X) #define _EIGEN_DECLARE_CONST_Packet8f_FROM_INT(NAME,X) \ const Packet8f p8f_##NAME = _mm256_castsi256_ps(pset1(X)) #define _EIGEN_DECLARE_CONST_Packet8i(NAME,X) \ const Packet8i p8i_##NAME = pset1(X) // Use the packet_traits defined in AVX512/PacketMath.h instead if we're going // to leverage AVX512 instructions. #ifndef EIGEN_VECTORIZE_AVX512 template<> struct packet_traits : default_packet_traits { typedef Packet8f type; typedef Packet4f half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 8, HasHalfPacket = 1, HasCmp = 1, HasDiv = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasLog = 1, HasLog1p = 1, HasExpm1 = 1, HasExp = 1, HasNdtri = 1, HasBessel = 1, HasSqrt = 1, HasRsqrt = 1, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, HasBlend = 1, HasRound = 1, HasFloor = 1, HasCeil = 1, HasRint = 1 }; }; template<> struct packet_traits : default_packet_traits { typedef Packet4d type; typedef Packet2d half; enum { Vectorizable = 1, AlignedOnScalar = 1, size=4, HasHalfPacket = 1, HasCmp = 1, HasDiv = 1, HasLog = 1, HasExp = 1, HasSqrt = 1, HasRsqrt = 1, HasBlend = 1, HasRound = 1, HasFloor = 1, HasCeil = 1, HasRint = 1 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet8h type; // There is no half-size packet for Packet8h. typedef Packet8h half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 8, HasHalfPacket = 0, HasCmp = 1, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasNegate = 1, HasAbs = 1, HasAbs2 = 0, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasLog = 1, HasLog1p = 1, HasExpm1 = 1, HasExp = 1, HasSqrt = 1, HasRsqrt = 1, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, HasBlend = 0, HasRound = 1, HasFloor = 1, HasCeil = 1, HasRint = 1, HasBessel = 1, HasNdtri = 1 }; }; template <> struct packet_traits : default_packet_traits { typedef Packet8bf type; // There is no half-size packet for current Packet8bf. // TODO: support as SSE path. typedef Packet8bf half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 8, HasHalfPacket = 0, HasCmp = 1, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasNegate = 1, HasAbs = 1, HasAbs2 = 0, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasLog = 1, HasLog1p = 1, HasExpm1 = 1, HasExp = 1, HasSqrt = 1, HasRsqrt = 1, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, HasBlend = 0, HasRound = 1, HasFloor = 1, HasCeil = 1, HasRint = 1, HasBessel = 1, HasNdtri = 1 }; }; #endif template<> struct scalar_div_cost { enum { value = 14 }; }; template<> struct scalar_div_cost { enum { value = 16 }; }; /* Proper support for integers is only provided by AVX2. In the meantime, we'll use SSE instructions and packets to deal with integers. template<> struct packet_traits : default_packet_traits { typedef Packet8i type; enum { Vectorizable = 1, AlignedOnScalar = 1, size=8 }; }; */ template<> struct unpacket_traits { typedef float type; typedef Packet4f half; typedef Packet8i integer_packet; typedef uint8_t mask_t; enum {size=8, alignment=Aligned32, vectorizable=true, masked_load_available=true, masked_store_available=true}; }; template<> struct unpacket_traits { typedef double type; typedef Packet2d half; enum {size=4, alignment=Aligned32, vectorizable=true, masked_load_available=false, masked_store_available=false}; }; template<> struct unpacket_traits { typedef int type; typedef Packet4i half; enum {size=8, alignment=Aligned32, vectorizable=false, masked_load_available=false, masked_store_available=false}; }; template<> struct unpacket_traits { typedef bfloat16 type; typedef Packet8bf half; enum {size=8, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; }; // Helper function for bit packing snippet of low precision comparison. // It packs the flags from 16x16 to 8x16. EIGEN_STRONG_INLINE __m128i Pack16To8(Packet8f rf) { return _mm_packs_epi32(_mm256_extractf128_si256(_mm256_castps_si256(rf), 0), _mm256_extractf128_si256(_mm256_castps_si256(rf), 1)); } template<> EIGEN_STRONG_INLINE Packet8f pset1(const float& from) { return _mm256_set1_ps(from); } template<> EIGEN_STRONG_INLINE Packet4d pset1(const double& from) { return _mm256_set1_pd(from); } template<> EIGEN_STRONG_INLINE Packet8i pset1(const int& from) { return _mm256_set1_epi32(from); } template<> EIGEN_STRONG_INLINE Packet8f pset1frombits(unsigned int from) { return _mm256_castsi256_ps(pset1(from)); } template<> EIGEN_STRONG_INLINE Packet4d pset1frombits(uint64_t from) { return _mm256_castsi256_pd(_mm256_set1_epi64x(from)); } template<> EIGEN_STRONG_INLINE Packet8f pzero(const Packet8f& /*a*/) { return _mm256_setzero_ps(); } template<> EIGEN_STRONG_INLINE Packet4d pzero(const Packet4d& /*a*/) { return _mm256_setzero_pd(); } template<> EIGEN_STRONG_INLINE Packet8i pzero(const Packet8i& /*a*/) { return _mm256_setzero_si256(); } template<> EIGEN_STRONG_INLINE Packet8f peven_mask(const Packet8f& /*a*/) { return _mm256_castsi256_ps(_mm256_set_epi32(0, -1, 0, -1, 0, -1, 0, -1)); } template<> EIGEN_STRONG_INLINE Packet8i peven_mask(const Packet8i& /*a*/) { return _mm256_set_epi32(0, -1, 0, -1, 0, -1, 0, -1); } template<> EIGEN_STRONG_INLINE Packet4d peven_mask(const Packet4d& /*a*/) { return _mm256_castsi256_pd(_mm256_set_epi32(0, 0, -1, -1, 0, 0, -1, -1)); } template<> EIGEN_STRONG_INLINE Packet8f pload1(const float* from) { return _mm256_broadcast_ss(from); } template<> EIGEN_STRONG_INLINE Packet4d pload1(const double* from) { return _mm256_broadcast_sd(from); } template<> EIGEN_STRONG_INLINE Packet8f plset(const float& a) { return _mm256_add_ps(_mm256_set1_ps(a), _mm256_set_ps(7.0,6.0,5.0,4.0,3.0,2.0,1.0,0.0)); } template<> EIGEN_STRONG_INLINE Packet4d plset(const double& a) { return _mm256_add_pd(_mm256_set1_pd(a), _mm256_set_pd(3.0,2.0,1.0,0.0)); } template<> EIGEN_STRONG_INLINE Packet8f padd(const Packet8f& a, const Packet8f& b) { return _mm256_add_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4d padd(const Packet4d& a, const Packet4d& b) { return _mm256_add_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet8i padd(const Packet8i& a, const Packet8i& b) { #ifdef EIGEN_VECTORIZE_AVX2 return _mm256_add_epi32(a,b); #else __m128i lo = _mm_add_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0)); __m128i hi = _mm_add_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)); return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); #endif } template<> EIGEN_STRONG_INLINE Packet8f psub(const Packet8f& a, const Packet8f& b) { return _mm256_sub_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4d psub(const Packet4d& a, const Packet4d& b) { return _mm256_sub_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet8i psub(const Packet8i& a, const Packet8i& b) { #ifdef EIGEN_VECTORIZE_AVX2 return _mm256_sub_epi32(a,b); #else __m128i lo = _mm_sub_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0)); __m128i hi = _mm_sub_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)); return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); #endif } template<> EIGEN_STRONG_INLINE Packet8f pnegate(const Packet8f& a) { return _mm256_sub_ps(_mm256_set1_ps(0.0),a); } template<> EIGEN_STRONG_INLINE Packet4d pnegate(const Packet4d& a) { return _mm256_sub_pd(_mm256_set1_pd(0.0),a); } template<> EIGEN_STRONG_INLINE Packet8f pconj(const Packet8f& a) { return a; } template<> EIGEN_STRONG_INLINE Packet4d pconj(const Packet4d& a) { return a; } template<> EIGEN_STRONG_INLINE Packet8i pconj(const Packet8i& a) { return a; } template<> EIGEN_STRONG_INLINE Packet8f pmul(const Packet8f& a, const Packet8f& b) { return _mm256_mul_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4d pmul(const Packet4d& a, const Packet4d& b) { return _mm256_mul_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet8i pmul(const Packet8i& a, const Packet8i& b) { #ifdef EIGEN_VECTORIZE_AVX2 return _mm256_mullo_epi32(a,b); #else const __m128i lo = _mm_mullo_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0)); const __m128i hi = _mm_mullo_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)); return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); #endif } template<> EIGEN_STRONG_INLINE Packet8f pdiv(const Packet8f& a, const Packet8f& b) { return _mm256_div_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4d pdiv(const Packet4d& a, const Packet4d& b) { return _mm256_div_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet8i pdiv(const Packet8i& /*a*/, const Packet8i& /*b*/) { eigen_assert(false && "packet integer division are not supported by AVX"); return pset1(0); } #ifdef EIGEN_VECTORIZE_FMA template<> EIGEN_STRONG_INLINE Packet8f pmadd(const Packet8f& a, const Packet8f& b, const Packet8f& c) { #if ( (EIGEN_COMP_GNUC_STRICT && EIGEN_COMP_GNUC<80) || (EIGEN_COMP_CLANG) ) // Clang stupidly generates a vfmadd213ps instruction plus some vmovaps on registers, // and even register spilling with clang>=6.0 (bug 1637). // Gcc stupidly generates a vfmadd132ps instruction. // So let's enforce it to generate a vfmadd231ps instruction since the most common use // case is to accumulate the result of the product. Packet8f res = c; __asm__("vfmadd231ps %[a], %[b], %[c]" : [c] "+x" (res) : [a] "x" (a), [b] "x" (b)); return res; #else return _mm256_fmadd_ps(a,b,c); #endif } template<> EIGEN_STRONG_INLINE Packet4d pmadd(const Packet4d& a, const Packet4d& b, const Packet4d& c) { #if ( (EIGEN_COMP_GNUC_STRICT && EIGEN_COMP_GNUC<80) || (EIGEN_COMP_CLANG) ) // see above Packet4d res = c; __asm__("vfmadd231pd %[a], %[b], %[c]" : [c] "+x" (res) : [a] "x" (a), [b] "x" (b)); return res; #else return _mm256_fmadd_pd(a,b,c); #endif } #endif template<> EIGEN_STRONG_INLINE Packet8f pcmp_le(const Packet8f& a, const Packet8f& b) { return _mm256_cmp_ps(a,b,_CMP_LE_OQ); } template<> EIGEN_STRONG_INLINE Packet8f pcmp_lt(const Packet8f& a, const Packet8f& b) { return _mm256_cmp_ps(a,b,_CMP_LT_OQ); } template<> EIGEN_STRONG_INLINE Packet8f pcmp_lt_or_nan(const Packet8f& a, const Packet8f& b) { return _mm256_cmp_ps(a, b, _CMP_NGE_UQ); } template<> EIGEN_STRONG_INLINE Packet8f pcmp_eq(const Packet8f& a, const Packet8f& b) { return _mm256_cmp_ps(a,b,_CMP_EQ_OQ); } template<> EIGEN_STRONG_INLINE Packet4d pcmp_le(const Packet4d& a, const Packet4d& b) { return _mm256_cmp_pd(a,b,_CMP_LE_OQ); } template<> EIGEN_STRONG_INLINE Packet4d pcmp_lt(const Packet4d& a, const Packet4d& b) { return _mm256_cmp_pd(a,b,_CMP_LT_OQ); } template<> EIGEN_STRONG_INLINE Packet4d pcmp_lt_or_nan(const Packet4d& a, const Packet4d& b) { return _mm256_cmp_pd(a, b, _CMP_NGE_UQ); } template<> EIGEN_STRONG_INLINE Packet4d pcmp_eq(const Packet4d& a, const Packet4d& b) { return _mm256_cmp_pd(a,b,_CMP_EQ_OQ); } template<> EIGEN_STRONG_INLINE Packet8i pcmp_eq(const Packet8i& a, const Packet8i& b) { #ifdef EIGEN_VECTORIZE_AVX2 return _mm256_cmpeq_epi32(a,b); #else __m128i lo = _mm_cmpeq_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0)); __m128i hi = _mm_cmpeq_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)); return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); #endif } template<> EIGEN_STRONG_INLINE Packet8f pmin(const Packet8f& a, const Packet8f& b) { #if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 // There appears to be a bug in GCC, by which the optimizer may flip // the argument order in calls to _mm_min_ps/_mm_max_ps, so we have to // resort to inline ASM here. This is supposed to be fixed in gcc6.3, // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 Packet8f res; asm("vminps %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); return res; #else // Arguments are swapped to match NaN propagation behavior of std::min. return _mm256_min_ps(b,a); #endif } template<> EIGEN_STRONG_INLINE Packet4d pmin(const Packet4d& a, const Packet4d& b) { #if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 // See pmin above Packet4d res; asm("vminpd %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); return res; #else // Arguments are swapped to match NaN propagation behavior of std::min. return _mm256_min_pd(b,a); #endif } template<> EIGEN_STRONG_INLINE Packet8f pmax(const Packet8f& a, const Packet8f& b) { #if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 // See pmin above Packet8f res; asm("vmaxps %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); return res; #else // Arguments are swapped to match NaN propagation behavior of std::max. return _mm256_max_ps(b,a); #endif } template<> EIGEN_STRONG_INLINE Packet4d pmax(const Packet4d& a, const Packet4d& b) { #if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63 // See pmin above Packet4d res; asm("vmaxpd %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b)); return res; #else // Arguments are swapped to match NaN propagation behavior of std::max. return _mm256_max_pd(b,a); #endif } // Add specializations for min/max with prescribed NaN progation. template<> EIGEN_STRONG_INLINE Packet8f pmin(const Packet8f& a, const Packet8f& b) { return pminmax_propagate_numbers(a, b, pmin); } template<> EIGEN_STRONG_INLINE Packet4d pmin(const Packet4d& a, const Packet4d& b) { return pminmax_propagate_numbers(a, b, pmin); } template<> EIGEN_STRONG_INLINE Packet8f pmax(const Packet8f& a, const Packet8f& b) { return pminmax_propagate_numbers(a, b, pmax); } template<> EIGEN_STRONG_INLINE Packet4d pmax(const Packet4d& a, const Packet4d& b) { return pminmax_propagate_numbers(a, b, pmax); } template<> EIGEN_STRONG_INLINE Packet8f pmin(const Packet8f& a, const Packet8f& b) { return pminmax_propagate_nan(a, b, pmin); } template<> EIGEN_STRONG_INLINE Packet4d pmin(const Packet4d& a, const Packet4d& b) { return pminmax_propagate_nan(a, b, pmin); } template<> EIGEN_STRONG_INLINE Packet8f pmax(const Packet8f& a, const Packet8f& b) { return pminmax_propagate_nan(a, b, pmax); } template<> EIGEN_STRONG_INLINE Packet4d pmax(const Packet4d& a, const Packet4d& b) { return pminmax_propagate_nan(a, b, pmax); } template<> EIGEN_STRONG_INLINE Packet8f print(const Packet8f& a) { return _mm256_round_ps(a, _MM_FROUND_CUR_DIRECTION); } template<> EIGEN_STRONG_INLINE Packet4d print(const Packet4d& a) { return _mm256_round_pd(a, _MM_FROUND_CUR_DIRECTION); } template<> EIGEN_STRONG_INLINE Packet8f pceil(const Packet8f& a) { return _mm256_ceil_ps(a); } template<> EIGEN_STRONG_INLINE Packet4d pceil(const Packet4d& a) { return _mm256_ceil_pd(a); } template<> EIGEN_STRONG_INLINE Packet8f pfloor(const Packet8f& a) { return _mm256_floor_ps(a); } template<> EIGEN_STRONG_INLINE Packet4d pfloor(const Packet4d& a) { return _mm256_floor_pd(a); } template<> EIGEN_STRONG_INLINE Packet8i ptrue(const Packet8i& a) { #ifdef EIGEN_VECTORIZE_AVX2 // vpcmpeqd has lower latency than the more general vcmpps return _mm256_cmpeq_epi32(a,a); #else const __m256 b = _mm256_castsi256_ps(a); return _mm256_castps_si256(_mm256_cmp_ps(b,b,_CMP_TRUE_UQ)); #endif } template<> EIGEN_STRONG_INLINE Packet8f ptrue(const Packet8f& a) { #ifdef EIGEN_VECTORIZE_AVX2 // vpcmpeqd has lower latency than the more general vcmpps const __m256i b = _mm256_castps_si256(a); return _mm256_castsi256_ps(_mm256_cmpeq_epi32(b,b)); #else return _mm256_cmp_ps(a,a,_CMP_TRUE_UQ); #endif } template<> EIGEN_STRONG_INLINE Packet4d ptrue(const Packet4d& a) { #ifdef EIGEN_VECTORIZE_AVX2 // vpcmpeqq has lower latency than the more general vcmppd const __m256i b = _mm256_castpd_si256(a); return _mm256_castsi256_pd(_mm256_cmpeq_epi64(b,b)); #else return _mm256_cmp_pd(a,a,_CMP_TRUE_UQ); #endif } template<> EIGEN_STRONG_INLINE Packet8f pand(const Packet8f& a, const Packet8f& b) { return _mm256_and_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4d pand(const Packet4d& a, const Packet4d& b) { return _mm256_and_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet8i pand(const Packet8i& a, const Packet8i& b) { #ifdef EIGEN_VECTORIZE_AVX2 return _mm256_and_si256(a,b); #else return _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(a),_mm256_castsi256_ps(b))); #endif } template<> EIGEN_STRONG_INLINE Packet8f por(const Packet8f& a, const Packet8f& b) { return _mm256_or_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4d por(const Packet4d& a, const Packet4d& b) { return _mm256_or_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet8i por(const Packet8i& a, const Packet8i& b) { #ifdef EIGEN_VECTORIZE_AVX2 return _mm256_or_si256(a,b); #else return _mm256_castps_si256(_mm256_or_ps(_mm256_castsi256_ps(a),_mm256_castsi256_ps(b))); #endif } template<> EIGEN_STRONG_INLINE Packet8f pxor(const Packet8f& a, const Packet8f& b) { return _mm256_xor_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet4d pxor(const Packet4d& a, const Packet4d& b) { return _mm256_xor_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet8i pxor(const Packet8i& a, const Packet8i& b) { #ifdef EIGEN_VECTORIZE_AVX2 return _mm256_xor_si256(a,b); #else return _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(a),_mm256_castsi256_ps(b))); #endif } template<> EIGEN_STRONG_INLINE Packet8f pandnot(const Packet8f& a, const Packet8f& b) { return _mm256_andnot_ps(b,a); } template<> EIGEN_STRONG_INLINE Packet4d pandnot(const Packet4d& a, const Packet4d& b) { return _mm256_andnot_pd(b,a); } template<> EIGEN_STRONG_INLINE Packet8i pandnot(const Packet8i& a, const Packet8i& b) { #ifdef EIGEN_VECTORIZE_AVX2 return _mm256_andnot_si256(b,a); #else return _mm256_castps_si256(_mm256_andnot_ps(_mm256_castsi256_ps(b),_mm256_castsi256_ps(a))); #endif } template<> EIGEN_STRONG_INLINE Packet8f pround(const Packet8f& a) { const Packet8f mask = pset1frombits(static_cast(0x80000000u)); const Packet8f prev0dot5 = pset1frombits(static_cast(0x3EFFFFFFu)); return _mm256_round_ps(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO); } template<> EIGEN_STRONG_INLINE Packet4d pround(const Packet4d& a) { const Packet4d mask = pset1frombits(static_cast(0x8000000000000000ull)); const Packet4d prev0dot5 = pset1frombits(static_cast(0x3FDFFFFFFFFFFFFFull)); return _mm256_round_pd(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO); } template<> EIGEN_STRONG_INLINE Packet8f pselect(const Packet8f& mask, const Packet8f& a, const Packet8f& b) { return _mm256_blendv_ps(b,a,mask); } template<> EIGEN_STRONG_INLINE Packet4d pselect(const Packet4d& mask, const Packet4d& a, const Packet4d& b) { return _mm256_blendv_pd(b,a,mask); } template EIGEN_STRONG_INLINE Packet8i parithmetic_shift_right(Packet8i a) { #ifdef EIGEN_VECTORIZE_AVX2 return _mm256_srai_epi32(a, N); #else __m128i lo = _mm_srai_epi32(_mm256_extractf128_si256(a, 0), N); __m128i hi = _mm_srai_epi32(_mm256_extractf128_si256(a, 1), N); return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); #endif } template EIGEN_STRONG_INLINE Packet8i plogical_shift_right(Packet8i a) { #ifdef EIGEN_VECTORIZE_AVX2 return _mm256_srli_epi32(a, N); #else __m128i lo = _mm_srli_epi32(_mm256_extractf128_si256(a, 0), N); __m128i hi = _mm_srli_epi32(_mm256_extractf128_si256(a, 1), N); return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); #endif } template EIGEN_STRONG_INLINE Packet8i plogical_shift_left(Packet8i a) { #ifdef EIGEN_VECTORIZE_AVX2 return _mm256_slli_epi32(a, N); #else __m128i lo = _mm_slli_epi32(_mm256_extractf128_si256(a, 0), N); __m128i hi = _mm_slli_epi32(_mm256_extractf128_si256(a, 1), N); return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1); #endif } template<> EIGEN_STRONG_INLINE Packet8f pload(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_ps(from); } template<> EIGEN_STRONG_INLINE Packet4d pload(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_pd(from); } template<> EIGEN_STRONG_INLINE Packet8i pload(const int* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_si256(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE Packet8f ploadu(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_ps(from); } template<> EIGEN_STRONG_INLINE Packet4d ploadu(const double* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_pd(from); } template<> EIGEN_STRONG_INLINE Packet8i ploadu(const int* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_si256(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE Packet8f ploadu(const float* from, uint8_t umask) { Packet8i mask = _mm256_set1_epi8(static_cast(umask)); const Packet8i bit_mask = _mm256_set_epi32(0xffffff7f, 0xffffffbf, 0xffffffdf, 0xffffffef, 0xfffffff7, 0xfffffffb, 0xfffffffd, 0xfffffffe); mask = por(mask, bit_mask); mask = pcmp_eq(mask, _mm256_set1_epi32(0xffffffff)); EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_maskload_ps(from, mask); } // Loads 4 floats from memory a returns the packet {a0, a0 a1, a1, a2, a2, a3, a3} template<> EIGEN_STRONG_INLINE Packet8f ploaddup(const float* from) { // TODO try to find a way to avoid the need of a temporary register // Packet8f tmp = _mm256_castps128_ps256(_mm_loadu_ps(from)); // tmp = _mm256_insertf128_ps(tmp, _mm_movehl_ps(_mm256_castps256_ps128(tmp),_mm256_castps256_ps128(tmp)), 1); // return _mm256_unpacklo_ps(tmp,tmp); // _mm256_insertf128_ps is very slow on Haswell, thus: Packet8f tmp = _mm256_broadcast_ps((const __m128*)(const void*)from); // mimic an "inplace" permutation of the lower 128bits using a blend tmp = _mm256_blend_ps(tmp,_mm256_castps128_ps256(_mm_permute_ps( _mm256_castps256_ps128(tmp), _MM_SHUFFLE(1,0,1,0))), 15); // then we can perform a consistent permutation on the global register to get everything in shape: return _mm256_permute_ps(tmp, _MM_SHUFFLE(3,3,2,2)); } // Loads 2 doubles from memory a returns the packet {a0, a0 a1, a1} template<> EIGEN_STRONG_INLINE Packet4d ploaddup(const double* from) { Packet4d tmp = _mm256_broadcast_pd((const __m128d*)(const void*)from); return _mm256_permute_pd(tmp, 3<<2); } // Loads 2 floats from memory a returns the packet {a0, a0 a0, a0, a1, a1, a1, a1} template<> EIGEN_STRONG_INLINE Packet8f ploadquad(const float* from) { Packet8f tmp = _mm256_castps128_ps256(_mm_broadcast_ss(from)); return _mm256_insertf128_ps(tmp, _mm_broadcast_ss(from+1), 1); } template<> EIGEN_STRONG_INLINE void pstore(float* to, const Packet8f& from) { EIGEN_DEBUG_ALIGNED_STORE _mm256_store_ps(to, from); } template<> EIGEN_STRONG_INLINE void pstore(double* to, const Packet4d& from) { EIGEN_DEBUG_ALIGNED_STORE _mm256_store_pd(to, from); } template<> EIGEN_STRONG_INLINE void pstore(int* to, const Packet8i& from) { EIGEN_DEBUG_ALIGNED_STORE _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from); } template<> EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet8f& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_ps(to, from); } template<> EIGEN_STRONG_INLINE void pstoreu(double* to, const Packet4d& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_pd(to, from); } template<> EIGEN_STRONG_INLINE void pstoreu(int* to, const Packet8i& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from); } template<> EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet8f& from, uint8_t umask) { Packet8i mask = _mm256_set1_epi8(static_cast(umask)); const Packet8i bit_mask = _mm256_set_epi32(0xffffff7f, 0xffffffbf, 0xffffffdf, 0xffffffef, 0xfffffff7, 0xfffffffb, 0xfffffffd, 0xfffffffe); mask = por(mask, bit_mask); mask = pcmp_eq(mask, _mm256_set1_epi32(0xffffffff)); EIGEN_DEBUG_UNALIGNED_STORE return _mm256_maskstore_ps(to, mask, from); } // NOTE: leverage _mm256_i32gather_ps and _mm256_i32gather_pd if AVX2 instructions are available // NOTE: for the record the following seems to be slower: return _mm256_i32gather_ps(from, _mm256_set1_epi32(stride), 4); template<> EIGEN_DEVICE_FUNC inline Packet8f pgather(const float* from, Index stride) { return _mm256_set_ps(from[7*stride], from[6*stride], from[5*stride], from[4*stride], from[3*stride], from[2*stride], from[1*stride], from[0*stride]); } template<> EIGEN_DEVICE_FUNC inline Packet4d pgather(const double* from, Index stride) { return _mm256_set_pd(from[3*stride], from[2*stride], from[1*stride], from[0*stride]); } template<> EIGEN_DEVICE_FUNC inline void pscatter(float* to, const Packet8f& from, Index stride) { __m128 low = _mm256_extractf128_ps(from, 0); to[stride*0] = _mm_cvtss_f32(low); to[stride*1] = _mm_cvtss_f32(_mm_shuffle_ps(low, low, 1)); to[stride*2] = _mm_cvtss_f32(_mm_shuffle_ps(low, low, 2)); to[stride*3] = _mm_cvtss_f32(_mm_shuffle_ps(low, low, 3)); __m128 high = _mm256_extractf128_ps(from, 1); to[stride*4] = _mm_cvtss_f32(high); to[stride*5] = _mm_cvtss_f32(_mm_shuffle_ps(high, high, 1)); to[stride*6] = _mm_cvtss_f32(_mm_shuffle_ps(high, high, 2)); to[stride*7] = _mm_cvtss_f32(_mm_shuffle_ps(high, high, 3)); } template<> EIGEN_DEVICE_FUNC inline void pscatter(double* to, const Packet4d& from, Index stride) { __m128d low = _mm256_extractf128_pd(from, 0); to[stride*0] = _mm_cvtsd_f64(low); to[stride*1] = _mm_cvtsd_f64(_mm_shuffle_pd(low, low, 1)); __m128d high = _mm256_extractf128_pd(from, 1); to[stride*2] = _mm_cvtsd_f64(high); to[stride*3] = _mm_cvtsd_f64(_mm_shuffle_pd(high, high, 1)); } template<> EIGEN_STRONG_INLINE void pstore1(float* to, const float& a) { Packet8f pa = pset1(a); pstore(to, pa); } template<> EIGEN_STRONG_INLINE void pstore1(double* to, const double& a) { Packet4d pa = pset1(a); pstore(to, pa); } template<> EIGEN_STRONG_INLINE void pstore1(int* to, const int& a) { Packet8i pa = pset1(a); pstore(to, pa); } #ifndef EIGEN_VECTORIZE_AVX512 template<> EIGEN_STRONG_INLINE void prefetch(const float* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } template<> EIGEN_STRONG_INLINE void prefetch(const double* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } template<> EIGEN_STRONG_INLINE void prefetch(const int* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } #endif template<> EIGEN_STRONG_INLINE float pfirst(const Packet8f& a) { return _mm_cvtss_f32(_mm256_castps256_ps128(a)); } template<> EIGEN_STRONG_INLINE double pfirst(const Packet4d& a) { return _mm_cvtsd_f64(_mm256_castpd256_pd128(a)); } template<> EIGEN_STRONG_INLINE int pfirst(const Packet8i& a) { return _mm_cvtsi128_si32(_mm256_castsi256_si128(a)); } template<> EIGEN_STRONG_INLINE Packet8f preverse(const Packet8f& a) { __m256 tmp = _mm256_shuffle_ps(a,a,0x1b); return _mm256_permute2f128_ps(tmp, tmp, 1); } template<> EIGEN_STRONG_INLINE Packet4d preverse(const Packet4d& a) { __m256d tmp = _mm256_shuffle_pd(a,a,5); return _mm256_permute2f128_pd(tmp, tmp, 1); #if 0 // This version is unlikely to be faster as _mm256_shuffle_ps and _mm256_permute_pd // exhibit the same latency/throughput, but it is here for future reference/benchmarking... __m256d swap_halves = _mm256_permute2f128_pd(a,a,1); return _mm256_permute_pd(swap_halves,5); #endif } // pabs should be ok template<> EIGEN_STRONG_INLINE Packet8f pabs(const Packet8f& a) { const Packet8f mask = _mm256_castsi256_ps(_mm256_setr_epi32(0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF)); return _mm256_and_ps(a,mask); } template<> EIGEN_STRONG_INLINE Packet4d pabs(const Packet4d& a) { const Packet4d mask = _mm256_castsi256_pd(_mm256_setr_epi32(0xFFFFFFFF,0x7FFFFFFF,0xFFFFFFFF,0x7FFFFFFF,0xFFFFFFFF,0x7FFFFFFF,0xFFFFFFFF,0x7FFFFFFF)); return _mm256_and_pd(a,mask); } template<> EIGEN_STRONG_INLINE Packet8f pfrexp(const Packet8f& a, Packet8f& exponent) { return pfrexp_generic(a,exponent); } // Extract exponent without existence of Packet4l. template<> EIGEN_STRONG_INLINE Packet4d pfrexp_generic_get_biased_exponent(const Packet4d& a) { const Packet4d cst_exp_mask = pset1frombits(static_cast(0x7ff0000000000000ull)); __m256i a_expo = _mm256_castpd_si256(pand(a, cst_exp_mask)); #ifdef EIGEN_VECTORIZE_AVX2 a_expo = _mm256_srli_epi64(a_expo, 52); __m128i lo = _mm256_extractf128_si256(a_expo, 0); __m128i hi = _mm256_extractf128_si256(a_expo, 1); #else __m128i lo = _mm256_extractf128_si256(a_expo, 0); __m128i hi = _mm256_extractf128_si256(a_expo, 1); lo = _mm_srli_epi64(lo, 52); hi = _mm_srli_epi64(hi, 52); #endif Packet2d exponent_lo = _mm_cvtepi32_pd(vec4i_swizzle1(lo, 0, 2, 1, 3)); Packet2d exponent_hi = _mm_cvtepi32_pd(vec4i_swizzle1(hi, 0, 2, 1, 3)); Packet4d exponent = _mm256_insertf128_pd(_mm256_setzero_pd(), exponent_lo, 0); exponent = _mm256_insertf128_pd(exponent, exponent_hi, 1); return exponent; } template<> EIGEN_STRONG_INLINE Packet4d pfrexp(const Packet4d& a, Packet4d& exponent) { return pfrexp_generic(a, exponent); } template<> EIGEN_STRONG_INLINE Packet8f pldexp(const Packet8f& a, const Packet8f& exponent) { return pldexp_generic(a, exponent); } template<> EIGEN_STRONG_INLINE Packet4d pldexp(const Packet4d& a, const Packet4d& exponent) { // Clamp exponent to [-2099, 2099] const Packet4d max_exponent = pset1(2099.0); const Packet4i e = _mm256_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent)); // Split 2^e into four factors and multiply. const Packet4i bias = pset1(1023); Packet4i b = parithmetic_shift_right<2>(e); // floor(e/4) // 2^b Packet4i hi = vec4i_swizzle1(padd(b, bias), 0, 2, 1, 3); Packet4i lo = _mm_slli_epi64(hi, 52); hi = _mm_slli_epi64(_mm_srli_epi64(hi, 32), 52); Packet4d c = _mm256_castsi256_pd(_mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1)); Packet4d out = pmul(pmul(pmul(a, c), c), c); // a * 2^(3b) // 2^(e - 3b) b = psub(psub(psub(e, b), b), b); // e - 3b hi = vec4i_swizzle1(padd(b, bias), 0, 2, 1, 3); lo = _mm_slli_epi64(hi, 52); hi = _mm_slli_epi64(_mm_srli_epi64(hi, 32), 52); c = _mm256_castsi256_pd(_mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1)); out = pmul(out, c); // a * 2^e return out; } template<> EIGEN_STRONG_INLINE float predux(const Packet8f& a) { return predux(Packet4f(_mm_add_ps(_mm256_castps256_ps128(a),_mm256_extractf128_ps(a,1)))); } template<> EIGEN_STRONG_INLINE double predux(const Packet4d& a) { return predux(Packet2d(_mm_add_pd(_mm256_castpd256_pd128(a),_mm256_extractf128_pd(a,1)))); } template<> EIGEN_STRONG_INLINE Packet4f predux_half_dowto4(const Packet8f& a) { return _mm_add_ps(_mm256_castps256_ps128(a),_mm256_extractf128_ps(a,1)); } template<> EIGEN_STRONG_INLINE float predux_mul(const Packet8f& a) { Packet8f tmp; tmp = _mm256_mul_ps(a, _mm256_permute2f128_ps(a,a,1)); tmp = _mm256_mul_ps(tmp, _mm256_shuffle_ps(tmp,tmp,_MM_SHUFFLE(1,0,3,2))); return pfirst(_mm256_mul_ps(tmp, _mm256_shuffle_ps(tmp,tmp,1))); } template<> EIGEN_STRONG_INLINE double predux_mul(const Packet4d& a) { Packet4d tmp; tmp = _mm256_mul_pd(a, _mm256_permute2f128_pd(a,a,1)); return pfirst(_mm256_mul_pd(tmp, _mm256_shuffle_pd(tmp,tmp,1))); } template<> EIGEN_STRONG_INLINE float predux_min(const Packet8f& a) { Packet8f tmp = _mm256_min_ps(a, _mm256_permute2f128_ps(a,a,1)); tmp = _mm256_min_ps(tmp, _mm256_shuffle_ps(tmp,tmp,_MM_SHUFFLE(1,0,3,2))); return pfirst(_mm256_min_ps(tmp, _mm256_shuffle_ps(tmp,tmp,1))); } template<> EIGEN_STRONG_INLINE double predux_min(const Packet4d& a) { Packet4d tmp = _mm256_min_pd(a, _mm256_permute2f128_pd(a,a,1)); return pfirst(_mm256_min_pd(tmp, _mm256_shuffle_pd(tmp, tmp, 1))); } template<> EIGEN_STRONG_INLINE float predux_max(const Packet8f& a) { Packet8f tmp = _mm256_max_ps(a, _mm256_permute2f128_ps(a,a,1)); tmp = _mm256_max_ps(tmp, _mm256_shuffle_ps(tmp,tmp,_MM_SHUFFLE(1,0,3,2))); return pfirst(_mm256_max_ps(tmp, _mm256_shuffle_ps(tmp,tmp,1))); } template<> EIGEN_STRONG_INLINE double predux_max(const Packet4d& a) { Packet4d tmp = _mm256_max_pd(a, _mm256_permute2f128_pd(a,a,1)); return pfirst(_mm256_max_pd(tmp, _mm256_shuffle_pd(tmp, tmp, 1))); } // not needed yet // template<> EIGEN_STRONG_INLINE bool predux_all(const Packet8f& x) // { // return _mm256_movemask_ps(x)==0xFF; // } template<> EIGEN_STRONG_INLINE bool predux_any(const Packet8f& x) { return _mm256_movemask_ps(x)!=0; } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { __m256 T0 = _mm256_unpacklo_ps(kernel.packet[0], kernel.packet[1]); __m256 T1 = _mm256_unpackhi_ps(kernel.packet[0], kernel.packet[1]); __m256 T2 = _mm256_unpacklo_ps(kernel.packet[2], kernel.packet[3]); __m256 T3 = _mm256_unpackhi_ps(kernel.packet[2], kernel.packet[3]); __m256 T4 = _mm256_unpacklo_ps(kernel.packet[4], kernel.packet[5]); __m256 T5 = _mm256_unpackhi_ps(kernel.packet[4], kernel.packet[5]); __m256 T6 = _mm256_unpacklo_ps(kernel.packet[6], kernel.packet[7]); __m256 T7 = _mm256_unpackhi_ps(kernel.packet[6], kernel.packet[7]); __m256 S0 = _mm256_shuffle_ps(T0,T2,_MM_SHUFFLE(1,0,1,0)); __m256 S1 = _mm256_shuffle_ps(T0,T2,_MM_SHUFFLE(3,2,3,2)); __m256 S2 = _mm256_shuffle_ps(T1,T3,_MM_SHUFFLE(1,0,1,0)); __m256 S3 = _mm256_shuffle_ps(T1,T3,_MM_SHUFFLE(3,2,3,2)); __m256 S4 = _mm256_shuffle_ps(T4,T6,_MM_SHUFFLE(1,0,1,0)); __m256 S5 = _mm256_shuffle_ps(T4,T6,_MM_SHUFFLE(3,2,3,2)); __m256 S6 = _mm256_shuffle_ps(T5,T7,_MM_SHUFFLE(1,0,1,0)); __m256 S7 = _mm256_shuffle_ps(T5,T7,_MM_SHUFFLE(3,2,3,2)); kernel.packet[0] = _mm256_permute2f128_ps(S0, S4, 0x20); kernel.packet[1] = _mm256_permute2f128_ps(S1, S5, 0x20); kernel.packet[2] = _mm256_permute2f128_ps(S2, S6, 0x20); kernel.packet[3] = _mm256_permute2f128_ps(S3, S7, 0x20); kernel.packet[4] = _mm256_permute2f128_ps(S0, S4, 0x31); kernel.packet[5] = _mm256_permute2f128_ps(S1, S5, 0x31); kernel.packet[6] = _mm256_permute2f128_ps(S2, S6, 0x31); kernel.packet[7] = _mm256_permute2f128_ps(S3, S7, 0x31); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { __m256 T0 = _mm256_unpacklo_ps(kernel.packet[0], kernel.packet[1]); __m256 T1 = _mm256_unpackhi_ps(kernel.packet[0], kernel.packet[1]); __m256 T2 = _mm256_unpacklo_ps(kernel.packet[2], kernel.packet[3]); __m256 T3 = _mm256_unpackhi_ps(kernel.packet[2], kernel.packet[3]); __m256 S0 = _mm256_shuffle_ps(T0,T2,_MM_SHUFFLE(1,0,1,0)); __m256 S1 = _mm256_shuffle_ps(T0,T2,_MM_SHUFFLE(3,2,3,2)); __m256 S2 = _mm256_shuffle_ps(T1,T3,_MM_SHUFFLE(1,0,1,0)); __m256 S3 = _mm256_shuffle_ps(T1,T3,_MM_SHUFFLE(3,2,3,2)); kernel.packet[0] = _mm256_permute2f128_ps(S0, S1, 0x20); kernel.packet[1] = _mm256_permute2f128_ps(S2, S3, 0x20); kernel.packet[2] = _mm256_permute2f128_ps(S0, S1, 0x31); kernel.packet[3] = _mm256_permute2f128_ps(S2, S3, 0x31); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { __m256d T0 = _mm256_shuffle_pd(kernel.packet[0], kernel.packet[1], 15); __m256d T1 = _mm256_shuffle_pd(kernel.packet[0], kernel.packet[1], 0); __m256d T2 = _mm256_shuffle_pd(kernel.packet[2], kernel.packet[3], 15); __m256d T3 = _mm256_shuffle_pd(kernel.packet[2], kernel.packet[3], 0); kernel.packet[1] = _mm256_permute2f128_pd(T0, T2, 32); kernel.packet[3] = _mm256_permute2f128_pd(T0, T2, 49); kernel.packet[0] = _mm256_permute2f128_pd(T1, T3, 32); kernel.packet[2] = _mm256_permute2f128_pd(T1, T3, 49); } template<> EIGEN_STRONG_INLINE Packet8f pblend(const Selector<8>& ifPacket, const Packet8f& thenPacket, const Packet8f& elsePacket) { const __m256 zero = _mm256_setzero_ps(); const __m256 select = _mm256_set_ps(ifPacket.select[7], ifPacket.select[6], ifPacket.select[5], ifPacket.select[4], ifPacket.select[3], ifPacket.select[2], ifPacket.select[1], ifPacket.select[0]); __m256 false_mask = _mm256_cmp_ps(select, zero, _CMP_EQ_UQ); return _mm256_blendv_ps(thenPacket, elsePacket, false_mask); } template<> EIGEN_STRONG_INLINE Packet4d pblend(const Selector<4>& ifPacket, const Packet4d& thenPacket, const Packet4d& elsePacket) { const __m256d zero = _mm256_setzero_pd(); const __m256d select = _mm256_set_pd(ifPacket.select[3], ifPacket.select[2], ifPacket.select[1], ifPacket.select[0]); __m256d false_mask = _mm256_cmp_pd(select, zero, _CMP_EQ_UQ); return _mm256_blendv_pd(thenPacket, elsePacket, false_mask); } // Packet math for Eigen::half template<> struct unpacket_traits { typedef Eigen::half type; enum {size=8, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet8h half; }; template<> EIGEN_STRONG_INLINE Packet8h pset1(const Eigen::half& from) { return _mm_set1_epi16(numext::bit_cast(from)); } template<> EIGEN_STRONG_INLINE Eigen::half pfirst(const Packet8h& from) { return numext::bit_cast(static_cast(_mm_extract_epi16(from, 0))); } template<> EIGEN_STRONG_INLINE Packet8h pload(const Eigen::half* from) { return _mm_load_si128(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE Packet8h ploadu(const Eigen::half* from) { return _mm_loadu_si128(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE void pstore(Eigen::half* to, const Packet8h& from) { _mm_store_si128(reinterpret_cast<__m128i*>(to), from); } template<> EIGEN_STRONG_INLINE void pstoreu(Eigen::half* to, const Packet8h& from) { _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from); } template<> EIGEN_STRONG_INLINE Packet8h ploaddup(const Eigen::half* from) { const numext::uint16_t a = numext::bit_cast(from[0]); const numext::uint16_t b = numext::bit_cast(from[1]); const numext::uint16_t c = numext::bit_cast(from[2]); const numext::uint16_t d = numext::bit_cast(from[3]); return _mm_set_epi16(d, d, c, c, b, b, a, a); } template<> EIGEN_STRONG_INLINE Packet8h ploadquad(const Eigen::half* from) { const numext::uint16_t a = numext::bit_cast(from[0]); const numext::uint16_t b = numext::bit_cast(from[1]); return _mm_set_epi16(b, b, b, b, a, a, a, a); } template<> EIGEN_STRONG_INLINE Packet8h ptrue(const Packet8h& a) { return _mm_cmpeq_epi32(a, a); } template <> EIGEN_STRONG_INLINE Packet8h pabs(const Packet8h& a) { const __m128i sign_mask = _mm_set1_epi16(static_cast(0x8000)); return _mm_andnot_si128(sign_mask, a); } EIGEN_STRONG_INLINE Packet8f half2float(const Packet8h& a) { #ifdef EIGEN_HAS_FP16_C return _mm256_cvtph_ps(a); #else EIGEN_ALIGN32 Eigen::half aux[8]; pstore(aux, a); float f0(aux[0]); float f1(aux[1]); float f2(aux[2]); float f3(aux[3]); float f4(aux[4]); float f5(aux[5]); float f6(aux[6]); float f7(aux[7]); return _mm256_set_ps(f7, f6, f5, f4, f3, f2, f1, f0); #endif } EIGEN_STRONG_INLINE Packet8h float2half(const Packet8f& a) { #ifdef EIGEN_HAS_FP16_C return _mm256_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT|_MM_FROUND_NO_EXC); #else EIGEN_ALIGN32 float aux[8]; pstore(aux, a); const numext::uint16_t s0 = numext::bit_cast(Eigen::half(aux[0])); const numext::uint16_t s1 = numext::bit_cast(Eigen::half(aux[1])); const numext::uint16_t s2 = numext::bit_cast(Eigen::half(aux[2])); const numext::uint16_t s3 = numext::bit_cast(Eigen::half(aux[3])); const numext::uint16_t s4 = numext::bit_cast(Eigen::half(aux[4])); const numext::uint16_t s5 = numext::bit_cast(Eigen::half(aux[5])); const numext::uint16_t s6 = numext::bit_cast(Eigen::half(aux[6])); const numext::uint16_t s7 = numext::bit_cast(Eigen::half(aux[7])); return _mm_set_epi16(s7, s6, s5, s4, s3, s2, s1, s0); #endif } template <> EIGEN_STRONG_INLINE Packet8h pmin(const Packet8h& a, const Packet8h& b) { return float2half(pmin(half2float(a), half2float(b))); } template <> EIGEN_STRONG_INLINE Packet8h pmax(const Packet8h& a, const Packet8h& b) { return float2half(pmax(half2float(a), half2float(b))); } template <> EIGEN_STRONG_INLINE Packet8h plset(const half& a) { return float2half(plset(static_cast(a))); } template<> EIGEN_STRONG_INLINE Packet8h por(const Packet8h& a,const Packet8h& b) { // in some cases Packet4i is a wrapper around __m128i, so we either need to // cast to Packet4i to directly call the intrinsics as below: return _mm_or_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet8h pxor(const Packet8h& a,const Packet8h& b) { return _mm_xor_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet8h pand(const Packet8h& a,const Packet8h& b) { return _mm_and_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet8h pandnot(const Packet8h& a,const Packet8h& b) { return _mm_andnot_si128(b,a); } template<> EIGEN_STRONG_INLINE Packet8h pselect(const Packet8h& mask, const Packet8h& a, const Packet8h& b) { return _mm_blendv_epi8(b, a, mask); } template<> EIGEN_STRONG_INLINE Packet8h pround(const Packet8h& a) { return float2half(pround(half2float(a))); } template<> EIGEN_STRONG_INLINE Packet8h print(const Packet8h& a) { return float2half(print(half2float(a))); } template<> EIGEN_STRONG_INLINE Packet8h pceil(const Packet8h& a) { return float2half(pceil(half2float(a))); } template<> EIGEN_STRONG_INLINE Packet8h pfloor(const Packet8h& a) { return float2half(pfloor(half2float(a))); } template<> EIGEN_STRONG_INLINE Packet8h pcmp_eq(const Packet8h& a,const Packet8h& b) { return Pack16To8(pcmp_eq(half2float(a), half2float(b))); } template<> EIGEN_STRONG_INLINE Packet8h pcmp_le(const Packet8h& a,const Packet8h& b) { return Pack16To8(pcmp_le(half2float(a), half2float(b))); } template<> EIGEN_STRONG_INLINE Packet8h pcmp_lt(const Packet8h& a,const Packet8h& b) { return Pack16To8(pcmp_lt(half2float(a), half2float(b))); } template<> EIGEN_STRONG_INLINE Packet8h pcmp_lt_or_nan(const Packet8h& a,const Packet8h& b) { return Pack16To8(pcmp_lt_or_nan(half2float(a), half2float(b))); } template<> EIGEN_STRONG_INLINE Packet8h pconj(const Packet8h& a) { return a; } template<> EIGEN_STRONG_INLINE Packet8h pnegate(const Packet8h& a) { Packet8h sign_mask = _mm_set1_epi16(static_cast(0x8000)); return _mm_xor_si128(a, sign_mask); } template<> EIGEN_STRONG_INLINE Packet8h padd(const Packet8h& a, const Packet8h& b) { Packet8f af = half2float(a); Packet8f bf = half2float(b); Packet8f rf = padd(af, bf); return float2half(rf); } template<> EIGEN_STRONG_INLINE Packet8h psub(const Packet8h& a, const Packet8h& b) { Packet8f af = half2float(a); Packet8f bf = half2float(b); Packet8f rf = psub(af, bf); return float2half(rf); } template<> EIGEN_STRONG_INLINE Packet8h pmul(const Packet8h& a, const Packet8h& b) { Packet8f af = half2float(a); Packet8f bf = half2float(b); Packet8f rf = pmul(af, bf); return float2half(rf); } template<> EIGEN_STRONG_INLINE Packet8h pdiv(const Packet8h& a, const Packet8h& b) { Packet8f af = half2float(a); Packet8f bf = half2float(b); Packet8f rf = pdiv(af, bf); return float2half(rf); } template<> EIGEN_STRONG_INLINE Packet8h pgather(const Eigen::half* from, Index stride) { const numext::uint16_t s0 = numext::bit_cast(from[0*stride]); const numext::uint16_t s1 = numext::bit_cast(from[1*stride]); const numext::uint16_t s2 = numext::bit_cast(from[2*stride]); const numext::uint16_t s3 = numext::bit_cast(from[3*stride]); const numext::uint16_t s4 = numext::bit_cast(from[4*stride]); const numext::uint16_t s5 = numext::bit_cast(from[5*stride]); const numext::uint16_t s6 = numext::bit_cast(from[6*stride]); const numext::uint16_t s7 = numext::bit_cast(from[7*stride]); return _mm_set_epi16(s7, s6, s5, s4, s3, s2, s1, s0); } template<> EIGEN_STRONG_INLINE void pscatter(Eigen::half* to, const Packet8h& from, Index stride) { EIGEN_ALIGN32 Eigen::half aux[8]; pstore(aux, from); to[stride*0] = aux[0]; to[stride*1] = aux[1]; to[stride*2] = aux[2]; to[stride*3] = aux[3]; to[stride*4] = aux[4]; to[stride*5] = aux[5]; to[stride*6] = aux[6]; to[stride*7] = aux[7]; } template<> EIGEN_STRONG_INLINE Eigen::half predux(const Packet8h& a) { Packet8f af = half2float(a); float reduced = predux(af); return Eigen::half(reduced); } template<> EIGEN_STRONG_INLINE Eigen::half predux_max(const Packet8h& a) { Packet8f af = half2float(a); float reduced = predux_max(af); return Eigen::half(reduced); } template<> EIGEN_STRONG_INLINE Eigen::half predux_min(const Packet8h& a) { Packet8f af = half2float(a); float reduced = predux_min(af); return Eigen::half(reduced); } template<> EIGEN_STRONG_INLINE Eigen::half predux_mul(const Packet8h& a) { Packet8f af = half2float(a); float reduced = predux_mul(af); return Eigen::half(reduced); } template<> EIGEN_STRONG_INLINE Packet8h preverse(const Packet8h& a) { __m128i m = _mm_setr_epi8(14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1); return _mm_shuffle_epi8(a,m); } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { __m128i a = kernel.packet[0]; __m128i b = kernel.packet[1]; __m128i c = kernel.packet[2]; __m128i d = kernel.packet[3]; __m128i e = kernel.packet[4]; __m128i f = kernel.packet[5]; __m128i g = kernel.packet[6]; __m128i h = kernel.packet[7]; __m128i a03b03 = _mm_unpacklo_epi16(a, b); __m128i c03d03 = _mm_unpacklo_epi16(c, d); __m128i e03f03 = _mm_unpacklo_epi16(e, f); __m128i g03h03 = _mm_unpacklo_epi16(g, h); __m128i a47b47 = _mm_unpackhi_epi16(a, b); __m128i c47d47 = _mm_unpackhi_epi16(c, d); __m128i e47f47 = _mm_unpackhi_epi16(e, f); __m128i g47h47 = _mm_unpackhi_epi16(g, h); __m128i a01b01c01d01 = _mm_unpacklo_epi32(a03b03, c03d03); __m128i a23b23c23d23 = _mm_unpackhi_epi32(a03b03, c03d03); __m128i e01f01g01h01 = _mm_unpacklo_epi32(e03f03, g03h03); __m128i e23f23g23h23 = _mm_unpackhi_epi32(e03f03, g03h03); __m128i a45b45c45d45 = _mm_unpacklo_epi32(a47b47, c47d47); __m128i a67b67c67d67 = _mm_unpackhi_epi32(a47b47, c47d47); __m128i e45f45g45h45 = _mm_unpacklo_epi32(e47f47, g47h47); __m128i e67f67g67h67 = _mm_unpackhi_epi32(e47f47, g47h47); __m128i a0b0c0d0e0f0g0h0 = _mm_unpacklo_epi64(a01b01c01d01, e01f01g01h01); __m128i a1b1c1d1e1f1g1h1 = _mm_unpackhi_epi64(a01b01c01d01, e01f01g01h01); __m128i a2b2c2d2e2f2g2h2 = _mm_unpacklo_epi64(a23b23c23d23, e23f23g23h23); __m128i a3b3c3d3e3f3g3h3 = _mm_unpackhi_epi64(a23b23c23d23, e23f23g23h23); __m128i a4b4c4d4e4f4g4h4 = _mm_unpacklo_epi64(a45b45c45d45, e45f45g45h45); __m128i a5b5c5d5e5f5g5h5 = _mm_unpackhi_epi64(a45b45c45d45, e45f45g45h45); __m128i a6b6c6d6e6f6g6h6 = _mm_unpacklo_epi64(a67b67c67d67, e67f67g67h67); __m128i a7b7c7d7e7f7g7h7 = _mm_unpackhi_epi64(a67b67c67d67, e67f67g67h67); kernel.packet[0] = a0b0c0d0e0f0g0h0; kernel.packet[1] = a1b1c1d1e1f1g1h1; kernel.packet[2] = a2b2c2d2e2f2g2h2; kernel.packet[3] = a3b3c3d3e3f3g3h3; kernel.packet[4] = a4b4c4d4e4f4g4h4; kernel.packet[5] = a5b5c5d5e5f5g5h5; kernel.packet[6] = a6b6c6d6e6f6g6h6; kernel.packet[7] = a7b7c7d7e7f7g7h7; } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { EIGEN_ALIGN32 Eigen::half in[4][8]; pstore(in[0], kernel.packet[0]); pstore(in[1], kernel.packet[1]); pstore(in[2], kernel.packet[2]); pstore(in[3], kernel.packet[3]); EIGEN_ALIGN32 Eigen::half out[4][8]; for (int i = 0; i < 4; ++i) { for (int j = 0; j < 4; ++j) { out[i][j] = in[j][2*i]; } for (int j = 0; j < 4; ++j) { out[i][j+4] = in[j][2*i+1]; } } kernel.packet[0] = pload(out[0]); kernel.packet[1] = pload(out[1]); kernel.packet[2] = pload(out[2]); kernel.packet[3] = pload(out[3]); } // BFloat16 implementation. EIGEN_STRONG_INLINE Packet8f Bf16ToF32(const Packet8bf& a) { #ifdef EIGEN_VECTORIZE_AVX2 __m256i extend = _mm256_cvtepu16_epi32(a); return _mm256_castsi256_ps(_mm256_slli_epi32(extend, 16)); #else __m128i lo = _mm_cvtepu16_epi32(a); __m128i hi = _mm_cvtepu16_epi32(_mm_srli_si128(a, 8)); __m128i lo_shift = _mm_slli_epi32(lo, 16); __m128i hi_shift = _mm_slli_epi32(hi, 16); return _mm256_castsi256_ps(_mm256_insertf128_si256(_mm256_castsi128_si256(lo_shift), hi_shift, 1)); #endif } // Convert float to bfloat16 according to round-to-nearest-even/denormals algorithm. EIGEN_STRONG_INLINE Packet8bf F32ToBf16(const Packet8f& a) { Packet8bf r; __m256i input = _mm256_castps_si256(a); #ifdef EIGEN_VECTORIZE_AVX2 // uint32_t lsb = (input >> 16); __m256i t = _mm256_srli_epi32(input, 16); // uint32_t lsb = lsb & 1; t = _mm256_and_si256(t, _mm256_set1_epi32(1)); // uint32_t rounding_bias = 0x7fff + lsb; t = _mm256_add_epi32(t, _mm256_set1_epi32(0x7fff)); // input += rounding_bias; t = _mm256_add_epi32(t, input); // input = input >> 16; t = _mm256_srli_epi32(t, 16); // Check NaN before converting back to bf16 __m256 mask = _mm256_cmp_ps(a, a, _CMP_ORD_Q); __m256i nan = _mm256_set1_epi32(0x7fc0); t = _mm256_blendv_epi8(nan, t, _mm256_castps_si256(mask)); // output = numext::bit_cast(input); return _mm_packus_epi32(_mm256_extractf128_si256(t, 0), _mm256_extractf128_si256(t, 1)); #else // uint32_t lsb = (input >> 16); __m128i lo = _mm_srli_epi32(_mm256_extractf128_si256(input, 0), 16); __m128i hi = _mm_srli_epi32(_mm256_extractf128_si256(input, 1), 16); // uint32_t lsb = lsb & 1; lo = _mm_and_si128(lo, _mm_set1_epi32(1)); hi = _mm_and_si128(hi, _mm_set1_epi32(1)); // uint32_t rounding_bias = 0x7fff + lsb; lo = _mm_add_epi32(lo, _mm_set1_epi32(0x7fff)); hi = _mm_add_epi32(hi, _mm_set1_epi32(0x7fff)); // input += rounding_bias; lo = _mm_add_epi32(lo, _mm256_extractf128_si256(input, 0)); hi = _mm_add_epi32(hi, _mm256_extractf128_si256(input, 1)); // input = input >> 16; lo = _mm_srli_epi32(lo, 16); hi = _mm_srli_epi32(hi, 16); // Check NaN before converting back to bf16 __m256 mask = _mm256_cmp_ps(a, a, _CMP_ORD_Q); __m128i nan = _mm_set1_epi32(0x7fc0); lo = _mm_blendv_epi8(nan, lo, _mm_castps_si128(_mm256_castps256_ps128(mask))); hi = _mm_blendv_epi8(nan, hi, _mm_castps_si128(_mm256_extractf128_ps(mask, 1))); // output = numext::bit_cast(input); return _mm_packus_epi32(lo, hi); #endif } template<> EIGEN_STRONG_INLINE Packet8bf pset1(const bfloat16& from) { return _mm_set1_epi16(numext::bit_cast(from)); } template<> EIGEN_STRONG_INLINE bfloat16 pfirst(const Packet8bf& from) { return numext::bit_cast(static_cast(_mm_extract_epi16(from, 0))); } template<> EIGEN_STRONG_INLINE Packet8bf pload(const bfloat16* from) { return _mm_load_si128(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE Packet8bf ploadu(const bfloat16* from) { return _mm_loadu_si128(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE void pstore(bfloat16* to, const Packet8bf& from) { _mm_store_si128(reinterpret_cast<__m128i*>(to), from); } template<> EIGEN_STRONG_INLINE void pstoreu(bfloat16* to, const Packet8bf& from) { _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from); } template<> EIGEN_STRONG_INLINE Packet8bf ploaddup(const bfloat16* from) { const numext::uint16_t a = numext::bit_cast(from[0]); const numext::uint16_t b = numext::bit_cast(from[1]); const numext::uint16_t c = numext::bit_cast(from[2]); const numext::uint16_t d = numext::bit_cast(from[3]); return _mm_set_epi16(d, d, c, c, b, b, a, a); } template<> EIGEN_STRONG_INLINE Packet8bf ploadquad(const bfloat16* from) { const numext::uint16_t a = numext::bit_cast(from[0]); const numext::uint16_t b = numext::bit_cast(from[1]); return _mm_set_epi16(b, b, b, b, a, a, a, a); } template<> EIGEN_STRONG_INLINE Packet8bf ptrue(const Packet8bf& a) { return _mm_cmpeq_epi32(a, a); } template <> EIGEN_STRONG_INLINE Packet8bf pabs(const Packet8bf& a) { const __m128i sign_mask = _mm_set1_epi16(static_cast(0x8000)); return _mm_andnot_si128(sign_mask, a); } template <> EIGEN_STRONG_INLINE Packet8bf pmin(const Packet8bf& a, const Packet8bf& b) { return F32ToBf16(pmin(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet8bf pmax(const Packet8bf& a, const Packet8bf& b) { return F32ToBf16(pmax(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet8bf plset(const bfloat16& a) { return F32ToBf16(plset(static_cast(a))); } template<> EIGEN_STRONG_INLINE Packet8bf por(const Packet8bf& a,const Packet8bf& b) { return _mm_or_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet8bf pxor(const Packet8bf& a,const Packet8bf& b) { return _mm_xor_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet8bf pand(const Packet8bf& a,const Packet8bf& b) { return _mm_and_si128(a,b); } template<> EIGEN_STRONG_INLINE Packet8bf pandnot(const Packet8bf& a,const Packet8bf& b) { return _mm_andnot_si128(b,a); } template<> EIGEN_STRONG_INLINE Packet8bf pselect(const Packet8bf& mask, const Packet8bf& a, const Packet8bf& b) { return _mm_blendv_epi8(b, a, mask); } template<> EIGEN_STRONG_INLINE Packet8bf pround(const Packet8bf& a) { return F32ToBf16(pround(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE Packet8bf print(const Packet8bf& a) { return F32ToBf16(print(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE Packet8bf pceil(const Packet8bf& a) { return F32ToBf16(pceil(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE Packet8bf pfloor(const Packet8bf& a) { return F32ToBf16(pfloor(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE Packet8bf pcmp_eq(const Packet8bf& a,const Packet8bf& b) { return Pack16To8(pcmp_eq(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet8bf pcmp_le(const Packet8bf& a,const Packet8bf& b) { return Pack16To8(pcmp_le(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet8bf pcmp_lt(const Packet8bf& a,const Packet8bf& b) { return Pack16To8(pcmp_lt(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet8bf pcmp_lt_or_nan(const Packet8bf& a,const Packet8bf& b) { return Pack16To8(pcmp_lt_or_nan(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet8bf pconj(const Packet8bf& a) { return a; } template<> EIGEN_STRONG_INLINE Packet8bf pnegate(const Packet8bf& a) { Packet8bf sign_mask = _mm_set1_epi16(static_cast(0x8000)); return _mm_xor_si128(a, sign_mask); } template<> EIGEN_STRONG_INLINE Packet8bf padd(const Packet8bf& a, const Packet8bf& b) { return F32ToBf16(padd(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet8bf psub(const Packet8bf& a, const Packet8bf& b) { return F32ToBf16(psub(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet8bf pmul(const Packet8bf& a, const Packet8bf& b) { return F32ToBf16(pmul(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet8bf pdiv(const Packet8bf& a, const Packet8bf& b) { return F32ToBf16(pdiv(Bf16ToF32(a), Bf16ToF32(b))); } template<> EIGEN_STRONG_INLINE Packet8bf pgather(const bfloat16* from, Index stride) { const numext::uint16_t s0 = numext::bit_cast(from[0*stride]); const numext::uint16_t s1 = numext::bit_cast(from[1*stride]); const numext::uint16_t s2 = numext::bit_cast(from[2*stride]); const numext::uint16_t s3 = numext::bit_cast(from[3*stride]); const numext::uint16_t s4 = numext::bit_cast(from[4*stride]); const numext::uint16_t s5 = numext::bit_cast(from[5*stride]); const numext::uint16_t s6 = numext::bit_cast(from[6*stride]); const numext::uint16_t s7 = numext::bit_cast(from[7*stride]); return _mm_set_epi16(s7, s6, s5, s4, s3, s2, s1, s0); } template<> EIGEN_STRONG_INLINE void pscatter(bfloat16* to, const Packet8bf& from, Index stride) { EIGEN_ALIGN32 bfloat16 aux[8]; pstore(aux, from); to[stride*0] = aux[0]; to[stride*1] = aux[1]; to[stride*2] = aux[2]; to[stride*3] = aux[3]; to[stride*4] = aux[4]; to[stride*5] = aux[5]; to[stride*6] = aux[6]; to[stride*7] = aux[7]; } template<> EIGEN_STRONG_INLINE bfloat16 predux(const Packet8bf& a) { return static_cast(predux(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE bfloat16 predux_max(const Packet8bf& a) { return static_cast(predux_max(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE bfloat16 predux_min(const Packet8bf& a) { return static_cast(predux_min(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE bfloat16 predux_mul(const Packet8bf& a) { return static_cast(predux_mul(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE Packet8bf preverse(const Packet8bf& a) { __m128i m = _mm_setr_epi8(14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1); return _mm_shuffle_epi8(a,m); } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { __m128i a = kernel.packet[0]; __m128i b = kernel.packet[1]; __m128i c = kernel.packet[2]; __m128i d = kernel.packet[3]; __m128i e = kernel.packet[4]; __m128i f = kernel.packet[5]; __m128i g = kernel.packet[6]; __m128i h = kernel.packet[7]; __m128i a03b03 = _mm_unpacklo_epi16(a, b); __m128i c03d03 = _mm_unpacklo_epi16(c, d); __m128i e03f03 = _mm_unpacklo_epi16(e, f); __m128i g03h03 = _mm_unpacklo_epi16(g, h); __m128i a47b47 = _mm_unpackhi_epi16(a, b); __m128i c47d47 = _mm_unpackhi_epi16(c, d); __m128i e47f47 = _mm_unpackhi_epi16(e, f); __m128i g47h47 = _mm_unpackhi_epi16(g, h); __m128i a01b01c01d01 = _mm_unpacklo_epi32(a03b03, c03d03); __m128i a23b23c23d23 = _mm_unpackhi_epi32(a03b03, c03d03); __m128i e01f01g01h01 = _mm_unpacklo_epi32(e03f03, g03h03); __m128i e23f23g23h23 = _mm_unpackhi_epi32(e03f03, g03h03); __m128i a45b45c45d45 = _mm_unpacklo_epi32(a47b47, c47d47); __m128i a67b67c67d67 = _mm_unpackhi_epi32(a47b47, c47d47); __m128i e45f45g45h45 = _mm_unpacklo_epi32(e47f47, g47h47); __m128i e67f67g67h67 = _mm_unpackhi_epi32(e47f47, g47h47); kernel.packet[0] = _mm_unpacklo_epi64(a01b01c01d01, e01f01g01h01); kernel.packet[1] = _mm_unpackhi_epi64(a01b01c01d01, e01f01g01h01); kernel.packet[2] = _mm_unpacklo_epi64(a23b23c23d23, e23f23g23h23); kernel.packet[3] = _mm_unpackhi_epi64(a23b23c23d23, e23f23g23h23); kernel.packet[4] = _mm_unpacklo_epi64(a45b45c45d45, e45f45g45h45); kernel.packet[5] = _mm_unpackhi_epi64(a45b45c45d45, e45f45g45h45); kernel.packet[6] = _mm_unpacklo_epi64(a67b67c67d67, e67f67g67h67); kernel.packet[7] = _mm_unpackhi_epi64(a67b67c67d67, e67f67g67h67); } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { __m128i a = kernel.packet[0]; __m128i b = kernel.packet[1]; __m128i c = kernel.packet[2]; __m128i d = kernel.packet[3]; __m128i ab_03 = _mm_unpacklo_epi16(a, b); __m128i cd_03 = _mm_unpacklo_epi16(c, d); __m128i ab_47 = _mm_unpackhi_epi16(a, b); __m128i cd_47 = _mm_unpackhi_epi16(c, d); kernel.packet[0] = _mm_unpacklo_epi32(ab_03, cd_03); kernel.packet[1] = _mm_unpackhi_epi32(ab_03, cd_03); kernel.packet[2] = _mm_unpacklo_epi32(ab_47, cd_47); kernel.packet[3] = _mm_unpackhi_epi32(ab_47, cd_47); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_PACKET_MATH_AVX_H RcppEigen/inst/include/Eigen/src/Core/arch/AVX/MathFunctions.h0000644000176200001440000001764614567757725023611 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com) // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_MATH_FUNCTIONS_AVX_H #define EIGEN_MATH_FUNCTIONS_AVX_H /* The sin and cos functions of this file are loosely derived from * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/ */ namespace Eigen { namespace internal { template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f psin(const Packet8f& _x) { return psin_float(_x); } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f pcos(const Packet8f& _x) { return pcos_float(_x); } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f plog(const Packet8f& _x) { return plog_float(_x); } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4d plog(const Packet4d& _x) { return plog_double(_x); } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f plog2(const Packet8f& _x) { return plog2_float(_x); } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4d plog2(const Packet4d& _x) { return plog2_double(_x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f plog1p(const Packet8f& _x) { return generic_plog1p(_x); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f pexpm1(const Packet8f& _x) { return generic_expm1(_x); } // Exponential function. Works by writing "x = m*log(2) + r" where // "m = floor(x/log(2)+1/2)" and "r" is the remainder. The result is then // "exp(x) = 2^m*exp(r)" where exp(r) is in the range [-1,1). template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f pexp(const Packet8f& _x) { return pexp_float(_x); } // Hyperbolic Tangent function. template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f ptanh(const Packet8f& _x) { return internal::generic_fast_tanh_float(_x); } // Exponential function for doubles. template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4d pexp(const Packet4d& _x) { return pexp_double(_x); } // Functions for sqrt. // The EIGEN_FAST_MATH version uses the _mm_rsqrt_ps approximation and one step // of Newton's method, at a cost of 1-2 bits of precision as opposed to the // exact solution. It does not handle +inf, or denormalized numbers correctly. // The main advantage of this approach is not just speed, but also the fact that // it can be inlined and pipelined with other computations, further reducing its // effective latency. This is similar to Quake3's fast inverse square root. // For detail see here: http://www.beyond3d.com/content/articles/8/ #if EIGEN_FAST_MATH template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f psqrt(const Packet8f& _x) { Packet8f minus_half_x = pmul(_x, pset1(-0.5f)); Packet8f denormal_mask = pandnot( pcmp_lt(_x, pset1((std::numeric_limits::min)())), pcmp_lt(_x, pzero(_x))); // Compute approximate reciprocal sqrt. Packet8f x = _mm256_rsqrt_ps(_x); // Do a single step of Newton's iteration. x = pmul(x, pmadd(minus_half_x, pmul(x,x), pset1(1.5f))); // Flush results for denormals to zero. return pandnot(pmul(_x,x), denormal_mask); } #else template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f psqrt(const Packet8f& _x) { return _mm256_sqrt_ps(_x); } #endif template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4d psqrt(const Packet4d& _x) { return _mm256_sqrt_pd(_x); } #if EIGEN_FAST_MATH template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f prsqrt(const Packet8f& _x) { _EIGEN_DECLARE_CONST_Packet8f_FROM_INT(inf, 0x7f800000); _EIGEN_DECLARE_CONST_Packet8f(one_point_five, 1.5f); _EIGEN_DECLARE_CONST_Packet8f(minus_half, -0.5f); _EIGEN_DECLARE_CONST_Packet8f_FROM_INT(flt_min, 0x00800000); Packet8f neg_half = pmul(_x, p8f_minus_half); // select only the inverse sqrt of positive normal inputs (denormals are // flushed to zero and cause infs as well). Packet8f lt_min_mask = _mm256_cmp_ps(_x, p8f_flt_min, _CMP_LT_OQ); Packet8f inf_mask = _mm256_cmp_ps(_x, p8f_inf, _CMP_EQ_OQ); Packet8f not_normal_finite_mask = _mm256_or_ps(lt_min_mask, inf_mask); // Compute an approximate result using the rsqrt intrinsic. Packet8f y_approx = _mm256_rsqrt_ps(_x); // Do a single step of Newton-Raphson iteration to improve the approximation. // This uses the formula y_{n+1} = y_n * (1.5 - y_n * (0.5 * x) * y_n). // It is essential to evaluate the inner term like this because forming // y_n^2 may over- or underflow. Packet8f y_newton = pmul(y_approx, pmadd(y_approx, pmul(neg_half, y_approx), p8f_one_point_five)); // Select the result of the Newton-Raphson step for positive normal arguments. // For other arguments, choose the output of the intrinsic. This will // return rsqrt(+inf) = 0, rsqrt(x) = NaN if x < 0, and rsqrt(x) = +inf if // x is zero or a positive denormalized float (equivalent to flushing positive // denormalized inputs to zero). return pselect(not_normal_finite_mask, y_approx, y_newton); } #else template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8f prsqrt(const Packet8f& _x) { _EIGEN_DECLARE_CONST_Packet8f(one, 1.0f); return _mm256_div_ps(p8f_one, _mm256_sqrt_ps(_x)); } #endif template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4d prsqrt(const Packet4d& _x) { _EIGEN_DECLARE_CONST_Packet4d(one, 1.0); return _mm256_div_pd(p4d_one, _mm256_sqrt_pd(_x)); } F16_PACKET_FUNCTION(Packet8f, Packet8h, psin) F16_PACKET_FUNCTION(Packet8f, Packet8h, pcos) F16_PACKET_FUNCTION(Packet8f, Packet8h, plog) F16_PACKET_FUNCTION(Packet8f, Packet8h, plog2) F16_PACKET_FUNCTION(Packet8f, Packet8h, plog1p) F16_PACKET_FUNCTION(Packet8f, Packet8h, pexpm1) F16_PACKET_FUNCTION(Packet8f, Packet8h, pexp) F16_PACKET_FUNCTION(Packet8f, Packet8h, ptanh) F16_PACKET_FUNCTION(Packet8f, Packet8h, psqrt) F16_PACKET_FUNCTION(Packet8f, Packet8h, prsqrt) template <> EIGEN_STRONG_INLINE Packet8h pfrexp(const Packet8h& a, Packet8h& exponent) { Packet8f fexponent; const Packet8h out = float2half(pfrexp(half2float(a), fexponent)); exponent = float2half(fexponent); return out; } template <> EIGEN_STRONG_INLINE Packet8h pldexp(const Packet8h& a, const Packet8h& exponent) { return float2half(pldexp(half2float(a), half2float(exponent))); } BF16_PACKET_FUNCTION(Packet8f, Packet8bf, psin) BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pcos) BF16_PACKET_FUNCTION(Packet8f, Packet8bf, plog) BF16_PACKET_FUNCTION(Packet8f, Packet8bf, plog2) BF16_PACKET_FUNCTION(Packet8f, Packet8bf, plog1p) BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pexpm1) BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pexp) BF16_PACKET_FUNCTION(Packet8f, Packet8bf, ptanh) BF16_PACKET_FUNCTION(Packet8f, Packet8bf, psqrt) BF16_PACKET_FUNCTION(Packet8f, Packet8bf, prsqrt) template <> EIGEN_STRONG_INLINE Packet8bf pfrexp(const Packet8bf& a, Packet8bf& exponent) { Packet8f fexponent; const Packet8bf out = F32ToBf16(pfrexp(Bf16ToF32(a), fexponent)); exponent = F32ToBf16(fexponent); return out; } template <> EIGEN_STRONG_INLINE Packet8bf pldexp(const Packet8bf& a, const Packet8bf& exponent) { return F32ToBf16(pldexp(Bf16ToF32(a), Bf16ToF32(exponent))); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_MATH_FUNCTIONS_AVX_H RcppEigen/inst/include/Eigen/src/Core/arch/AVX512/0000755000176200001440000000000014567757725021070 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/arch/AVX512/TypeCasting.h0000644000176200001440000000412614567757725023476 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2019 Rasmus Munk Larsen // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_TYPE_CASTING_AVX512_H #define EIGEN_TYPE_CASTING_AVX512_H namespace Eigen { namespace internal { template<> EIGEN_STRONG_INLINE Packet16i pcast(const Packet16f& a) { return _mm512_cvttps_epi32(a); } template<> EIGEN_STRONG_INLINE Packet16f pcast(const Packet16i& a) { return _mm512_cvtepi32_ps(a); } template<> EIGEN_STRONG_INLINE Packet16i preinterpret(const Packet16f& a) { return _mm512_castps_si512(a); } template<> EIGEN_STRONG_INLINE Packet16f preinterpret(const Packet16i& a) { return _mm512_castsi512_ps(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template<> EIGEN_STRONG_INLINE Packet16f pcast(const Packet16h& a) { return half2float(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template<> EIGEN_STRONG_INLINE Packet16h pcast(const Packet16f& a) { return float2half(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template<> EIGEN_STRONG_INLINE Packet16f pcast(const Packet16bf& a) { return Bf16ToF32(a); } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template<> EIGEN_STRONG_INLINE Packet16bf pcast(const Packet16f& a) { return F32ToBf16(a); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_TYPE_CASTING_AVX512_H RcppEigen/inst/include/Eigen/src/Core/arch/AVX512/Complex.h0000644000176200001440000004141014567757725022650 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2018 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_COMPLEX_AVX512_H #define EIGEN_COMPLEX_AVX512_H namespace Eigen { namespace internal { //---------- float ---------- struct Packet8cf { EIGEN_STRONG_INLINE Packet8cf() {} EIGEN_STRONG_INLINE explicit Packet8cf(const __m512& a) : v(a) {} __m512 v; }; template<> struct packet_traits > : default_packet_traits { typedef Packet8cf type; typedef Packet4cf half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 8, HasHalfPacket = 1, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, HasSqrt = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, HasMax = 0, HasSetLinear = 0 }; }; template<> struct unpacket_traits { typedef std::complex type; typedef Packet4cf half; typedef Packet16f as_real; enum { size = 8, alignment=unpacket_traits::alignment, vectorizable=true, masked_load_available=false, masked_store_available=false }; }; template<> EIGEN_STRONG_INLINE Packet8cf ptrue(const Packet8cf& a) { return Packet8cf(ptrue(Packet16f(a.v))); } template<> EIGEN_STRONG_INLINE Packet8cf padd(const Packet8cf& a, const Packet8cf& b) { return Packet8cf(_mm512_add_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet8cf psub(const Packet8cf& a, const Packet8cf& b) { return Packet8cf(_mm512_sub_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet8cf pnegate(const Packet8cf& a) { return Packet8cf(pnegate(a.v)); } template<> EIGEN_STRONG_INLINE Packet8cf pconj(const Packet8cf& a) { const __m512 mask = _mm512_castsi512_ps(_mm512_setr_epi32( 0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000, 0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000)); return Packet8cf(pxor(a.v,mask)); } template<> EIGEN_STRONG_INLINE Packet8cf pmul(const Packet8cf& a, const Packet8cf& b) { __m512 tmp2 = _mm512_mul_ps(_mm512_movehdup_ps(a.v), _mm512_permute_ps(b.v, _MM_SHUFFLE(2,3,0,1))); return Packet8cf(_mm512_fmaddsub_ps(_mm512_moveldup_ps(a.v), b.v, tmp2)); } template<> EIGEN_STRONG_INLINE Packet8cf pand (const Packet8cf& a, const Packet8cf& b) { return Packet8cf(pand(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet8cf por (const Packet8cf& a, const Packet8cf& b) { return Packet8cf(por(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet8cf pxor (const Packet8cf& a, const Packet8cf& b) { return Packet8cf(pxor(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet8cf pandnot(const Packet8cf& a, const Packet8cf& b) { return Packet8cf(pandnot(a.v,b.v)); } template <> EIGEN_STRONG_INLINE Packet8cf pcmp_eq(const Packet8cf& a, const Packet8cf& b) { __m512 eq = pcmp_eq(a.v, b.v); return Packet8cf(pand(eq, _mm512_permute_ps(eq, 0xB1))); } template<> EIGEN_STRONG_INLINE Packet8cf pload (const std::complex* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet8cf(pload(&numext::real_ref(*from))); } template<> EIGEN_STRONG_INLINE Packet8cf ploadu(const std::complex* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet8cf(ploadu(&numext::real_ref(*from))); } template<> EIGEN_STRONG_INLINE Packet8cf pset1(const std::complex& from) { return Packet8cf(_mm512_castpd_ps(pload1((const double*)(const void*)&from))); } template<> EIGEN_STRONG_INLINE Packet8cf ploaddup(const std::complex* from) { return Packet8cf( _mm512_castpd_ps( ploaddup((const double*)(const void*)from )) ); } template<> EIGEN_STRONG_INLINE Packet8cf ploadquad(const std::complex* from) { return Packet8cf( _mm512_castpd_ps( ploadquad((const double*)(const void*)from )) ); } template<> EIGEN_STRONG_INLINE void pstore >(std::complex* to, const Packet8cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), from.v); } template<> EIGEN_STRONG_INLINE void pstoreu >(std::complex* to, const Packet8cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), from.v); } template<> EIGEN_DEVICE_FUNC inline Packet8cf pgather, Packet8cf>(const std::complex* from, Index stride) { return Packet8cf(_mm512_castpd_ps(pgather((const double*)(const void*)from, stride))); } template<> EIGEN_DEVICE_FUNC inline void pscatter, Packet8cf>(std::complex* to, const Packet8cf& from, Index stride) { pscatter((double*)(void*)to, _mm512_castps_pd(from.v), stride); } template<> EIGEN_STRONG_INLINE std::complex pfirst(const Packet8cf& a) { return pfirst(Packet2cf(_mm512_castps512_ps128(a.v))); } template<> EIGEN_STRONG_INLINE Packet8cf preverse(const Packet8cf& a) { return Packet8cf(_mm512_castsi512_ps( _mm512_permutexvar_epi64( _mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), _mm512_castps_si512(a.v)))); } template<> EIGEN_STRONG_INLINE std::complex predux(const Packet8cf& a) { return predux(padd(Packet4cf(extract256<0>(a.v)), Packet4cf(extract256<1>(a.v)))); } template<> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet8cf& a) { return predux_mul(pmul(Packet4cf(extract256<0>(a.v)), Packet4cf(extract256<1>(a.v)))); } template <> EIGEN_STRONG_INLINE Packet4cf predux_half_dowto4(const Packet8cf& a) { __m256 lane0 = extract256<0>(a.v); __m256 lane1 = extract256<1>(a.v); __m256 res = _mm256_add_ps(lane0, lane1); return Packet4cf(res); } EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet8cf,Packet16f) template<> EIGEN_STRONG_INLINE Packet8cf pdiv(const Packet8cf& a, const Packet8cf& b) { Packet8cf num = pmul(a, pconj(b)); __m512 tmp = _mm512_mul_ps(b.v, b.v); __m512 tmp2 = _mm512_shuffle_ps(tmp,tmp,0xB1); __m512 denom = _mm512_add_ps(tmp, tmp2); return Packet8cf(_mm512_div_ps(num.v, denom)); } template<> EIGEN_STRONG_INLINE Packet8cf pcplxflip(const Packet8cf& x) { return Packet8cf(_mm512_shuffle_ps(x.v, x.v, _MM_SHUFFLE(2, 3, 0 ,1))); } //---------- double ---------- struct Packet4cd { EIGEN_STRONG_INLINE Packet4cd() {} EIGEN_STRONG_INLINE explicit Packet4cd(const __m512d& a) : v(a) {} __m512d v; }; template<> struct packet_traits > : default_packet_traits { typedef Packet4cd type; typedef Packet2cd half; enum { Vectorizable = 1, AlignedOnScalar = 0, size = 4, HasHalfPacket = 1, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, HasSqrt = 1, HasAbs = 0, HasAbs2 = 0, HasMin = 0, HasMax = 0, HasSetLinear = 0 }; }; template<> struct unpacket_traits { typedef std::complex type; typedef Packet2cd half; typedef Packet8d as_real; enum { size = 4, alignment = unpacket_traits::alignment, vectorizable=true, masked_load_available=false, masked_store_available=false }; }; template<> EIGEN_STRONG_INLINE Packet4cd padd(const Packet4cd& a, const Packet4cd& b) { return Packet4cd(_mm512_add_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet4cd psub(const Packet4cd& a, const Packet4cd& b) { return Packet4cd(_mm512_sub_pd(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet4cd pnegate(const Packet4cd& a) { return Packet4cd(pnegate(a.v)); } template<> EIGEN_STRONG_INLINE Packet4cd pconj(const Packet4cd& a) { const __m512d mask = _mm512_castsi512_pd( _mm512_set_epi32(0x80000000,0x0,0x0,0x0,0x80000000,0x0,0x0,0x0, 0x80000000,0x0,0x0,0x0,0x80000000,0x0,0x0,0x0)); return Packet4cd(pxor(a.v,mask)); } template<> EIGEN_STRONG_INLINE Packet4cd pmul(const Packet4cd& a, const Packet4cd& b) { __m512d tmp1 = _mm512_shuffle_pd(a.v,a.v,0x0); __m512d tmp2 = _mm512_shuffle_pd(a.v,a.v,0xFF); __m512d tmp3 = _mm512_shuffle_pd(b.v,b.v,0x55); __m512d odd = _mm512_mul_pd(tmp2, tmp3); return Packet4cd(_mm512_fmaddsub_pd(tmp1, b.v, odd)); } template<> EIGEN_STRONG_INLINE Packet4cd ptrue(const Packet4cd& a) { return Packet4cd(ptrue(Packet8d(a.v))); } template<> EIGEN_STRONG_INLINE Packet4cd pand (const Packet4cd& a, const Packet4cd& b) { return Packet4cd(pand(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet4cd por (const Packet4cd& a, const Packet4cd& b) { return Packet4cd(por(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet4cd pxor (const Packet4cd& a, const Packet4cd& b) { return Packet4cd(pxor(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet4cd pandnot(const Packet4cd& a, const Packet4cd& b) { return Packet4cd(pandnot(a.v,b.v)); } template <> EIGEN_STRONG_INLINE Packet4cd pcmp_eq(const Packet4cd& a, const Packet4cd& b) { __m512d eq = pcmp_eq(a.v, b.v); return Packet4cd(pand(eq, _mm512_permute_pd(eq, 0x55))); } template<> EIGEN_STRONG_INLINE Packet4cd pload (const std::complex* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet4cd(pload((const double*)from)); } template<> EIGEN_STRONG_INLINE Packet4cd ploadu(const std::complex* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet4cd(ploadu((const double*)from)); } template<> EIGEN_STRONG_INLINE Packet4cd pset1(const std::complex& from) { #ifdef EIGEN_VECTORIZE_AVX512DQ return Packet4cd(_mm512_broadcast_f64x2(pset1(from).v)); #else return Packet4cd(_mm512_castps_pd(_mm512_broadcast_f32x4( _mm_castpd_ps(pset1(from).v)))); #endif } template<> EIGEN_STRONG_INLINE Packet4cd ploaddup(const std::complex* from) { return Packet4cd(_mm512_insertf64x4( _mm512_castpd256_pd512(ploaddup(from).v), ploaddup(from+1).v, 1)); } template<> EIGEN_STRONG_INLINE void pstore >(std::complex * to, const Packet4cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v); } template<> EIGEN_STRONG_INLINE void pstoreu >(std::complex * to, const Packet4cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, from.v); } template<> EIGEN_DEVICE_FUNC inline Packet4cd pgather, Packet4cd>(const std::complex* from, Index stride) { return Packet4cd(_mm512_insertf64x4(_mm512_castpd256_pd512( _mm256_insertf128_pd(_mm256_castpd128_pd256(ploadu(from+0*stride).v), ploadu(from+1*stride).v,1)), _mm256_insertf128_pd(_mm256_castpd128_pd256(ploadu(from+2*stride).v), ploadu(from+3*stride).v,1), 1)); } template<> EIGEN_DEVICE_FUNC inline void pscatter, Packet4cd>(std::complex* to, const Packet4cd& from, Index stride) { __m512i fromi = _mm512_castpd_si512(from.v); double* tod = (double*)(void*)to; _mm_storeu_pd(tod+0*stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi,0)) ); _mm_storeu_pd(tod+2*stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi,1)) ); _mm_storeu_pd(tod+4*stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi,2)) ); _mm_storeu_pd(tod+6*stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi,3)) ); } template<> EIGEN_STRONG_INLINE std::complex pfirst(const Packet4cd& a) { __m128d low = extract128<0>(a.v); EIGEN_ALIGN16 double res[2]; _mm_store_pd(res, low); return std::complex(res[0],res[1]); } template<> EIGEN_STRONG_INLINE Packet4cd preverse(const Packet4cd& a) { return Packet4cd(_mm512_shuffle_f64x2(a.v, a.v, (shuffle_mask<3,2,1,0>::mask))); } template<> EIGEN_STRONG_INLINE std::complex predux(const Packet4cd& a) { return predux(padd(Packet2cd(_mm512_extractf64x4_pd(a.v,0)), Packet2cd(_mm512_extractf64x4_pd(a.v,1)))); } template<> EIGEN_STRONG_INLINE std::complex predux_mul(const Packet4cd& a) { return predux_mul(pmul(Packet2cd(_mm512_extractf64x4_pd(a.v,0)), Packet2cd(_mm512_extractf64x4_pd(a.v,1)))); } template<> struct conj_helper { EIGEN_STRONG_INLINE Packet4cd pmadd(const Packet4cd& x, const Packet4cd& y, const Packet4cd& c) const { return padd(pmul(x,y),c); } EIGEN_STRONG_INLINE Packet4cd pmul(const Packet4cd& a, const Packet4cd& b) const { return internal::pmul(a, pconj(b)); } }; template<> struct conj_helper { EIGEN_STRONG_INLINE Packet4cd pmadd(const Packet4cd& x, const Packet4cd& y, const Packet4cd& c) const { return padd(pmul(x,y),c); } EIGEN_STRONG_INLINE Packet4cd pmul(const Packet4cd& a, const Packet4cd& b) const { return internal::pmul(pconj(a), b); } }; template<> struct conj_helper { EIGEN_STRONG_INLINE Packet4cd pmadd(const Packet4cd& x, const Packet4cd& y, const Packet4cd& c) const { return padd(pmul(x,y),c); } EIGEN_STRONG_INLINE Packet4cd pmul(const Packet4cd& a, const Packet4cd& b) const { return pconj(internal::pmul(a, b)); } }; EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet4cd,Packet8d) template<> EIGEN_STRONG_INLINE Packet4cd pdiv(const Packet4cd& a, const Packet4cd& b) { Packet4cd num = pmul(a, pconj(b)); __m512d tmp = _mm512_mul_pd(b.v, b.v); __m512d denom = padd(_mm512_permute_pd(tmp,0x55), tmp); return Packet4cd(_mm512_div_pd(num.v, denom)); } template<> EIGEN_STRONG_INLINE Packet4cd pcplxflip(const Packet4cd& x) { return Packet4cd(_mm512_permute_pd(x.v,0x55)); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { PacketBlock pb; pb.packet[0] = _mm512_castps_pd(kernel.packet[0].v); pb.packet[1] = _mm512_castps_pd(kernel.packet[1].v); pb.packet[2] = _mm512_castps_pd(kernel.packet[2].v); pb.packet[3] = _mm512_castps_pd(kernel.packet[3].v); ptranspose(pb); kernel.packet[0].v = _mm512_castpd_ps(pb.packet[0]); kernel.packet[1].v = _mm512_castpd_ps(pb.packet[1]); kernel.packet[2].v = _mm512_castpd_ps(pb.packet[2]); kernel.packet[3].v = _mm512_castpd_ps(pb.packet[3]); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { PacketBlock pb; pb.packet[0] = _mm512_castps_pd(kernel.packet[0].v); pb.packet[1] = _mm512_castps_pd(kernel.packet[1].v); pb.packet[2] = _mm512_castps_pd(kernel.packet[2].v); pb.packet[3] = _mm512_castps_pd(kernel.packet[3].v); pb.packet[4] = _mm512_castps_pd(kernel.packet[4].v); pb.packet[5] = _mm512_castps_pd(kernel.packet[5].v); pb.packet[6] = _mm512_castps_pd(kernel.packet[6].v); pb.packet[7] = _mm512_castps_pd(kernel.packet[7].v); ptranspose(pb); kernel.packet[0].v = _mm512_castpd_ps(pb.packet[0]); kernel.packet[1].v = _mm512_castpd_ps(pb.packet[1]); kernel.packet[2].v = _mm512_castpd_ps(pb.packet[2]); kernel.packet[3].v = _mm512_castpd_ps(pb.packet[3]); kernel.packet[4].v = _mm512_castpd_ps(pb.packet[4]); kernel.packet[5].v = _mm512_castpd_ps(pb.packet[5]); kernel.packet[6].v = _mm512_castpd_ps(pb.packet[6]); kernel.packet[7].v = _mm512_castpd_ps(pb.packet[7]); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { __m512d T0 = _mm512_shuffle_f64x2(kernel.packet[0].v, kernel.packet[1].v, (shuffle_mask<0,1,0,1>::mask)); // [a0 a1 b0 b1] __m512d T1 = _mm512_shuffle_f64x2(kernel.packet[0].v, kernel.packet[1].v, (shuffle_mask<2,3,2,3>::mask)); // [a2 a3 b2 b3] __m512d T2 = _mm512_shuffle_f64x2(kernel.packet[2].v, kernel.packet[3].v, (shuffle_mask<0,1,0,1>::mask)); // [c0 c1 d0 d1] __m512d T3 = _mm512_shuffle_f64x2(kernel.packet[2].v, kernel.packet[3].v, (shuffle_mask<2,3,2,3>::mask)); // [c2 c3 d2 d3] kernel.packet[3] = Packet4cd(_mm512_shuffle_f64x2(T1, T3, (shuffle_mask<1,3,1,3>::mask))); // [a3 b3 c3 d3] kernel.packet[2] = Packet4cd(_mm512_shuffle_f64x2(T1, T3, (shuffle_mask<0,2,0,2>::mask))); // [a2 b2 c2 d2] kernel.packet[1] = Packet4cd(_mm512_shuffle_f64x2(T0, T2, (shuffle_mask<1,3,1,3>::mask))); // [a1 b1 c1 d1] kernel.packet[0] = Packet4cd(_mm512_shuffle_f64x2(T0, T2, (shuffle_mask<0,2,0,2>::mask))); // [a0 b0 c0 d0] } template<> EIGEN_STRONG_INLINE Packet4cd psqrt(const Packet4cd& a) { return psqrt_complex(a); } template<> EIGEN_STRONG_INLINE Packet8cf psqrt(const Packet8cf& a) { return psqrt_complex(a); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_COMPLEX_AVX512_H RcppEigen/inst/include/Eigen/src/Core/arch/AVX512/PacketMath.h0000644000176200001440000025352314567757725023274 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2016 Benoit Steiner (benoit.steiner.goog@gmail.com) // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_PACKET_MATH_AVX512_H #define EIGEN_PACKET_MATH_AVX512_H namespace Eigen { namespace internal { #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 #endif #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32 #endif #ifdef EIGEN_VECTORIZE_FMA #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD #endif #endif typedef __m512 Packet16f; typedef __m512i Packet16i; typedef __m512d Packet8d; typedef eigen_packet_wrapper<__m256i, 1> Packet16h; typedef eigen_packet_wrapper<__m256i, 2> Packet16bf; template <> struct is_arithmetic<__m512> { enum { value = true }; }; template <> struct is_arithmetic<__m512i> { enum { value = true }; }; template <> struct is_arithmetic<__m512d> { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template <> struct packet_traits : default_packet_traits { typedef Packet16h type; // There is no half-size packet for Packet16h. typedef Packet16h half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 16, HasHalfPacket = 1, HasCmp = 1, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasNegate = 1, HasAbs = 1, HasAbs2 = 0, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasLog = 1, HasLog1p = 1, HasExpm1 = 1, HasExp = 1, HasSqrt = 1, HasRsqrt = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, HasBlend = 0, HasRound = 1, HasFloor = 1, HasCeil = 1, HasRint = 1, HasBessel = 1, HasNdtri = 1 }; }; template<> struct packet_traits : default_packet_traits { typedef Packet16f type; typedef Packet8f half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 16, HasHalfPacket = 1, HasAbs = 1, HasMin = 1, HasMax = 1, HasConj = 1, HasBlend = 0, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, #if EIGEN_GNUC_AT_LEAST(5, 3) || (!EIGEN_COMP_GNUC_STRICT) HasLog = 1, HasLog1p = 1, HasExpm1 = 1, HasNdtri = 1, HasBessel = 1, HasExp = 1, HasSqrt = EIGEN_FAST_MATH, HasRsqrt = EIGEN_FAST_MATH, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, #endif HasCmp = 1, HasDiv = 1, HasRound = 1, HasFloor = 1, HasCeil = 1, HasRint = 1 }; }; template<> struct packet_traits : default_packet_traits { typedef Packet8d type; typedef Packet4d half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 8, HasHalfPacket = 1, #if EIGEN_GNUC_AT_LEAST(5, 3) || (!EIGEN_COMP_GNUC_STRICT) HasLog = 1, HasExp = 1, HasSqrt = EIGEN_FAST_MATH, HasRsqrt = EIGEN_FAST_MATH, #endif HasCmp = 1, HasDiv = 1, HasRound = 1, HasFloor = 1, HasCeil = 1, HasRint = 1 }; }; /* TODO Implement AVX512 for integers template<> struct packet_traits : default_packet_traits { typedef Packet16i type; enum { Vectorizable = 1, AlignedOnScalar = 1, size=8 }; }; */ template <> struct unpacket_traits { typedef float type; typedef Packet8f half; typedef Packet16i integer_packet; typedef uint16_t mask_t; enum { size = 16, alignment=Aligned64, vectorizable=true, masked_load_available=true, masked_store_available=true }; }; template <> struct unpacket_traits { typedef double type; typedef Packet4d half; enum { size = 8, alignment=Aligned64, vectorizable=true, masked_load_available=false, masked_store_available=false }; }; template <> struct unpacket_traits { typedef int type; typedef Packet8i half; enum { size = 16, alignment=Aligned64, vectorizable=false, masked_load_available=false, masked_store_available=false }; }; template<> struct unpacket_traits { typedef Eigen::half type; typedef Packet8h half; enum {size=16, alignment=Aligned32, vectorizable=true, masked_load_available=false, masked_store_available=false}; }; template <> EIGEN_STRONG_INLINE Packet16f pset1(const float& from) { return _mm512_set1_ps(from); } template <> EIGEN_STRONG_INLINE Packet8d pset1(const double& from) { return _mm512_set1_pd(from); } template <> EIGEN_STRONG_INLINE Packet16i pset1(const int& from) { return _mm512_set1_epi32(from); } template <> EIGEN_STRONG_INLINE Packet16f pset1frombits(unsigned int from) { return _mm512_castsi512_ps(_mm512_set1_epi32(from)); } template <> EIGEN_STRONG_INLINE Packet8d pset1frombits(const numext::uint64_t from) { return _mm512_castsi512_pd(_mm512_set1_epi64(from)); } template<> EIGEN_STRONG_INLINE Packet16f pzero(const Packet16f& /*a*/) { return _mm512_setzero_ps(); } template<> EIGEN_STRONG_INLINE Packet8d pzero(const Packet8d& /*a*/) { return _mm512_setzero_pd(); } template<> EIGEN_STRONG_INLINE Packet16i pzero(const Packet16i& /*a*/) { return _mm512_setzero_si512(); } template<> EIGEN_STRONG_INLINE Packet16f peven_mask(const Packet16f& /*a*/) { return _mm512_castsi512_ps(_mm512_set_epi32(0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1)); } template<> EIGEN_STRONG_INLINE Packet16i peven_mask(const Packet16i& /*a*/) { return _mm512_set_epi32(0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1); } template<> EIGEN_STRONG_INLINE Packet8d peven_mask(const Packet8d& /*a*/) { return _mm512_castsi512_pd(_mm512_set_epi32(0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1)); } template <> EIGEN_STRONG_INLINE Packet16f pload1(const float* from) { return _mm512_broadcastss_ps(_mm_load_ps1(from)); } template <> EIGEN_STRONG_INLINE Packet8d pload1(const double* from) { return _mm512_set1_pd(*from); } template <> EIGEN_STRONG_INLINE Packet16f plset(const float& a) { return _mm512_add_ps( _mm512_set1_ps(a), _mm512_set_ps(15.0f, 14.0f, 13.0f, 12.0f, 11.0f, 10.0f, 9.0f, 8.0f, 7.0f, 6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 1.0f, 0.0f)); } template <> EIGEN_STRONG_INLINE Packet8d plset(const double& a) { return _mm512_add_pd(_mm512_set1_pd(a), _mm512_set_pd(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0)); } template <> EIGEN_STRONG_INLINE Packet16f padd(const Packet16f& a, const Packet16f& b) { return _mm512_add_ps(a, b); } template <> EIGEN_STRONG_INLINE Packet8d padd(const Packet8d& a, const Packet8d& b) { return _mm512_add_pd(a, b); } template <> EIGEN_STRONG_INLINE Packet16i padd(const Packet16i& a, const Packet16i& b) { return _mm512_add_epi32(a, b); } template <> EIGEN_STRONG_INLINE Packet16f psub(const Packet16f& a, const Packet16f& b) { return _mm512_sub_ps(a, b); } template <> EIGEN_STRONG_INLINE Packet8d psub(const Packet8d& a, const Packet8d& b) { return _mm512_sub_pd(a, b); } template <> EIGEN_STRONG_INLINE Packet16i psub(const Packet16i& a, const Packet16i& b) { return _mm512_sub_epi32(a, b); } template <> EIGEN_STRONG_INLINE Packet16f pnegate(const Packet16f& a) { return _mm512_sub_ps(_mm512_set1_ps(0.0), a); } template <> EIGEN_STRONG_INLINE Packet8d pnegate(const Packet8d& a) { return _mm512_sub_pd(_mm512_set1_pd(0.0), a); } template <> EIGEN_STRONG_INLINE Packet16f pconj(const Packet16f& a) { return a; } template <> EIGEN_STRONG_INLINE Packet8d pconj(const Packet8d& a) { return a; } template <> EIGEN_STRONG_INLINE Packet16i pconj(const Packet16i& a) { return a; } template <> EIGEN_STRONG_INLINE Packet16f pmul(const Packet16f& a, const Packet16f& b) { return _mm512_mul_ps(a, b); } template <> EIGEN_STRONG_INLINE Packet8d pmul(const Packet8d& a, const Packet8d& b) { return _mm512_mul_pd(a, b); } template <> EIGEN_STRONG_INLINE Packet16i pmul(const Packet16i& a, const Packet16i& b) { return _mm512_mullo_epi32(a, b); } template <> EIGEN_STRONG_INLINE Packet16f pdiv(const Packet16f& a, const Packet16f& b) { return _mm512_div_ps(a, b); } template <> EIGEN_STRONG_INLINE Packet8d pdiv(const Packet8d& a, const Packet8d& b) { return _mm512_div_pd(a, b); } #ifdef EIGEN_VECTORIZE_FMA template <> EIGEN_STRONG_INLINE Packet16f pmadd(const Packet16f& a, const Packet16f& b, const Packet16f& c) { return _mm512_fmadd_ps(a, b, c); } template <> EIGEN_STRONG_INLINE Packet8d pmadd(const Packet8d& a, const Packet8d& b, const Packet8d& c) { return _mm512_fmadd_pd(a, b, c); } #endif template <> EIGEN_DEVICE_FUNC inline Packet16f pselect(const Packet16f& mask, const Packet16f& a, const Packet16f& b) { __mmask16 mask16 = _mm512_cmp_epi32_mask( _mm512_castps_si512(mask), _mm512_setzero_epi32(), _MM_CMPINT_EQ); return _mm512_mask_blend_ps(mask16, a, b); } template <> EIGEN_DEVICE_FUNC inline Packet8d pselect(const Packet8d& mask, const Packet8d& a, const Packet8d& b) { __mmask8 mask8 = _mm512_cmp_epi64_mask(_mm512_castpd_si512(mask), _mm512_setzero_epi32(), _MM_CMPINT_EQ); return _mm512_mask_blend_pd(mask8, a, b); } template <> EIGEN_STRONG_INLINE Packet16f pmin(const Packet16f& a, const Packet16f& b) { // Arguments are reversed to match NaN propagation behavior of std::min. return _mm512_min_ps(b, a); } template <> EIGEN_STRONG_INLINE Packet8d pmin(const Packet8d& a, const Packet8d& b) { // Arguments are reversed to match NaN propagation behavior of std::min. return _mm512_min_pd(b, a); } template <> EIGEN_STRONG_INLINE Packet16f pmax(const Packet16f& a, const Packet16f& b) { // Arguments are reversed to match NaN propagation behavior of std::max. return _mm512_max_ps(b, a); } template <> EIGEN_STRONG_INLINE Packet8d pmax(const Packet8d& a, const Packet8d& b) { // Arguments are reversed to match NaN propagation behavior of std::max. return _mm512_max_pd(b, a); } // Add specializations for min/max with prescribed NaN progation. template<> EIGEN_STRONG_INLINE Packet16f pmin(const Packet16f& a, const Packet16f& b) { return pminmax_propagate_numbers(a, b, pmin); } template<> EIGEN_STRONG_INLINE Packet8d pmin(const Packet8d& a, const Packet8d& b) { return pminmax_propagate_numbers(a, b, pmin); } template<> EIGEN_STRONG_INLINE Packet16f pmax(const Packet16f& a, const Packet16f& b) { return pminmax_propagate_numbers(a, b, pmax); } template<> EIGEN_STRONG_INLINE Packet8d pmax(const Packet8d& a, const Packet8d& b) { return pminmax_propagate_numbers(a, b, pmax); } template<> EIGEN_STRONG_INLINE Packet16f pmin(const Packet16f& a, const Packet16f& b) { return pminmax_propagate_nan(a, b, pmin); } template<> EIGEN_STRONG_INLINE Packet8d pmin(const Packet8d& a, const Packet8d& b) { return pminmax_propagate_nan(a, b, pmin); } template<> EIGEN_STRONG_INLINE Packet16f pmax(const Packet16f& a, const Packet16f& b) { return pminmax_propagate_nan(a, b, pmax); } template<> EIGEN_STRONG_INLINE Packet8d pmax(const Packet8d& a, const Packet8d& b) { return pminmax_propagate_nan(a, b, pmax); } #ifdef EIGEN_VECTORIZE_AVX512DQ template EIGEN_STRONG_INLINE Packet8f extract256(Packet16f x) { return _mm512_extractf32x8_ps(x,I_); } template EIGEN_STRONG_INLINE Packet2d extract128(Packet8d x) { return _mm512_extractf64x2_pd(x,I_); } EIGEN_STRONG_INLINE Packet16f cat256(Packet8f a, Packet8f b) { return _mm512_insertf32x8(_mm512_castps256_ps512(a),b,1); } #else // AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512 template EIGEN_STRONG_INLINE Packet8f extract256(Packet16f x) { return _mm256_castsi256_ps(_mm512_extracti64x4_epi64( _mm512_castps_si512(x),I_)); } // AVX512F does not define _mm512_extractf64x2_pd to extract _m128 from _m512 template EIGEN_STRONG_INLINE Packet2d extract128(Packet8d x) { return _mm_castsi128_pd(_mm512_extracti32x4_epi32( _mm512_castpd_si512(x),I_)); } EIGEN_STRONG_INLINE Packet16f cat256(Packet8f a, Packet8f b) { return _mm512_castsi512_ps(_mm512_inserti64x4(_mm512_castsi256_si512(_mm256_castps_si256(a)), _mm256_castps_si256(b),1)); } #endif // Helper function for bit packing snippet of low precision comparison. // It packs the flags from 32x16 to 16x16. EIGEN_STRONG_INLINE __m256i Pack32To16(Packet16f rf) { // Split data into small pieces and handle with AVX instructions // to guarantee internal order of vector. // Operation: // dst[15:0] := Saturate16(rf[31:0]) // dst[31:16] := Saturate16(rf[63:32]) // ... // dst[255:240] := Saturate16(rf[255:224]) __m256i lo = _mm256_castps_si256(extract256<0>(rf)); __m256i hi = _mm256_castps_si256(extract256<1>(rf)); __m128i result_lo = _mm_packs_epi32(_mm256_extractf128_si256(lo, 0), _mm256_extractf128_si256(lo, 1)); __m128i result_hi = _mm_packs_epi32(_mm256_extractf128_si256(hi, 0), _mm256_extractf128_si256(hi, 1)); return _mm256_insertf128_si256(_mm256_castsi128_si256(result_lo), result_hi, 1); } template <> EIGEN_STRONG_INLINE Packet16f pcmp_eq(const Packet16f& a, const Packet16f& b) { __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_EQ_OQ); return _mm512_castsi512_ps( _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu)); } template<> EIGEN_STRONG_INLINE Packet16f pcmp_le(const Packet16f& a, const Packet16f& b) { __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LE_OQ); return _mm512_castsi512_ps( _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu)); } template<> EIGEN_STRONG_INLINE Packet16f pcmp_lt(const Packet16f& a, const Packet16f& b) { __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LT_OQ); return _mm512_castsi512_ps( _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu)); } template<> EIGEN_STRONG_INLINE Packet16f pcmp_lt_or_nan(const Packet16f& a, const Packet16f& b) { __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_NGE_UQ); return _mm512_castsi512_ps( _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu)); } template<> EIGEN_STRONG_INLINE Packet16i pcmp_eq(const Packet16i& a, const Packet16i& b) { __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _CMP_EQ_OQ); return _mm512_mask_set1_epi32(_mm512_set1_epi32(0), mask, 0xffffffffu); } template <> EIGEN_STRONG_INLINE Packet8d pcmp_eq(const Packet8d& a, const Packet8d& b) { __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_EQ_OQ); return _mm512_castsi512_pd( _mm512_mask_set1_epi64(_mm512_set1_epi64(0), mask, 0xffffffffffffffffu)); } template <> EIGEN_STRONG_INLINE Packet8d pcmp_le(const Packet8d& a, const Packet8d& b) { __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LE_OQ); return _mm512_castsi512_pd( _mm512_mask_set1_epi64(_mm512_set1_epi64(0), mask, 0xffffffffffffffffu)); } template <> EIGEN_STRONG_INLINE Packet8d pcmp_lt(const Packet8d& a, const Packet8d& b) { __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LT_OQ); return _mm512_castsi512_pd( _mm512_mask_set1_epi64(_mm512_set1_epi64(0), mask, 0xffffffffffffffffu)); } template <> EIGEN_STRONG_INLINE Packet8d pcmp_lt_or_nan(const Packet8d& a, const Packet8d& b) { __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_NGE_UQ); return _mm512_castsi512_pd( _mm512_mask_set1_epi64(_mm512_set1_epi64(0), mask, 0xffffffffffffffffu)); } template<> EIGEN_STRONG_INLINE Packet16f print(const Packet16f& a) { return _mm512_roundscale_ps(a, _MM_FROUND_CUR_DIRECTION); } template<> EIGEN_STRONG_INLINE Packet8d print(const Packet8d& a) { return _mm512_roundscale_pd(a, _MM_FROUND_CUR_DIRECTION); } template<> EIGEN_STRONG_INLINE Packet16f pceil(const Packet16f& a) { return _mm512_roundscale_ps(a, _MM_FROUND_TO_POS_INF); } template<> EIGEN_STRONG_INLINE Packet8d pceil(const Packet8d& a) { return _mm512_roundscale_pd(a, _MM_FROUND_TO_POS_INF); } template<> EIGEN_STRONG_INLINE Packet16f pfloor(const Packet16f& a) { return _mm512_roundscale_ps(a, _MM_FROUND_TO_NEG_INF); } template<> EIGEN_STRONG_INLINE Packet8d pfloor(const Packet8d& a) { return _mm512_roundscale_pd(a, _MM_FROUND_TO_NEG_INF); } template <> EIGEN_STRONG_INLINE Packet16i ptrue(const Packet16i& /*a*/) { return _mm512_set1_epi32(0xffffffffu); } template <> EIGEN_STRONG_INLINE Packet16f ptrue(const Packet16f& a) { return _mm512_castsi512_ps(ptrue(_mm512_castps_si512(a))); } template <> EIGEN_STRONG_INLINE Packet8d ptrue(const Packet8d& a) { return _mm512_castsi512_pd(ptrue(_mm512_castpd_si512(a))); } template <> EIGEN_STRONG_INLINE Packet16i pand(const Packet16i& a, const Packet16i& b) { return _mm512_and_si512(a,b); } template <> EIGEN_STRONG_INLINE Packet16f pand(const Packet16f& a, const Packet16f& b) { #ifdef EIGEN_VECTORIZE_AVX512DQ return _mm512_and_ps(a, b); #else return _mm512_castsi512_ps(pand(_mm512_castps_si512(a),_mm512_castps_si512(b))); #endif } template <> EIGEN_STRONG_INLINE Packet8d pand(const Packet8d& a, const Packet8d& b) { #ifdef EIGEN_VECTORIZE_AVX512DQ return _mm512_and_pd(a, b); #else Packet8d res = _mm512_undefined_pd(); Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0); Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0); res = _mm512_insertf64x4(res, _mm256_and_pd(lane0_a, lane0_b), 0); Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1); Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1); return _mm512_insertf64x4(res, _mm256_and_pd(lane1_a, lane1_b), 1); #endif } template <> EIGEN_STRONG_INLINE Packet16i por(const Packet16i& a, const Packet16i& b) { return _mm512_or_si512(a, b); } template <> EIGEN_STRONG_INLINE Packet16f por(const Packet16f& a, const Packet16f& b) { #ifdef EIGEN_VECTORIZE_AVX512DQ return _mm512_or_ps(a, b); #else return _mm512_castsi512_ps(por(_mm512_castps_si512(a),_mm512_castps_si512(b))); #endif } template <> EIGEN_STRONG_INLINE Packet8d por(const Packet8d& a, const Packet8d& b) { #ifdef EIGEN_VECTORIZE_AVX512DQ return _mm512_or_pd(a, b); #else return _mm512_castsi512_pd(por(_mm512_castpd_si512(a),_mm512_castpd_si512(b))); #endif } template <> EIGEN_STRONG_INLINE Packet16i pxor(const Packet16i& a, const Packet16i& b) { return _mm512_xor_si512(a, b); } template <> EIGEN_STRONG_INLINE Packet16f pxor(const Packet16f& a, const Packet16f& b) { #ifdef EIGEN_VECTORIZE_AVX512DQ return _mm512_xor_ps(a, b); #else return _mm512_castsi512_ps(pxor(_mm512_castps_si512(a),_mm512_castps_si512(b))); #endif } template <> EIGEN_STRONG_INLINE Packet8d pxor(const Packet8d& a, const Packet8d& b) { #ifdef EIGEN_VECTORIZE_AVX512DQ return _mm512_xor_pd(a, b); #else return _mm512_castsi512_pd(pxor(_mm512_castpd_si512(a),_mm512_castpd_si512(b))); #endif } template <> EIGEN_STRONG_INLINE Packet16i pandnot(const Packet16i& a, const Packet16i& b) { return _mm512_andnot_si512(b, a); } template <> EIGEN_STRONG_INLINE Packet16f pandnot(const Packet16f& a, const Packet16f& b) { #ifdef EIGEN_VECTORIZE_AVX512DQ return _mm512_andnot_ps(b, a); #else return _mm512_castsi512_ps(pandnot(_mm512_castps_si512(a),_mm512_castps_si512(b))); #endif } template <> EIGEN_STRONG_INLINE Packet8d pandnot(const Packet8d& a,const Packet8d& b) { #ifdef EIGEN_VECTORIZE_AVX512DQ return _mm512_andnot_pd(b, a); #else return _mm512_castsi512_pd(pandnot(_mm512_castpd_si512(a),_mm512_castpd_si512(b))); #endif } template<> EIGEN_STRONG_INLINE Packet16f pround(const Packet16f& a) { // Work-around for default std::round rounding mode. const Packet16f mask = pset1frombits(static_cast(0x80000000u)); const Packet16f prev0dot5 = pset1frombits(static_cast(0x3EFFFFFFu)); return _mm512_roundscale_ps(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO); } template<> EIGEN_STRONG_INLINE Packet8d pround(const Packet8d& a) { // Work-around for default std::round rounding mode. const Packet8d mask = pset1frombits(static_cast(0x8000000000000000ull)); const Packet8d prev0dot5 = pset1frombits(static_cast(0x3FDFFFFFFFFFFFFFull)); return _mm512_roundscale_pd(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO); } template EIGEN_STRONG_INLINE Packet16i parithmetic_shift_right(Packet16i a) { return _mm512_srai_epi32(a, N); } template EIGEN_STRONG_INLINE Packet16i plogical_shift_right(Packet16i a) { return _mm512_srli_epi32(a, N); } template EIGEN_STRONG_INLINE Packet16i plogical_shift_left(Packet16i a) { return _mm512_slli_epi32(a, N); } template <> EIGEN_STRONG_INLINE Packet16f pload(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ps(from); } template <> EIGEN_STRONG_INLINE Packet8d pload(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_pd(from); } template <> EIGEN_STRONG_INLINE Packet16i pload(const int* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_si512( reinterpret_cast(from)); } template <> EIGEN_STRONG_INLINE Packet16f ploadu(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ps(from); } template <> EIGEN_STRONG_INLINE Packet8d ploadu(const double* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_pd(from); } template <> EIGEN_STRONG_INLINE Packet16i ploadu(const int* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_si512( reinterpret_cast(from)); } template <> EIGEN_STRONG_INLINE Packet16f ploadu(const float* from, uint16_t umask) { __mmask16 mask = static_cast<__mmask16>(umask); EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_maskz_loadu_ps(mask, from); } // Loads 8 floats from memory a returns the packet // {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7} template <> EIGEN_STRONG_INLINE Packet16f ploaddup(const float* from) { // an unaligned load is required here as there is no requirement // on the alignment of input pointer 'from' __m256i low_half = _mm256_loadu_si256(reinterpret_cast(from)); __m512 even_elements = _mm512_castsi512_ps(_mm512_cvtepu32_epi64(low_half)); __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0)); return pairs; } #ifdef EIGEN_VECTORIZE_AVX512DQ // FIXME: this does not look optimal, better load a Packet4d and shuffle... // Loads 4 doubles from memory a returns the packet {a0, a0 a1, a1, a2, a2, a3, // a3} template <> EIGEN_STRONG_INLINE Packet8d ploaddup(const double* from) { __m512d x = _mm512_setzero_pd(); x = _mm512_insertf64x2(x, _mm_loaddup_pd(&from[0]), 0); x = _mm512_insertf64x2(x, _mm_loaddup_pd(&from[1]), 1); x = _mm512_insertf64x2(x, _mm_loaddup_pd(&from[2]), 2); x = _mm512_insertf64x2(x, _mm_loaddup_pd(&from[3]), 3); return x; } #else template <> EIGEN_STRONG_INLINE Packet8d ploaddup(const double* from) { __m512d x = _mm512_setzero_pd(); x = _mm512_mask_broadcastsd_pd(x, 0x3<<0, _mm_load_sd(from+0)); x = _mm512_mask_broadcastsd_pd(x, 0x3<<2, _mm_load_sd(from+1)); x = _mm512_mask_broadcastsd_pd(x, 0x3<<4, _mm_load_sd(from+2)); x = _mm512_mask_broadcastsd_pd(x, 0x3<<6, _mm_load_sd(from+3)); return x; } #endif // Loads 4 floats from memory a returns the packet // {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3} template <> EIGEN_STRONG_INLINE Packet16f ploadquad(const float* from) { Packet16f tmp = _mm512_castps128_ps512(ploadu(from)); const Packet16i scatter_mask = _mm512_set_epi32(3,3,3,3, 2,2,2,2, 1,1,1,1, 0,0,0,0); return _mm512_permutexvar_ps(scatter_mask, tmp); } // Loads 2 doubles from memory a returns the packet // {a0, a0 a0, a0, a1, a1, a1, a1} template <> EIGEN_STRONG_INLINE Packet8d ploadquad(const double* from) { __m256d lane0 = _mm256_set1_pd(*from); __m256d lane1 = _mm256_set1_pd(*(from+1)); __m512d tmp = _mm512_undefined_pd(); tmp = _mm512_insertf64x4(tmp, lane0, 0); return _mm512_insertf64x4(tmp, lane1, 1); } template <> EIGEN_STRONG_INLINE void pstore(float* to, const Packet16f& from) { EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ps(to, from); } template <> EIGEN_STRONG_INLINE void pstore(double* to, const Packet8d& from) { EIGEN_DEBUG_ALIGNED_STORE _mm512_store_pd(to, from); } template <> EIGEN_STRONG_INLINE void pstore(int* to, const Packet16i& from) { EIGEN_DEBUG_ALIGNED_STORE _mm512_storeu_si512(reinterpret_cast<__m512i*>(to), from); } template <> EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet16f& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ps(to, from); } template <> EIGEN_STRONG_INLINE void pstoreu(double* to, const Packet8d& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_pd(to, from); } template <> EIGEN_STRONG_INLINE void pstoreu(int* to, const Packet16i& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_si512( reinterpret_cast<__m512i*>(to), from); } template <> EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet16f& from, uint16_t umask) { __mmask16 mask = static_cast<__mmask16>(umask); EIGEN_DEBUG_UNALIGNED_STORE return _mm512_mask_storeu_ps(to, mask, from); } template <> EIGEN_DEVICE_FUNC inline Packet16f pgather(const float* from, Index stride) { Packet16i stride_vector = _mm512_set1_epi32(convert_index(stride)); Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); return _mm512_i32gather_ps(indices, from, 4); } template <> EIGEN_DEVICE_FUNC inline Packet8d pgather(const double* from, Index stride) { Packet8i stride_vector = _mm256_set1_epi32(convert_index(stride)); Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier); return _mm512_i32gather_pd(indices, from, 8); } template <> EIGEN_DEVICE_FUNC inline void pscatter(float* to, const Packet16f& from, Index stride) { Packet16i stride_vector = _mm512_set1_epi32(convert_index(stride)); Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier); _mm512_i32scatter_ps(to, indices, from, 4); } template <> EIGEN_DEVICE_FUNC inline void pscatter(double* to, const Packet8d& from, Index stride) { Packet8i stride_vector = _mm256_set1_epi32(convert_index(stride)); Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier); _mm512_i32scatter_pd(to, indices, from, 8); } template <> EIGEN_STRONG_INLINE void pstore1(float* to, const float& a) { Packet16f pa = pset1(a); pstore(to, pa); } template <> EIGEN_STRONG_INLINE void pstore1(double* to, const double& a) { Packet8d pa = pset1(a); pstore(to, pa); } template <> EIGEN_STRONG_INLINE void pstore1(int* to, const int& a) { Packet16i pa = pset1(a); pstore(to, pa); } template<> EIGEN_STRONG_INLINE void prefetch(const float* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } template<> EIGEN_STRONG_INLINE void prefetch(const double* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } template<> EIGEN_STRONG_INLINE void prefetch(const int* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); } template <> EIGEN_STRONG_INLINE float pfirst(const Packet16f& a) { return _mm_cvtss_f32(_mm512_extractf32x4_ps(a, 0)); } template <> EIGEN_STRONG_INLINE double pfirst(const Packet8d& a) { return _mm_cvtsd_f64(_mm256_extractf128_pd(_mm512_extractf64x4_pd(a, 0), 0)); } template <> EIGEN_STRONG_INLINE int pfirst(const Packet16i& a) { return _mm_extract_epi32(_mm512_extracti32x4_epi32(a, 0), 0); } template<> EIGEN_STRONG_INLINE Packet16f preverse(const Packet16f& a) { return _mm512_permutexvar_ps(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a); } template<> EIGEN_STRONG_INLINE Packet8d preverse(const Packet8d& a) { return _mm512_permutexvar_pd(_mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), a); } template<> EIGEN_STRONG_INLINE Packet16f pabs(const Packet16f& a) { // _mm512_abs_ps intrinsic not found, so hack around it return _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(a), _mm512_set1_epi32(0x7fffffff))); } template <> EIGEN_STRONG_INLINE Packet8d pabs(const Packet8d& a) { // _mm512_abs_ps intrinsic not found, so hack around it return _mm512_castsi512_pd(_mm512_and_si512(_mm512_castpd_si512(a), _mm512_set1_epi64(0x7fffffffffffffff))); } template<> EIGEN_STRONG_INLINE Packet16f pfrexp(const Packet16f& a, Packet16f& exponent){ return pfrexp_generic(a, exponent); } // Extract exponent without existence of Packet8l. template<> EIGEN_STRONG_INLINE Packet8d pfrexp_generic_get_biased_exponent(const Packet8d& a) { const Packet8d cst_exp_mask = pset1frombits(static_cast(0x7ff0000000000000ull)); #ifdef EIGEN_VECTORIZE_AVX512DQ return _mm512_cvtepi64_pd(_mm512_srli_epi64(_mm512_castpd_si512(pand(a, cst_exp_mask)), 52)); #else return _mm512_cvtepi32_pd(_mm512_cvtepi64_epi32(_mm512_srli_epi64(_mm512_castpd_si512(pand(a, cst_exp_mask)), 52))); #endif } template<> EIGEN_STRONG_INLINE Packet8d pfrexp(const Packet8d& a, Packet8d& exponent) { return pfrexp_generic(a, exponent); } template<> EIGEN_STRONG_INLINE Packet16f pldexp(const Packet16f& a, const Packet16f& exponent) { return pldexp_generic(a, exponent); } template<> EIGEN_STRONG_INLINE Packet8d pldexp(const Packet8d& a, const Packet8d& exponent) { // Clamp exponent to [-2099, 2099] const Packet8d max_exponent = pset1(2099.0); const Packet8i e = _mm512_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent)); // Split 2^e into four factors and multiply. const Packet8i bias = pset1(1023); Packet8i b = parithmetic_shift_right<2>(e); // floor(e/4) // 2^b const Packet8i permute_idx = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); Packet8i hi = _mm256_permutevar8x32_epi32(padd(b, bias), permute_idx); Packet8i lo = _mm256_slli_epi64(hi, 52); hi = _mm256_slli_epi64(_mm256_srli_epi64(hi, 32), 52); Packet8d c = _mm512_castsi512_pd(_mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1)); Packet8d out = pmul(pmul(pmul(a, c), c), c); // a * 2^(3b) // 2^(e - 3b) b = psub(psub(psub(e, b), b), b); // e - 3b hi = _mm256_permutevar8x32_epi32(padd(b, bias), permute_idx); lo = _mm256_slli_epi64(hi, 52); hi = _mm256_slli_epi64(_mm256_srli_epi64(hi, 32), 52); c = _mm512_castsi512_pd(_mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1)); out = pmul(out, c); // a * 2^e return out; } #ifdef EIGEN_VECTORIZE_AVX512DQ // AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512 #define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \ __m256 OUTPUT##_0 = _mm512_extractf32x8_ps(INPUT, 0); \ __m256 OUTPUT##_1 = _mm512_extractf32x8_ps(INPUT, 1) #else #define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \ __m256 OUTPUT##_0 = _mm256_insertf128_ps( \ _mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 0)), \ _mm512_extractf32x4_ps(INPUT, 1), 1); \ __m256 OUTPUT##_1 = _mm256_insertf128_ps( \ _mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 2)), \ _mm512_extractf32x4_ps(INPUT, 3), 1); #endif #ifdef EIGEN_VECTORIZE_AVX512DQ #define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \ OUTPUT = _mm512_insertf32x8(_mm512_castps256_ps512(INPUTA), INPUTB, 1); #else #define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \ OUTPUT = _mm512_undefined_ps(); \ OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 0), 0); \ OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 1), 1); \ OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 0), 2); \ OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 1), 3); #endif template <> EIGEN_STRONG_INLINE float predux(const Packet16f& a) { #ifdef EIGEN_VECTORIZE_AVX512DQ __m256 lane0 = _mm512_extractf32x8_ps(a, 0); __m256 lane1 = _mm512_extractf32x8_ps(a, 1); Packet8f x = _mm256_add_ps(lane0, lane1); return predux(x); #else __m128 lane0 = _mm512_extractf32x4_ps(a, 0); __m128 lane1 = _mm512_extractf32x4_ps(a, 1); __m128 lane2 = _mm512_extractf32x4_ps(a, 2); __m128 lane3 = _mm512_extractf32x4_ps(a, 3); __m128 sum = _mm_add_ps(_mm_add_ps(lane0, lane1), _mm_add_ps(lane2, lane3)); sum = _mm_hadd_ps(sum, sum); sum = _mm_hadd_ps(sum, _mm_permute_ps(sum, 1)); return _mm_cvtss_f32(sum); #endif } template <> EIGEN_STRONG_INLINE double predux(const Packet8d& a) { __m256d lane0 = _mm512_extractf64x4_pd(a, 0); __m256d lane1 = _mm512_extractf64x4_pd(a, 1); __m256d sum = _mm256_add_pd(lane0, lane1); __m256d tmp0 = _mm256_hadd_pd(sum, _mm256_permute2f128_pd(sum, sum, 1)); return _mm_cvtsd_f64(_mm256_castpd256_pd128(_mm256_hadd_pd(tmp0, tmp0))); } template <> EIGEN_STRONG_INLINE Packet8f predux_half_dowto4(const Packet16f& a) { #ifdef EIGEN_VECTORIZE_AVX512DQ __m256 lane0 = _mm512_extractf32x8_ps(a, 0); __m256 lane1 = _mm512_extractf32x8_ps(a, 1); return _mm256_add_ps(lane0, lane1); #else __m128 lane0 = _mm512_extractf32x4_ps(a, 0); __m128 lane1 = _mm512_extractf32x4_ps(a, 1); __m128 lane2 = _mm512_extractf32x4_ps(a, 2); __m128 lane3 = _mm512_extractf32x4_ps(a, 3); __m128 sum0 = _mm_add_ps(lane0, lane2); __m128 sum1 = _mm_add_ps(lane1, lane3); return _mm256_insertf128_ps(_mm256_castps128_ps256(sum0), sum1, 1); #endif } template <> EIGEN_STRONG_INLINE Packet4d predux_half_dowto4(const Packet8d& a) { __m256d lane0 = _mm512_extractf64x4_pd(a, 0); __m256d lane1 = _mm512_extractf64x4_pd(a, 1); return _mm256_add_pd(lane0, lane1); } template <> EIGEN_STRONG_INLINE float predux_mul(const Packet16f& a) { //#ifdef EIGEN_VECTORIZE_AVX512DQ #if 0 Packet8f lane0 = _mm512_extractf32x8_ps(a, 0); Packet8f lane1 = _mm512_extractf32x8_ps(a, 1); Packet8f res = pmul(lane0, lane1); res = pmul(res, _mm256_permute2f128_ps(res, res, 1)); res = pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2))); return pfirst(pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1)))); #else __m128 lane0 = _mm512_extractf32x4_ps(a, 0); __m128 lane1 = _mm512_extractf32x4_ps(a, 1); __m128 lane2 = _mm512_extractf32x4_ps(a, 2); __m128 lane3 = _mm512_extractf32x4_ps(a, 3); __m128 res = pmul(pmul(lane0, lane1), pmul(lane2, lane3)); res = pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2))); return pfirst(pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1)))); #endif } template <> EIGEN_STRONG_INLINE double predux_mul(const Packet8d& a) { __m256d lane0 = _mm512_extractf64x4_pd(a, 0); __m256d lane1 = _mm512_extractf64x4_pd(a, 1); __m256d res = pmul(lane0, lane1); res = pmul(res, _mm256_permute2f128_pd(res, res, 1)); return pfirst(pmul(res, _mm256_shuffle_pd(res, res, 1))); } template <> EIGEN_STRONG_INLINE float predux_min(const Packet16f& a) { __m128 lane0 = _mm512_extractf32x4_ps(a, 0); __m128 lane1 = _mm512_extractf32x4_ps(a, 1); __m128 lane2 = _mm512_extractf32x4_ps(a, 2); __m128 lane3 = _mm512_extractf32x4_ps(a, 3); __m128 res = _mm_min_ps(_mm_min_ps(lane0, lane1), _mm_min_ps(lane2, lane3)); res = _mm_min_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2))); return pfirst(_mm_min_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1)))); } template <> EIGEN_STRONG_INLINE double predux_min(const Packet8d& a) { __m256d lane0 = _mm512_extractf64x4_pd(a, 0); __m256d lane1 = _mm512_extractf64x4_pd(a, 1); __m256d res = _mm256_min_pd(lane0, lane1); res = _mm256_min_pd(res, _mm256_permute2f128_pd(res, res, 1)); return pfirst(_mm256_min_pd(res, _mm256_shuffle_pd(res, res, 1))); } template <> EIGEN_STRONG_INLINE float predux_max(const Packet16f& a) { __m128 lane0 = _mm512_extractf32x4_ps(a, 0); __m128 lane1 = _mm512_extractf32x4_ps(a, 1); __m128 lane2 = _mm512_extractf32x4_ps(a, 2); __m128 lane3 = _mm512_extractf32x4_ps(a, 3); __m128 res = _mm_max_ps(_mm_max_ps(lane0, lane1), _mm_max_ps(lane2, lane3)); res = _mm_max_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2))); return pfirst(_mm_max_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1)))); } template <> EIGEN_STRONG_INLINE double predux_max(const Packet8d& a) { __m256d lane0 = _mm512_extractf64x4_pd(a, 0); __m256d lane1 = _mm512_extractf64x4_pd(a, 1); __m256d res = _mm256_max_pd(lane0, lane1); res = _mm256_max_pd(res, _mm256_permute2f128_pd(res, res, 1)); return pfirst(_mm256_max_pd(res, _mm256_shuffle_pd(res, res, 1))); } template<> EIGEN_STRONG_INLINE bool predux_any(const Packet16f& x) { Packet16i xi = _mm512_castps_si512(x); __mmask16 tmp = _mm512_test_epi32_mask(xi,xi); return !_mm512_kortestz(tmp,tmp); } #define PACK_OUTPUT(OUTPUT, INPUT, INDEX, STRIDE) \ EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]); EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]); __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]); __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]); __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]); __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]); __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]); __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]); __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]); __m512 T8 = _mm512_unpacklo_ps(kernel.packet[8], kernel.packet[9]); __m512 T9 = _mm512_unpackhi_ps(kernel.packet[8], kernel.packet[9]); __m512 T10 = _mm512_unpacklo_ps(kernel.packet[10], kernel.packet[11]); __m512 T11 = _mm512_unpackhi_ps(kernel.packet[10], kernel.packet[11]); __m512 T12 = _mm512_unpacklo_ps(kernel.packet[12], kernel.packet[13]); __m512 T13 = _mm512_unpackhi_ps(kernel.packet[12], kernel.packet[13]); __m512 T14 = _mm512_unpacklo_ps(kernel.packet[14], kernel.packet[15]); __m512 T15 = _mm512_unpackhi_ps(kernel.packet[14], kernel.packet[15]); __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0)); __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2)); __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0)); __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2)); __m512 S4 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(1, 0, 1, 0)); __m512 S5 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(3, 2, 3, 2)); __m512 S6 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(1, 0, 1, 0)); __m512 S7 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(3, 2, 3, 2)); __m512 S8 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(1, 0, 1, 0)); __m512 S9 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(3, 2, 3, 2)); __m512 S10 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(1, 0, 1, 0)); __m512 S11 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(3, 2, 3, 2)); __m512 S12 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(1, 0, 1, 0)); __m512 S13 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(3, 2, 3, 2)); __m512 S14 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(1, 0, 1, 0)); __m512 S15 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(3, 2, 3, 2)); EIGEN_EXTRACT_8f_FROM_16f(S0, S0); EIGEN_EXTRACT_8f_FROM_16f(S1, S1); EIGEN_EXTRACT_8f_FROM_16f(S2, S2); EIGEN_EXTRACT_8f_FROM_16f(S3, S3); EIGEN_EXTRACT_8f_FROM_16f(S4, S4); EIGEN_EXTRACT_8f_FROM_16f(S5, S5); EIGEN_EXTRACT_8f_FROM_16f(S6, S6); EIGEN_EXTRACT_8f_FROM_16f(S7, S7); EIGEN_EXTRACT_8f_FROM_16f(S8, S8); EIGEN_EXTRACT_8f_FROM_16f(S9, S9); EIGEN_EXTRACT_8f_FROM_16f(S10, S10); EIGEN_EXTRACT_8f_FROM_16f(S11, S11); EIGEN_EXTRACT_8f_FROM_16f(S12, S12); EIGEN_EXTRACT_8f_FROM_16f(S13, S13); EIGEN_EXTRACT_8f_FROM_16f(S14, S14); EIGEN_EXTRACT_8f_FROM_16f(S15, S15); PacketBlock tmp; tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S4_0, 0x20); tmp.packet[1] = _mm256_permute2f128_ps(S1_0, S5_0, 0x20); tmp.packet[2] = _mm256_permute2f128_ps(S2_0, S6_0, 0x20); tmp.packet[3] = _mm256_permute2f128_ps(S3_0, S7_0, 0x20); tmp.packet[4] = _mm256_permute2f128_ps(S0_0, S4_0, 0x31); tmp.packet[5] = _mm256_permute2f128_ps(S1_0, S5_0, 0x31); tmp.packet[6] = _mm256_permute2f128_ps(S2_0, S6_0, 0x31); tmp.packet[7] = _mm256_permute2f128_ps(S3_0, S7_0, 0x31); tmp.packet[8] = _mm256_permute2f128_ps(S0_1, S4_1, 0x20); tmp.packet[9] = _mm256_permute2f128_ps(S1_1, S5_1, 0x20); tmp.packet[10] = _mm256_permute2f128_ps(S2_1, S6_1, 0x20); tmp.packet[11] = _mm256_permute2f128_ps(S3_1, S7_1, 0x20); tmp.packet[12] = _mm256_permute2f128_ps(S0_1, S4_1, 0x31); tmp.packet[13] = _mm256_permute2f128_ps(S1_1, S5_1, 0x31); tmp.packet[14] = _mm256_permute2f128_ps(S2_1, S6_1, 0x31); tmp.packet[15] = _mm256_permute2f128_ps(S3_1, S7_1, 0x31); // Second set of _m256 outputs tmp.packet[16] = _mm256_permute2f128_ps(S8_0, S12_0, 0x20); tmp.packet[17] = _mm256_permute2f128_ps(S9_0, S13_0, 0x20); tmp.packet[18] = _mm256_permute2f128_ps(S10_0, S14_0, 0x20); tmp.packet[19] = _mm256_permute2f128_ps(S11_0, S15_0, 0x20); tmp.packet[20] = _mm256_permute2f128_ps(S8_0, S12_0, 0x31); tmp.packet[21] = _mm256_permute2f128_ps(S9_0, S13_0, 0x31); tmp.packet[22] = _mm256_permute2f128_ps(S10_0, S14_0, 0x31); tmp.packet[23] = _mm256_permute2f128_ps(S11_0, S15_0, 0x31); tmp.packet[24] = _mm256_permute2f128_ps(S8_1, S12_1, 0x20); tmp.packet[25] = _mm256_permute2f128_ps(S9_1, S13_1, 0x20); tmp.packet[26] = _mm256_permute2f128_ps(S10_1, S14_1, 0x20); tmp.packet[27] = _mm256_permute2f128_ps(S11_1, S15_1, 0x20); tmp.packet[28] = _mm256_permute2f128_ps(S8_1, S12_1, 0x31); tmp.packet[29] = _mm256_permute2f128_ps(S9_1, S13_1, 0x31); tmp.packet[30] = _mm256_permute2f128_ps(S10_1, S14_1, 0x31); tmp.packet[31] = _mm256_permute2f128_ps(S11_1, S15_1, 0x31); // Pack them into the output PACK_OUTPUT(kernel.packet, tmp.packet, 0, 16); PACK_OUTPUT(kernel.packet, tmp.packet, 1, 16); PACK_OUTPUT(kernel.packet, tmp.packet, 2, 16); PACK_OUTPUT(kernel.packet, tmp.packet, 3, 16); PACK_OUTPUT(kernel.packet, tmp.packet, 4, 16); PACK_OUTPUT(kernel.packet, tmp.packet, 5, 16); PACK_OUTPUT(kernel.packet, tmp.packet, 6, 16); PACK_OUTPUT(kernel.packet, tmp.packet, 7, 16); PACK_OUTPUT(kernel.packet, tmp.packet, 8, 16); PACK_OUTPUT(kernel.packet, tmp.packet, 9, 16); PACK_OUTPUT(kernel.packet, tmp.packet, 10, 16); PACK_OUTPUT(kernel.packet, tmp.packet, 11, 16); PACK_OUTPUT(kernel.packet, tmp.packet, 12, 16); PACK_OUTPUT(kernel.packet, tmp.packet, 13, 16); PACK_OUTPUT(kernel.packet, tmp.packet, 14, 16); PACK_OUTPUT(kernel.packet, tmp.packet, 15, 16); } #define PACK_OUTPUT_2(OUTPUT, INPUT, INDEX, STRIDE) \ EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[2 * INDEX], \ INPUT[2 * INDEX + STRIDE]); EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]); __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]); __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]); __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]); __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0)); __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2)); __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0)); __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2)); EIGEN_EXTRACT_8f_FROM_16f(S0, S0); EIGEN_EXTRACT_8f_FROM_16f(S1, S1); EIGEN_EXTRACT_8f_FROM_16f(S2, S2); EIGEN_EXTRACT_8f_FROM_16f(S3, S3); PacketBlock tmp; tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S1_0, 0x20); tmp.packet[1] = _mm256_permute2f128_ps(S2_0, S3_0, 0x20); tmp.packet[2] = _mm256_permute2f128_ps(S0_0, S1_0, 0x31); tmp.packet[3] = _mm256_permute2f128_ps(S2_0, S3_0, 0x31); tmp.packet[4] = _mm256_permute2f128_ps(S0_1, S1_1, 0x20); tmp.packet[5] = _mm256_permute2f128_ps(S2_1, S3_1, 0x20); tmp.packet[6] = _mm256_permute2f128_ps(S0_1, S1_1, 0x31); tmp.packet[7] = _mm256_permute2f128_ps(S2_1, S3_1, 0x31); PACK_OUTPUT_2(kernel.packet, tmp.packet, 0, 1); PACK_OUTPUT_2(kernel.packet, tmp.packet, 1, 1); PACK_OUTPUT_2(kernel.packet, tmp.packet, 2, 1); PACK_OUTPUT_2(kernel.packet, tmp.packet, 3, 1); } #define PACK_OUTPUT_SQ_D(OUTPUT, INPUT, INDEX, STRIDE) \ OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX], 0); \ OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX + STRIDE], 1); #define PACK_OUTPUT_D(OUTPUT, INPUT, INDEX, STRIDE) \ OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX)], 0); \ OUTPUT[INDEX] = \ _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX) + STRIDE], 1); EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { __m512d T0 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0); __m512d T1 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0xff); __m512d T2 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0); __m512d T3 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0xff); PacketBlock tmp; tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), _mm512_extractf64x4_pd(T2, 0), 0x20); tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), _mm512_extractf64x4_pd(T3, 0), 0x20); tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), _mm512_extractf64x4_pd(T2, 0), 0x31); tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), _mm512_extractf64x4_pd(T3, 0), 0x31); tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), _mm512_extractf64x4_pd(T2, 1), 0x20); tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), _mm512_extractf64x4_pd(T3, 1), 0x20); tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), _mm512_extractf64x4_pd(T2, 1), 0x31); tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), _mm512_extractf64x4_pd(T3, 1), 0x31); PACK_OUTPUT_D(kernel.packet, tmp.packet, 0, 1); PACK_OUTPUT_D(kernel.packet, tmp.packet, 1, 1); PACK_OUTPUT_D(kernel.packet, tmp.packet, 2, 1); PACK_OUTPUT_D(kernel.packet, tmp.packet, 3, 1); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { __m512d T0 = _mm512_unpacklo_pd(kernel.packet[0], kernel.packet[1]); __m512d T1 = _mm512_unpackhi_pd(kernel.packet[0], kernel.packet[1]); __m512d T2 = _mm512_unpacklo_pd(kernel.packet[2], kernel.packet[3]); __m512d T3 = _mm512_unpackhi_pd(kernel.packet[2], kernel.packet[3]); __m512d T4 = _mm512_unpacklo_pd(kernel.packet[4], kernel.packet[5]); __m512d T5 = _mm512_unpackhi_pd(kernel.packet[4], kernel.packet[5]); __m512d T6 = _mm512_unpacklo_pd(kernel.packet[6], kernel.packet[7]); __m512d T7 = _mm512_unpackhi_pd(kernel.packet[6], kernel.packet[7]); PacketBlock tmp; tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), _mm512_extractf64x4_pd(T2, 0), 0x20); tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), _mm512_extractf64x4_pd(T3, 0), 0x20); tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), _mm512_extractf64x4_pd(T2, 0), 0x31); tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), _mm512_extractf64x4_pd(T3, 0), 0x31); tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), _mm512_extractf64x4_pd(T2, 1), 0x20); tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), _mm512_extractf64x4_pd(T3, 1), 0x20); tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), _mm512_extractf64x4_pd(T2, 1), 0x31); tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), _mm512_extractf64x4_pd(T3, 1), 0x31); tmp.packet[8] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 0), _mm512_extractf64x4_pd(T6, 0), 0x20); tmp.packet[9] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 0), _mm512_extractf64x4_pd(T7, 0), 0x20); tmp.packet[10] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 0), _mm512_extractf64x4_pd(T6, 0), 0x31); tmp.packet[11] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 0), _mm512_extractf64x4_pd(T7, 0), 0x31); tmp.packet[12] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 1), _mm512_extractf64x4_pd(T6, 1), 0x20); tmp.packet[13] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 1), _mm512_extractf64x4_pd(T7, 1), 0x20); tmp.packet[14] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 1), _mm512_extractf64x4_pd(T6, 1), 0x31); tmp.packet[15] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 1), _mm512_extractf64x4_pd(T7, 1), 0x31); PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 0, 8); PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 1, 8); PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 2, 8); PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 3, 8); PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 4, 8); PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 5, 8); PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 6, 8); PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 7, 8); } template <> EIGEN_STRONG_INLINE Packet16f pblend(const Selector<16>& /*ifPacket*/, const Packet16f& /*thenPacket*/, const Packet16f& /*elsePacket*/) { assert(false && "To be implemented"); return Packet16f(); } template <> EIGEN_STRONG_INLINE Packet8d pblend(const Selector<8>& ifPacket, const Packet8d& thenPacket, const Packet8d& elsePacket) { __mmask8 m = (ifPacket.select[0] ) | (ifPacket.select[1]<<1) | (ifPacket.select[2]<<2) | (ifPacket.select[3]<<3) | (ifPacket.select[4]<<4) | (ifPacket.select[5]<<5) | (ifPacket.select[6]<<6) | (ifPacket.select[7]<<7); return _mm512_mask_blend_pd(m, elsePacket, thenPacket); } // Packet math for Eigen::half template<> EIGEN_STRONG_INLINE Packet16h pset1(const Eigen::half& from) { return _mm256_set1_epi16(from.x); } template<> EIGEN_STRONG_INLINE Eigen::half pfirst(const Packet16h& from) { return half_impl::raw_uint16_to_half(static_cast(_mm256_extract_epi16(from, 0))); } template<> EIGEN_STRONG_INLINE Packet16h pload(const Eigen::half* from) { return _mm256_load_si256(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE Packet16h ploadu(const Eigen::half* from) { return _mm256_loadu_si256(reinterpret_cast(from)); } template<> EIGEN_STRONG_INLINE void pstore(Eigen::half* to, const Packet16h& from) { // (void*) -> workaround clang warning: // cast from 'Eigen::half *' to '__m256i *' increases required alignment from 2 to 32 _mm256_store_si256((__m256i*)(void*)to, from); } template<> EIGEN_STRONG_INLINE void pstoreu(Eigen::half* to, const Packet16h& from) { // (void*) -> workaround clang warning: // cast from 'Eigen::half *' to '__m256i *' increases required alignment from 2 to 32 _mm256_storeu_si256((__m256i*)(void*)to, from); } template<> EIGEN_STRONG_INLINE Packet16h ploaddup(const Eigen::half* from) { unsigned short a = from[0].x; unsigned short b = from[1].x; unsigned short c = from[2].x; unsigned short d = from[3].x; unsigned short e = from[4].x; unsigned short f = from[5].x; unsigned short g = from[6].x; unsigned short h = from[7].x; return _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a); } template<> EIGEN_STRONG_INLINE Packet16h ploadquad(const Eigen::half* from) { unsigned short a = from[0].x; unsigned short b = from[1].x; unsigned short c = from[2].x; unsigned short d = from[3].x; return _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a); } EIGEN_STRONG_INLINE Packet16f half2float(const Packet16h& a) { #ifdef EIGEN_HAS_FP16_C return _mm512_cvtph_ps(a); #else EIGEN_ALIGN64 half aux[16]; pstore(aux, a); float f0(aux[0]); float f1(aux[1]); float f2(aux[2]); float f3(aux[3]); float f4(aux[4]); float f5(aux[5]); float f6(aux[6]); float f7(aux[7]); float f8(aux[8]); float f9(aux[9]); float fa(aux[10]); float fb(aux[11]); float fc(aux[12]); float fd(aux[13]); float fe(aux[14]); float ff(aux[15]); return _mm512_set_ps( ff, fe, fd, fc, fb, fa, f9, f8, f7, f6, f5, f4, f3, f2, f1, f0); #endif } EIGEN_STRONG_INLINE Packet16h float2half(const Packet16f& a) { #ifdef EIGEN_HAS_FP16_C return _mm512_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT|_MM_FROUND_NO_EXC); #else EIGEN_ALIGN64 float aux[16]; pstore(aux, a); half h0(aux[0]); half h1(aux[1]); half h2(aux[2]); half h3(aux[3]); half h4(aux[4]); half h5(aux[5]); half h6(aux[6]); half h7(aux[7]); half h8(aux[8]); half h9(aux[9]); half ha(aux[10]); half hb(aux[11]); half hc(aux[12]); half hd(aux[13]); half he(aux[14]); half hf(aux[15]); return _mm256_set_epi16( hf.x, he.x, hd.x, hc.x, hb.x, ha.x, h9.x, h8.x, h7.x, h6.x, h5.x, h4.x, h3.x, h2.x, h1.x, h0.x); #endif } template<> EIGEN_STRONG_INLINE Packet16h ptrue(const Packet16h& a) { return ptrue(Packet8i(a)); } template <> EIGEN_STRONG_INLINE Packet16h pabs(const Packet16h& a) { const __m256i sign_mask = _mm256_set1_epi16(static_cast(0x8000)); return _mm256_andnot_si256(sign_mask, a); } template <> EIGEN_STRONG_INLINE Packet16h pmin(const Packet16h& a, const Packet16h& b) { return float2half(pmin(half2float(a), half2float(b))); } template <> EIGEN_STRONG_INLINE Packet16h pmax(const Packet16h& a, const Packet16h& b) { return float2half(pmax(half2float(a), half2float(b))); } template <> EIGEN_STRONG_INLINE Packet16h plset(const half& a) { return float2half(plset(static_cast(a))); } template<> EIGEN_STRONG_INLINE Packet16h por(const Packet16h& a,const Packet16h& b) { // in some cases Packet8i is a wrapper around __m256i, so we need to // cast to Packet8i to call the correct overload. return por(Packet8i(a),Packet8i(b)); } template<> EIGEN_STRONG_INLINE Packet16h pxor(const Packet16h& a,const Packet16h& b) { return pxor(Packet8i(a),Packet8i(b)); } template<> EIGEN_STRONG_INLINE Packet16h pand(const Packet16h& a,const Packet16h& b) { return pand(Packet8i(a),Packet8i(b)); } template<> EIGEN_STRONG_INLINE Packet16h pandnot(const Packet16h& a,const Packet16h& b) { return pandnot(Packet8i(a),Packet8i(b)); } template<> EIGEN_STRONG_INLINE Packet16h pselect(const Packet16h& mask, const Packet16h& a, const Packet16h& b) { return _mm256_blendv_epi8(b, a, mask); } template<> EIGEN_STRONG_INLINE Packet16h pround(const Packet16h& a) { return float2half(pround(half2float(a))); } template<> EIGEN_STRONG_INLINE Packet16h print(const Packet16h& a) { return float2half(print(half2float(a))); } template<> EIGEN_STRONG_INLINE Packet16h pceil(const Packet16h& a) { return float2half(pceil(half2float(a))); } template<> EIGEN_STRONG_INLINE Packet16h pfloor(const Packet16h& a) { return float2half(pfloor(half2float(a))); } template<> EIGEN_STRONG_INLINE Packet16h pcmp_eq(const Packet16h& a,const Packet16h& b) { Packet16f af = half2float(a); Packet16f bf = half2float(b); return Pack32To16(pcmp_eq(af, bf)); } template<> EIGEN_STRONG_INLINE Packet16h pcmp_le(const Packet16h& a,const Packet16h& b) { return Pack32To16(pcmp_le(half2float(a), half2float(b))); } template<> EIGEN_STRONG_INLINE Packet16h pcmp_lt(const Packet16h& a,const Packet16h& b) { return Pack32To16(pcmp_lt(half2float(a), half2float(b))); } template<> EIGEN_STRONG_INLINE Packet16h pcmp_lt_or_nan(const Packet16h& a,const Packet16h& b) { return Pack32To16(pcmp_lt_or_nan(half2float(a), half2float(b))); } template<> EIGEN_STRONG_INLINE Packet16h pconj(const Packet16h& a) { return a; } template<> EIGEN_STRONG_INLINE Packet16h pnegate(const Packet16h& a) { Packet16h sign_mask = _mm256_set1_epi16(static_cast(0x8000)); return _mm256_xor_si256(a, sign_mask); } template<> EIGEN_STRONG_INLINE Packet16h padd(const Packet16h& a, const Packet16h& b) { Packet16f af = half2float(a); Packet16f bf = half2float(b); Packet16f rf = padd(af, bf); return float2half(rf); } template<> EIGEN_STRONG_INLINE Packet16h psub(const Packet16h& a, const Packet16h& b) { Packet16f af = half2float(a); Packet16f bf = half2float(b); Packet16f rf = psub(af, bf); return float2half(rf); } template<> EIGEN_STRONG_INLINE Packet16h pmul(const Packet16h& a, const Packet16h& b) { Packet16f af = half2float(a); Packet16f bf = half2float(b); Packet16f rf = pmul(af, bf); return float2half(rf); } template<> EIGEN_STRONG_INLINE Packet16h pdiv(const Packet16h& a, const Packet16h& b) { Packet16f af = half2float(a); Packet16f bf = half2float(b); Packet16f rf = pdiv(af, bf); return float2half(rf); } template<> EIGEN_STRONG_INLINE half predux(const Packet16h& from) { Packet16f from_float = half2float(from); return half(predux(from_float)); } template <> EIGEN_STRONG_INLINE Packet8h predux_half_dowto4(const Packet16h& a) { Packet8h lane0 = _mm256_extractf128_si256(a, 0); Packet8h lane1 = _mm256_extractf128_si256(a, 1); return padd(lane0, lane1); } template<> EIGEN_STRONG_INLINE Eigen::half predux_max(const Packet16h& a) { Packet16f af = half2float(a); float reduced = predux_max(af); return Eigen::half(reduced); } template<> EIGEN_STRONG_INLINE Eigen::half predux_min(const Packet16h& a) { Packet16f af = half2float(a); float reduced = predux_min(af); return Eigen::half(reduced); } template<> EIGEN_STRONG_INLINE half predux_mul(const Packet16h& from) { Packet16f from_float = half2float(from); return half(predux_mul(from_float)); } template<> EIGEN_STRONG_INLINE Packet16h preverse(const Packet16h& a) { __m128i m = _mm_setr_epi8(14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1); return _mm256_insertf128_si256( _mm256_castsi128_si256(_mm_shuffle_epi8(_mm256_extractf128_si256(a,1),m)), _mm_shuffle_epi8(_mm256_extractf128_si256(a,0),m), 1); } template<> EIGEN_STRONG_INLINE Packet16h pgather(const Eigen::half* from, Index stride) { return _mm256_set_epi16( from[15*stride].x, from[14*stride].x, from[13*stride].x, from[12*stride].x, from[11*stride].x, from[10*stride].x, from[9*stride].x, from[8*stride].x, from[7*stride].x, from[6*stride].x, from[5*stride].x, from[4*stride].x, from[3*stride].x, from[2*stride].x, from[1*stride].x, from[0*stride].x); } template<> EIGEN_STRONG_INLINE void pscatter(half* to, const Packet16h& from, Index stride) { EIGEN_ALIGN64 half aux[16]; pstore(aux, from); to[stride*0] = aux[0]; to[stride*1] = aux[1]; to[stride*2] = aux[2]; to[stride*3] = aux[3]; to[stride*4] = aux[4]; to[stride*5] = aux[5]; to[stride*6] = aux[6]; to[stride*7] = aux[7]; to[stride*8] = aux[8]; to[stride*9] = aux[9]; to[stride*10] = aux[10]; to[stride*11] = aux[11]; to[stride*12] = aux[12]; to[stride*13] = aux[13]; to[stride*14] = aux[14]; to[stride*15] = aux[15]; } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { __m256i a = kernel.packet[0]; __m256i b = kernel.packet[1]; __m256i c = kernel.packet[2]; __m256i d = kernel.packet[3]; __m256i e = kernel.packet[4]; __m256i f = kernel.packet[5]; __m256i g = kernel.packet[6]; __m256i h = kernel.packet[7]; __m256i i = kernel.packet[8]; __m256i j = kernel.packet[9]; __m256i k = kernel.packet[10]; __m256i l = kernel.packet[11]; __m256i m = kernel.packet[12]; __m256i n = kernel.packet[13]; __m256i o = kernel.packet[14]; __m256i p = kernel.packet[15]; __m256i ab_07 = _mm256_unpacklo_epi16(a, b); __m256i cd_07 = _mm256_unpacklo_epi16(c, d); __m256i ef_07 = _mm256_unpacklo_epi16(e, f); __m256i gh_07 = _mm256_unpacklo_epi16(g, h); __m256i ij_07 = _mm256_unpacklo_epi16(i, j); __m256i kl_07 = _mm256_unpacklo_epi16(k, l); __m256i mn_07 = _mm256_unpacklo_epi16(m, n); __m256i op_07 = _mm256_unpacklo_epi16(o, p); __m256i ab_8f = _mm256_unpackhi_epi16(a, b); __m256i cd_8f = _mm256_unpackhi_epi16(c, d); __m256i ef_8f = _mm256_unpackhi_epi16(e, f); __m256i gh_8f = _mm256_unpackhi_epi16(g, h); __m256i ij_8f = _mm256_unpackhi_epi16(i, j); __m256i kl_8f = _mm256_unpackhi_epi16(k, l); __m256i mn_8f = _mm256_unpackhi_epi16(m, n); __m256i op_8f = _mm256_unpackhi_epi16(o, p); __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07); __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07); __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07); __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07); __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07); __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07); __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07); __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07); __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f); __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f); __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f); __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f); __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f); __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f); __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f); __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f); __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03); __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03); __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03); __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03); __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47); __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47); __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47); __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47); __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b); __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b); __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b); __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b); __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf); __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf); __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf); __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf); // NOTE: no unpacklo/hi instr in this case, so using permute instr. __m256i a_p_0 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20); __m256i a_p_1 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20); __m256i a_p_2 = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20); __m256i a_p_3 = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20); __m256i a_p_4 = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20); __m256i a_p_5 = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20); __m256i a_p_6 = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20); __m256i a_p_7 = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20); __m256i a_p_8 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31); __m256i a_p_9 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31); __m256i a_p_a = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31); __m256i a_p_b = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31); __m256i a_p_c = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31); __m256i a_p_d = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31); __m256i a_p_e = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31); __m256i a_p_f = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31); kernel.packet[0] = a_p_0; kernel.packet[1] = a_p_1; kernel.packet[2] = a_p_2; kernel.packet[3] = a_p_3; kernel.packet[4] = a_p_4; kernel.packet[5] = a_p_5; kernel.packet[6] = a_p_6; kernel.packet[7] = a_p_7; kernel.packet[8] = a_p_8; kernel.packet[9] = a_p_9; kernel.packet[10] = a_p_a; kernel.packet[11] = a_p_b; kernel.packet[12] = a_p_c; kernel.packet[13] = a_p_d; kernel.packet[14] = a_p_e; kernel.packet[15] = a_p_f; } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { EIGEN_ALIGN64 half in[8][16]; pstore(in[0], kernel.packet[0]); pstore(in[1], kernel.packet[1]); pstore(in[2], kernel.packet[2]); pstore(in[3], kernel.packet[3]); pstore(in[4], kernel.packet[4]); pstore(in[5], kernel.packet[5]); pstore(in[6], kernel.packet[6]); pstore(in[7], kernel.packet[7]); EIGEN_ALIGN64 half out[8][16]; for (int i = 0; i < 8; ++i) { for (int j = 0; j < 8; ++j) { out[i][j] = in[j][2*i]; } for (int j = 0; j < 8; ++j) { out[i][j+8] = in[j][2*i+1]; } } kernel.packet[0] = pload(out[0]); kernel.packet[1] = pload(out[1]); kernel.packet[2] = pload(out[2]); kernel.packet[3] = pload(out[3]); kernel.packet[4] = pload(out[4]); kernel.packet[5] = pload(out[5]); kernel.packet[6] = pload(out[6]); kernel.packet[7] = pload(out[7]); } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { EIGEN_ALIGN64 half in[4][16]; pstore(in[0], kernel.packet[0]); pstore(in[1], kernel.packet[1]); pstore(in[2], kernel.packet[2]); pstore(in[3], kernel.packet[3]); EIGEN_ALIGN64 half out[4][16]; for (int i = 0; i < 4; ++i) { for (int j = 0; j < 4; ++j) { out[i][j] = in[j][4*i]; } for (int j = 0; j < 4; ++j) { out[i][j+4] = in[j][4*i+1]; } for (int j = 0; j < 4; ++j) { out[i][j+8] = in[j][4*i+2]; } for (int j = 0; j < 4; ++j) { out[i][j+12] = in[j][4*i+3]; } } kernel.packet[0] = pload(out[0]); kernel.packet[1] = pload(out[1]); kernel.packet[2] = pload(out[2]); kernel.packet[3] = pload(out[3]); } template <> struct is_arithmetic { enum { value = true }; }; template <> struct packet_traits : default_packet_traits { typedef Packet16bf type; typedef Packet8bf half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = 16, HasHalfPacket = 1, HasBlend = 0, HasInsert = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, #if EIGEN_GNUC_AT_LEAST(5, 3) || (!EIGEN_COMP_GNUC_STRICT) #ifdef EIGEN_VECTORIZE_AVX512DQ HasLog = 1, // Currently fails test with bad accuracy. HasLog1p = 1, HasExpm1 = 1, HasNdtri = 1, HasBessel = 1, #endif HasExp = 1, HasSqrt = EIGEN_FAST_MATH, HasRsqrt = EIGEN_FAST_MATH, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, #endif HasCmp = 1, HasDiv = 1 }; }; template <> struct unpacket_traits { typedef bfloat16 type; enum {size=16, alignment=Aligned32, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet8bf half; }; template <> EIGEN_STRONG_INLINE Packet16bf pset1(const bfloat16& from) { return _mm256_set1_epi16(from.value); } template <> EIGEN_STRONG_INLINE bfloat16 pfirst(const Packet16bf& from) { bfloat16 t; t.value = static_cast(_mm256_extract_epi16(from, 0)); return t; } template <> EIGEN_STRONG_INLINE Packet16bf pload(const bfloat16* from) { return _mm256_load_si256(reinterpret_cast(from)); } template <> EIGEN_STRONG_INLINE Packet16bf ploadu(const bfloat16* from) { return _mm256_loadu_si256(reinterpret_cast(from)); } template <> EIGEN_STRONG_INLINE void pstore(bfloat16* to, const Packet16bf& from) { _mm256_store_si256(reinterpret_cast<__m256i*>(to), from); } template <> EIGEN_STRONG_INLINE void pstoreu(bfloat16* to, const Packet16bf& from) { _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from); } template<> EIGEN_STRONG_INLINE Packet16bf ploaddup(const bfloat16* from) { Packet16bf r; unsigned short a = from[0].value; unsigned short b = from[1].value; unsigned short c = from[2].value; unsigned short d = from[3].value; unsigned short e = from[4].value; unsigned short f = from[5].value; unsigned short g = from[6].value; unsigned short h = from[7].value; return _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a); } template<> EIGEN_STRONG_INLINE Packet16bf ploadquad(const bfloat16* from) { Packet16bf r; unsigned short a = from[0].value; unsigned short b = from[1].value; unsigned short c = from[2].value; unsigned short d = from[3].value; return _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a); } EIGEN_STRONG_INLINE Packet16f Bf16ToF32(const Packet16bf& a) { return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16)); } // Convert float to bfloat16 according to round-to-nearest-even/denormals algorithm. EIGEN_STRONG_INLINE Packet16bf F32ToBf16(const Packet16f& a) { Packet16bf r; #if defined(EIGEN_VECTORIZE_AVX512BF16) && EIGEN_GNUC_AT_LEAST(10, 1) // Since GCC 10.1 supports avx512bf16 and C style explicit cast // (C++ static_cast is not supported yet), do converion via intrinsic // and register path for performance. r = (__m256i)(_mm512_cvtneps_pbh(a)); #else __m512i t; __m512i input = _mm512_castps_si512(a); __m512i nan = _mm512_set1_epi32(0x7fc0); // uint32_t lsb = (input >> 16) & 1; t = _mm512_and_si512(_mm512_srli_epi32(input, 16), _mm512_set1_epi32(1)); // uint32_t rounding_bias = 0x7fff + lsb; t = _mm512_add_epi32(t, _mm512_set1_epi32(0x7fff)); // input += rounding_bias; t = _mm512_add_epi32(t, input); // input = input >> 16; t = _mm512_srli_epi32(t, 16); // Check NaN before converting back to bf16 __mmask16 mask = _mm512_cmp_ps_mask(a, a, _CMP_ORD_Q); t = _mm512_mask_blend_epi32(mask, nan, t); // output.value = static_cast(input); r = _mm512_cvtepi32_epi16(t); #endif // EIGEN_VECTORIZE_AVX512BF16 return r; } template <> EIGEN_STRONG_INLINE Packet16bf ptrue(const Packet16bf& a) { return ptrue(a); } template <> EIGEN_STRONG_INLINE Packet16bf por(const Packet16bf& a, const Packet16bf& b) { return por(a, b); } template <> EIGEN_STRONG_INLINE Packet16bf pxor(const Packet16bf& a, const Packet16bf& b) { return pxor(a, b); } template <> EIGEN_STRONG_INLINE Packet16bf pand(const Packet16bf& a, const Packet16bf& b) { return pand(a, b); } template <> EIGEN_STRONG_INLINE Packet16bf pandnot(const Packet16bf& a, const Packet16bf& b) { return pandnot(a, b); } template <> EIGEN_STRONG_INLINE Packet16bf pselect(const Packet16bf& mask, const Packet16bf& a, const Packet16bf& b) { // Input mask is expected to be all 0/1, handle it with 8-bit // intrinsic for performance. return _mm256_blendv_epi8(b, a, mask); } template<> EIGEN_STRONG_INLINE Packet16bf pround(const Packet16bf& a) { return F32ToBf16(pround(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE Packet16bf print(const Packet16bf& a) { return F32ToBf16(print(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE Packet16bf pceil(const Packet16bf& a) { return F32ToBf16(pceil(Bf16ToF32(a))); } template<> EIGEN_STRONG_INLINE Packet16bf pfloor(const Packet16bf& a) { return F32ToBf16(pfloor(Bf16ToF32(a))); } template <> EIGEN_STRONG_INLINE Packet16bf pcmp_eq(const Packet16bf& a, const Packet16bf& b) { return Pack32To16(pcmp_eq(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet16bf pcmp_le(const Packet16bf& a, const Packet16bf& b) { return Pack32To16(pcmp_le(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet16bf pcmp_lt(const Packet16bf& a, const Packet16bf& b) { return Pack32To16(pcmp_lt(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet16bf pcmp_lt_or_nan(const Packet16bf& a, const Packet16bf& b) { return Pack32To16(pcmp_lt_or_nan(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet16bf pnegate(const Packet16bf& a) { Packet16bf sign_mask = _mm256_set1_epi16(static_cast(0x8000)); return _mm256_xor_si256(a, sign_mask); } template <> EIGEN_STRONG_INLINE Packet16bf pconj(const Packet16bf& a) { return a; } template <> EIGEN_STRONG_INLINE Packet16bf pabs(const Packet16bf& a) { const __m256i sign_mask = _mm256_set1_epi16(static_cast(0x8000)); return _mm256_andnot_si256(sign_mask, a); } template <> EIGEN_STRONG_INLINE Packet16bf padd(const Packet16bf& a, const Packet16bf& b) { return F32ToBf16(padd(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet16bf psub(const Packet16bf& a, const Packet16bf& b) { return F32ToBf16(psub(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet16bf pmul(const Packet16bf& a, const Packet16bf& b) { return F32ToBf16(pmul(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet16bf pdiv(const Packet16bf& a, const Packet16bf& b) { return F32ToBf16(pdiv(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet16bf pmin(const Packet16bf& a, const Packet16bf& b) { return F32ToBf16(pmin(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet16bf pmax(const Packet16bf& a, const Packet16bf& b) { return F32ToBf16(pmax(Bf16ToF32(a), Bf16ToF32(b))); } template <> EIGEN_STRONG_INLINE Packet16bf plset(const bfloat16& a) { return F32ToBf16(plset(static_cast(a))); } template <> EIGEN_STRONG_INLINE Packet8bf predux_half_dowto4(const Packet16bf& a) { Packet8bf lane0 = _mm256_extractf128_si256(a, 0); Packet8bf lane1 = _mm256_extractf128_si256(a, 1); return padd(lane0, lane1); } template <> EIGEN_STRONG_INLINE bfloat16 predux(const Packet16bf& p) { return static_cast(predux(Bf16ToF32(p))); } template <> EIGEN_STRONG_INLINE bfloat16 predux_mul(const Packet16bf& from) { return static_cast(predux_mul(Bf16ToF32(from))); } template <> EIGEN_STRONG_INLINE bfloat16 predux_min(const Packet16bf& from) { return static_cast(predux_min(Bf16ToF32(from))); } template <> EIGEN_STRONG_INLINE bfloat16 predux_max(const Packet16bf& from) { return static_cast(predux_max(Bf16ToF32(from))); } template <> EIGEN_STRONG_INLINE Packet16bf preverse(const Packet16bf& a) { __m256i m = _mm256_setr_epi8(14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1, 14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1); Packet16bf res; // Swap hi and lo first because shuffle is in 128-bit lanes. res = _mm256_permute2x128_si256(a, a, 1); // Shuffle 8-bit values in src within 2*128-bit lanes. return _mm256_shuffle_epi8(res, m); } template <> EIGEN_STRONG_INLINE Packet16bf pgather(const bfloat16* from, Index stride) { return _mm256_set_epi16( from[15*stride].value, from[14*stride].value, from[13*stride].value, from[12*stride].value, from[11*stride].value, from[10*stride].value, from[9*stride].value, from[8*stride].value, from[7*stride].value, from[6*stride].value, from[5*stride].value, from[4*stride].value, from[3*stride].value, from[2*stride].value, from[1*stride].value, from[0*stride].value); } template <> EIGEN_STRONG_INLINE void pscatter(bfloat16* to, const Packet16bf& from, Index stride) { EIGEN_ALIGN64 bfloat16 aux[16]; pstore(aux, from); to[stride*0] = aux[0]; to[stride*1] = aux[1]; to[stride*2] = aux[2]; to[stride*3] = aux[3]; to[stride*4] = aux[4]; to[stride*5] = aux[5]; to[stride*6] = aux[6]; to[stride*7] = aux[7]; to[stride*8] = aux[8]; to[stride*9] = aux[9]; to[stride*10] = aux[10]; to[stride*11] = aux[11]; to[stride*12] = aux[12]; to[stride*13] = aux[13]; to[stride*14] = aux[14]; to[stride*15] = aux[15]; } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { __m256i a = kernel.packet[0]; __m256i b = kernel.packet[1]; __m256i c = kernel.packet[2]; __m256i d = kernel.packet[3]; __m256i e = kernel.packet[4]; __m256i f = kernel.packet[5]; __m256i g = kernel.packet[6]; __m256i h = kernel.packet[7]; __m256i i = kernel.packet[8]; __m256i j = kernel.packet[9]; __m256i k = kernel.packet[10]; __m256i l = kernel.packet[11]; __m256i m = kernel.packet[12]; __m256i n = kernel.packet[13]; __m256i o = kernel.packet[14]; __m256i p = kernel.packet[15]; __m256i ab_07 = _mm256_unpacklo_epi16(a, b); __m256i cd_07 = _mm256_unpacklo_epi16(c, d); __m256i ef_07 = _mm256_unpacklo_epi16(e, f); __m256i gh_07 = _mm256_unpacklo_epi16(g, h); __m256i ij_07 = _mm256_unpacklo_epi16(i, j); __m256i kl_07 = _mm256_unpacklo_epi16(k, l); __m256i mn_07 = _mm256_unpacklo_epi16(m, n); __m256i op_07 = _mm256_unpacklo_epi16(o, p); __m256i ab_8f = _mm256_unpackhi_epi16(a, b); __m256i cd_8f = _mm256_unpackhi_epi16(c, d); __m256i ef_8f = _mm256_unpackhi_epi16(e, f); __m256i gh_8f = _mm256_unpackhi_epi16(g, h); __m256i ij_8f = _mm256_unpackhi_epi16(i, j); __m256i kl_8f = _mm256_unpackhi_epi16(k, l); __m256i mn_8f = _mm256_unpackhi_epi16(m, n); __m256i op_8f = _mm256_unpackhi_epi16(o, p); __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07); __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07); __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07); __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07); __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07); __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07); __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07); __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07); __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f); __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f); __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f); __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f); __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f); __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f); __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f); __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f); __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03); __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03); __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03); __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03); __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47); __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47); __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47); __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47); __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b); __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b); __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b); __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b); __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf); __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf); __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf); __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf); // NOTE: no unpacklo/hi instr in this case, so using permute instr. kernel.packet[0] = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20); kernel.packet[1] = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20); kernel.packet[2] = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20); kernel.packet[3] = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20); kernel.packet[4] = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20); kernel.packet[5] = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20); kernel.packet[6] = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20); kernel.packet[7] = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20); kernel.packet[8] = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31); kernel.packet[9] = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31); kernel.packet[10] = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31); kernel.packet[11] = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31); kernel.packet[12] = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31); kernel.packet[13] = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31); kernel.packet[14] = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31); kernel.packet[15] = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31); } EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { __m256i a = kernel.packet[0]; __m256i b = kernel.packet[1]; __m256i c = kernel.packet[2]; __m256i d = kernel.packet[3]; __m256i ab_07 = _mm256_unpacklo_epi16(a, b); __m256i cd_07 = _mm256_unpacklo_epi16(c, d); __m256i ab_8f = _mm256_unpackhi_epi16(a, b); __m256i cd_8f = _mm256_unpackhi_epi16(c, d); __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07); __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07); __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f); __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f); // NOTE: no unpacklo/hi instr in this case, so using permute instr. kernel.packet[0] = _mm256_permute2x128_si256(abcd_03, abcd_47, 0x20); kernel.packet[1] = _mm256_permute2x128_si256(abcd_8b, abcd_cf, 0x20); kernel.packet[2] = _mm256_permute2x128_si256(abcd_03, abcd_47, 0x31); kernel.packet[3] = _mm256_permute2x128_si256(abcd_8b, abcd_cf, 0x31); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_PACKET_MATH_AVX512_H RcppEigen/inst/include/Eigen/src/Core/arch/AVX512/MathFunctions.h0000644000176200001440000003204014567757725024022 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2016 Pedro Gonnet (pedro.gonnet@gmail.com) // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef THIRD_PARTY_EIGEN3_EIGEN_SRC_CORE_ARCH_AVX512_MATHFUNCTIONS_H_ #define THIRD_PARTY_EIGEN3_EIGEN_SRC_CORE_ARCH_AVX512_MATHFUNCTIONS_H_ namespace Eigen { namespace internal { // Disable the code for older versions of gcc that don't support many of the required avx512 instrinsics. #if EIGEN_GNUC_AT_LEAST(5, 3) || EIGEN_COMP_CLANG || EIGEN_COMP_MSVC >= 1923 #define _EIGEN_DECLARE_CONST_Packet16f(NAME, X) \ const Packet16f p16f_##NAME = pset1(X) #define _EIGEN_DECLARE_CONST_Packet16f_FROM_INT(NAME, X) \ const Packet16f p16f_##NAME = preinterpret(pset1(X)) #define _EIGEN_DECLARE_CONST_Packet8d(NAME, X) \ const Packet8d p8d_##NAME = pset1(X) #define _EIGEN_DECLARE_CONST_Packet8d_FROM_INT64(NAME, X) \ const Packet8d p8d_##NAME = _mm512_castsi512_pd(_mm512_set1_epi64(X)) #define _EIGEN_DECLARE_CONST_Packet16bf(NAME, X) \ const Packet16bf p16bf_##NAME = pset1(X) #define _EIGEN_DECLARE_CONST_Packet16bf_FROM_INT(NAME, X) \ const Packet16bf p16bf_##NAME = preinterpret(pset1(X)) template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f plog(const Packet16f& _x) { return plog_float(_x); } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8d plog(const Packet8d& _x) { return plog_double(_x); } F16_PACKET_FUNCTION(Packet16f, Packet16h, plog) BF16_PACKET_FUNCTION(Packet16f, Packet16bf, plog) template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f plog2(const Packet16f& _x) { return plog2_float(_x); } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8d plog2(const Packet8d& _x) { return plog2_double(_x); } F16_PACKET_FUNCTION(Packet16f, Packet16h, plog2) BF16_PACKET_FUNCTION(Packet16f, Packet16bf, plog2) // Exponential function. Works by writing "x = m*log(2) + r" where // "m = floor(x/log(2)+1/2)" and "r" is the remainder. The result is then // "exp(x) = 2^m*exp(r)" where exp(r) is in the range [-1,1). template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f pexp(const Packet16f& _x) { _EIGEN_DECLARE_CONST_Packet16f(1, 1.0f); _EIGEN_DECLARE_CONST_Packet16f(half, 0.5f); _EIGEN_DECLARE_CONST_Packet16f(127, 127.0f); _EIGEN_DECLARE_CONST_Packet16f(exp_hi, 88.3762626647950f); _EIGEN_DECLARE_CONST_Packet16f(exp_lo, -88.3762626647949f); _EIGEN_DECLARE_CONST_Packet16f(cephes_LOG2EF, 1.44269504088896341f); _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p0, 1.9875691500E-4f); _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p1, 1.3981999507E-3f); _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p2, 8.3334519073E-3f); _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p3, 4.1665795894E-2f); _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p4, 1.6666665459E-1f); _EIGEN_DECLARE_CONST_Packet16f(cephes_exp_p5, 5.0000001201E-1f); // Clamp x. Packet16f x = pmax(pmin(_x, p16f_exp_hi), p16f_exp_lo); // Express exp(x) as exp(m*ln(2) + r), start by extracting // m = floor(x/ln(2) + 0.5). Packet16f m = _mm512_floor_ps(pmadd(x, p16f_cephes_LOG2EF, p16f_half)); // Get r = x - m*ln(2). Note that we can do this without losing more than one // ulp precision due to the FMA instruction. _EIGEN_DECLARE_CONST_Packet16f(nln2, -0.6931471805599453f); Packet16f r = _mm512_fmadd_ps(m, p16f_nln2, x); Packet16f r2 = pmul(r, r); Packet16f r3 = pmul(r2, r); // Evaluate the polynomial approximant,improved by instruction-level parallelism. Packet16f y, y1, y2; y = pmadd(p16f_cephes_exp_p0, r, p16f_cephes_exp_p1); y1 = pmadd(p16f_cephes_exp_p3, r, p16f_cephes_exp_p4); y2 = padd(r, p16f_1); y = pmadd(y, r, p16f_cephes_exp_p2); y1 = pmadd(y1, r, p16f_cephes_exp_p5); y = pmadd(y, r3, y1); y = pmadd(y, r2, y2); // Build emm0 = 2^m. Packet16i emm0 = _mm512_cvttps_epi32(padd(m, p16f_127)); emm0 = _mm512_slli_epi32(emm0, 23); // Return 2^m * exp(r). return pmax(pmul(y, _mm512_castsi512_ps(emm0)), _x); } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8d pexp(const Packet8d& _x) { return pexp_double(_x); } F16_PACKET_FUNCTION(Packet16f, Packet16h, pexp) BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pexp) template <> EIGEN_STRONG_INLINE Packet16h pfrexp(const Packet16h& a, Packet16h& exponent) { Packet16f fexponent; const Packet16h out = float2half(pfrexp(half2float(a), fexponent)); exponent = float2half(fexponent); return out; } template <> EIGEN_STRONG_INLINE Packet16h pldexp(const Packet16h& a, const Packet16h& exponent) { return float2half(pldexp(half2float(a), half2float(exponent))); } template <> EIGEN_STRONG_INLINE Packet16bf pfrexp(const Packet16bf& a, Packet16bf& exponent) { Packet16f fexponent; const Packet16bf out = F32ToBf16(pfrexp(Bf16ToF32(a), fexponent)); exponent = F32ToBf16(fexponent); return out; } template <> EIGEN_STRONG_INLINE Packet16bf pldexp(const Packet16bf& a, const Packet16bf& exponent) { return F32ToBf16(pldexp(Bf16ToF32(a), Bf16ToF32(exponent))); } // Functions for sqrt. // The EIGEN_FAST_MATH version uses the _mm_rsqrt_ps approximation and one step // of Newton's method, at a cost of 1-2 bits of precision as opposed to the // exact solution. The main advantage of this approach is not just speed, but // also the fact that it can be inlined and pipelined with other computations, // further reducing its effective latency. #if EIGEN_FAST_MATH template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f psqrt(const Packet16f& _x) { Packet16f neg_half = pmul(_x, pset1(-.5f)); __mmask16 denormal_mask = _mm512_kand( _mm512_cmp_ps_mask(_x, pset1((std::numeric_limits::min)()), _CMP_LT_OQ), _mm512_cmp_ps_mask(_x, _mm512_setzero_ps(), _CMP_GE_OQ)); Packet16f x = _mm512_rsqrt14_ps(_x); // Do a single step of Newton's iteration. x = pmul(x, pmadd(neg_half, pmul(x, x), pset1(1.5f))); // Flush results for denormals to zero. return _mm512_mask_blend_ps(denormal_mask, pmul(_x,x), _mm512_setzero_ps()); } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8d psqrt(const Packet8d& _x) { Packet8d neg_half = pmul(_x, pset1(-.5)); __mmask16 denormal_mask = _mm512_kand( _mm512_cmp_pd_mask(_x, pset1((std::numeric_limits::min)()), _CMP_LT_OQ), _mm512_cmp_pd_mask(_x, _mm512_setzero_pd(), _CMP_GE_OQ)); Packet8d x = _mm512_rsqrt14_pd(_x); // Do a single step of Newton's iteration. x = pmul(x, pmadd(neg_half, pmul(x, x), pset1(1.5))); // Do a second step of Newton's iteration. x = pmul(x, pmadd(neg_half, pmul(x, x), pset1(1.5))); return _mm512_mask_blend_pd(denormal_mask, pmul(_x,x), _mm512_setzero_pd()); } #else template <> EIGEN_STRONG_INLINE Packet16f psqrt(const Packet16f& x) { return _mm512_sqrt_ps(x); } template <> EIGEN_STRONG_INLINE Packet8d psqrt(const Packet8d& x) { return _mm512_sqrt_pd(x); } #endif F16_PACKET_FUNCTION(Packet16f, Packet16h, psqrt) BF16_PACKET_FUNCTION(Packet16f, Packet16bf, psqrt) // prsqrt for float. #if defined(EIGEN_VECTORIZE_AVX512ER) template <> EIGEN_STRONG_INLINE Packet16f prsqrt(const Packet16f& x) { return _mm512_rsqrt28_ps(x); } #elif EIGEN_FAST_MATH template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f prsqrt(const Packet16f& _x) { _EIGEN_DECLARE_CONST_Packet16f_FROM_INT(inf, 0x7f800000); _EIGEN_DECLARE_CONST_Packet16f(one_point_five, 1.5f); _EIGEN_DECLARE_CONST_Packet16f(minus_half, -0.5f); Packet16f neg_half = pmul(_x, p16f_minus_half); // Identity infinite, negative and denormal arguments. __mmask16 inf_mask = _mm512_cmp_ps_mask(_x, p16f_inf, _CMP_EQ_OQ); __mmask16 not_pos_mask = _mm512_cmp_ps_mask(_x, _mm512_setzero_ps(), _CMP_LE_OQ); __mmask16 not_finite_pos_mask = not_pos_mask | inf_mask; // Compute an approximate result using the rsqrt intrinsic, forcing +inf // for denormals for consistency with AVX and SSE implementations. Packet16f y_approx = _mm512_rsqrt14_ps(_x); // Do a single step of Newton-Raphson iteration to improve the approximation. // This uses the formula y_{n+1} = y_n * (1.5 - y_n * (0.5 * x) * y_n). // It is essential to evaluate the inner term like this because forming // y_n^2 may over- or underflow. Packet16f y_newton = pmul(y_approx, pmadd(y_approx, pmul(neg_half, y_approx), p16f_one_point_five)); // Select the result of the Newton-Raphson step for positive finite arguments. // For other arguments, choose the output of the intrinsic. This will // return rsqrt(+inf) = 0, rsqrt(x) = NaN if x < 0, and rsqrt(0) = +inf. return _mm512_mask_blend_ps(not_finite_pos_mask, y_newton, y_approx); } #else template <> EIGEN_STRONG_INLINE Packet16f prsqrt(const Packet16f& x) { _EIGEN_DECLARE_CONST_Packet16f(one, 1.0f); return _mm512_div_ps(p16f_one, _mm512_sqrt_ps(x)); } #endif F16_PACKET_FUNCTION(Packet16f, Packet16h, prsqrt) BF16_PACKET_FUNCTION(Packet16f, Packet16bf, prsqrt) // prsqrt for double. #if EIGEN_FAST_MATH template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet8d prsqrt(const Packet8d& _x) { _EIGEN_DECLARE_CONST_Packet8d(one_point_five, 1.5); _EIGEN_DECLARE_CONST_Packet8d(minus_half, -0.5); _EIGEN_DECLARE_CONST_Packet8d_FROM_INT64(inf, 0x7ff0000000000000LL); Packet8d neg_half = pmul(_x, p8d_minus_half); // Identity infinite, negative and denormal arguments. __mmask8 inf_mask = _mm512_cmp_pd_mask(_x, p8d_inf, _CMP_EQ_OQ); __mmask8 not_pos_mask = _mm512_cmp_pd_mask(_x, _mm512_setzero_pd(), _CMP_LE_OQ); __mmask8 not_finite_pos_mask = not_pos_mask | inf_mask; // Compute an approximate result using the rsqrt intrinsic, forcing +inf // for denormals for consistency with AVX and SSE implementations. #if defined(EIGEN_VECTORIZE_AVX512ER) Packet8d y_approx = _mm512_rsqrt28_pd(_x); #else Packet8d y_approx = _mm512_rsqrt14_pd(_x); #endif // Do one or two steps of Newton-Raphson's to improve the approximation, depending on the // starting accuracy (either 2^-14 or 2^-28, depending on whether AVX512ER is available). // The Newton-Raphson algorithm has quadratic convergence and roughly doubles the number // of correct digits for each step. // This uses the formula y_{n+1} = y_n * (1.5 - y_n * (0.5 * x) * y_n). // It is essential to evaluate the inner term like this because forming // y_n^2 may over- or underflow. Packet8d y_newton = pmul(y_approx, pmadd(neg_half, pmul(y_approx, y_approx), p8d_one_point_five)); #if !defined(EIGEN_VECTORIZE_AVX512ER) y_newton = pmul(y_newton, pmadd(y_newton, pmul(neg_half, y_newton), p8d_one_point_five)); #endif // Select the result of the Newton-Raphson step for positive finite arguments. // For other arguments, choose the output of the intrinsic. This will // return rsqrt(+inf) = 0, rsqrt(x) = NaN if x < 0, and rsqrt(0) = +inf. return _mm512_mask_blend_pd(not_finite_pos_mask, y_newton, y_approx); } #else template <> EIGEN_STRONG_INLINE Packet8d prsqrt(const Packet8d& x) { _EIGEN_DECLARE_CONST_Packet8d(one, 1.0f); return _mm512_div_pd(p8d_one, _mm512_sqrt_pd(x)); } #endif template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f plog1p(const Packet16f& _x) { return generic_plog1p(_x); } F16_PACKET_FUNCTION(Packet16f, Packet16h, plog1p) BF16_PACKET_FUNCTION(Packet16f, Packet16bf, plog1p) template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f pexpm1(const Packet16f& _x) { return generic_expm1(_x); } F16_PACKET_FUNCTION(Packet16f, Packet16h, pexpm1) BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pexpm1) #endif template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f psin(const Packet16f& _x) { return psin_float(_x); } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f pcos(const Packet16f& _x) { return pcos_float(_x); } template <> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet16f ptanh(const Packet16f& _x) { return internal::generic_fast_tanh_float(_x); } F16_PACKET_FUNCTION(Packet16f, Packet16h, psin) F16_PACKET_FUNCTION(Packet16f, Packet16h, pcos) F16_PACKET_FUNCTION(Packet16f, Packet16h, ptanh) BF16_PACKET_FUNCTION(Packet16f, Packet16bf, psin) BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pcos) BF16_PACKET_FUNCTION(Packet16f, Packet16bf, ptanh) } // end namespace internal } // end namespace Eigen #endif // THIRD_PARTY_EIGEN3_EIGEN_SRC_CORE_ARCH_AVX512_MATHFUNCTIONS_H_ RcppEigen/inst/include/Eigen/src/Core/arch/SVE/0000755000176200001440000000000014567757725020637 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/arch/SVE/TypeCasting.h0000644000176200001440000000250714567757725023246 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2020, Arm Limited and Contributors // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_TYPE_CASTING_SVE_H #define EIGEN_TYPE_CASTING_SVE_H namespace Eigen { namespace internal { template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; }; template <> EIGEN_STRONG_INLINE PacketXf pcast(const PacketXi& a) { return svcvt_f32_s32_z(svptrue_b32(), a); } template <> EIGEN_STRONG_INLINE PacketXi pcast(const PacketXf& a) { return svcvt_s32_f32_z(svptrue_b32(), a); } template <> EIGEN_STRONG_INLINE PacketXf preinterpret(const PacketXi& a) { return svreinterpret_f32_s32(a); } template <> EIGEN_STRONG_INLINE PacketXi preinterpret(const PacketXf& a) { return svreinterpret_s32_f32(a); } } // namespace internal } // namespace Eigen #endif // EIGEN_TYPE_CASTING_SVE_H RcppEigen/inst/include/Eigen/src/Core/arch/SVE/PacketMath.h0000644000176200001440000005132014567757725023032 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2020, Arm Limited and Contributors // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_PACKET_MATH_SVE_H #define EIGEN_PACKET_MATH_SVE_H namespace Eigen { namespace internal { #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 #endif #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD #endif #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32 template struct sve_packet_size_selector { enum { size = SVEVectorLength / (sizeof(Scalar) * CHAR_BIT) }; }; /********************************* int32 **************************************/ typedef svint32_t PacketXi __attribute__((arm_sve_vector_bits(EIGEN_ARM64_SVE_VL))); template <> struct packet_traits : default_packet_traits { typedef PacketXi type; typedef PacketXi half; // Half not implemented yet enum { Vectorizable = 1, AlignedOnScalar = 1, size = sve_packet_size_selector::size, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasNegate = 1, HasAbs = 1, HasArg = 0, HasAbs2 = 1, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasBlend = 0, HasReduxp = 0 // Not implemented in SVE }; }; template <> struct unpacket_traits { typedef numext::int32_t type; typedef PacketXi half; // Half not yet implemented enum { size = sve_packet_size_selector::size, alignment = Aligned64, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template <> EIGEN_STRONG_INLINE void prefetch(const numext::int32_t* addr) { svprfw(svptrue_b32(), addr, SV_PLDL1KEEP); } template <> EIGEN_STRONG_INLINE PacketXi pset1(const numext::int32_t& from) { return svdup_n_s32(from); } template <> EIGEN_STRONG_INLINE PacketXi plset(const numext::int32_t& a) { numext::int32_t c[packet_traits::size]; for (int i = 0; i < packet_traits::size; i++) c[i] = i; return svadd_s32_z(svptrue_b32(), pset1(a), svld1_s32(svptrue_b32(), c)); } template <> EIGEN_STRONG_INLINE PacketXi padd(const PacketXi& a, const PacketXi& b) { return svadd_s32_z(svptrue_b32(), a, b); } template <> EIGEN_STRONG_INLINE PacketXi psub(const PacketXi& a, const PacketXi& b) { return svsub_s32_z(svptrue_b32(), a, b); } template <> EIGEN_STRONG_INLINE PacketXi pnegate(const PacketXi& a) { return svneg_s32_z(svptrue_b32(), a); } template <> EIGEN_STRONG_INLINE PacketXi pconj(const PacketXi& a) { return a; } template <> EIGEN_STRONG_INLINE PacketXi pmul(const PacketXi& a, const PacketXi& b) { return svmul_s32_z(svptrue_b32(), a, b); } template <> EIGEN_STRONG_INLINE PacketXi pdiv(const PacketXi& a, const PacketXi& b) { return svdiv_s32_z(svptrue_b32(), a, b); } template <> EIGEN_STRONG_INLINE PacketXi pmadd(const PacketXi& a, const PacketXi& b, const PacketXi& c) { return svmla_s32_z(svptrue_b32(), c, a, b); } template <> EIGEN_STRONG_INLINE PacketXi pmin(const PacketXi& a, const PacketXi& b) { return svmin_s32_z(svptrue_b32(), a, b); } template <> EIGEN_STRONG_INLINE PacketXi pmax(const PacketXi& a, const PacketXi& b) { return svmax_s32_z(svptrue_b32(), a, b); } template <> EIGEN_STRONG_INLINE PacketXi pcmp_le(const PacketXi& a, const PacketXi& b) { return svdup_n_s32_z(svcmplt_s32(svptrue_b32(), a, b), 0xffffffffu); } template <> EIGEN_STRONG_INLINE PacketXi pcmp_lt(const PacketXi& a, const PacketXi& b) { return svdup_n_s32_z(svcmplt_s32(svptrue_b32(), a, b), 0xffffffffu); } template <> EIGEN_STRONG_INLINE PacketXi pcmp_eq(const PacketXi& a, const PacketXi& b) { return svdup_n_s32_z(svcmpeq_s32(svptrue_b32(), a, b), 0xffffffffu); } template <> EIGEN_STRONG_INLINE PacketXi ptrue(const PacketXi& /*a*/) { return svdup_n_s32_z(svptrue_b32(), 0xffffffffu); } template <> EIGEN_STRONG_INLINE PacketXi pzero(const PacketXi& /*a*/) { return svdup_n_s32_z(svptrue_b32(), 0); } template <> EIGEN_STRONG_INLINE PacketXi pand(const PacketXi& a, const PacketXi& b) { return svand_s32_z(svptrue_b32(), a, b); } template <> EIGEN_STRONG_INLINE PacketXi por(const PacketXi& a, const PacketXi& b) { return svorr_s32_z(svptrue_b32(), a, b); } template <> EIGEN_STRONG_INLINE PacketXi pxor(const PacketXi& a, const PacketXi& b) { return sveor_s32_z(svptrue_b32(), a, b); } template <> EIGEN_STRONG_INLINE PacketXi pandnot(const PacketXi& a, const PacketXi& b) { return svbic_s32_z(svptrue_b32(), a, b); } template EIGEN_STRONG_INLINE PacketXi parithmetic_shift_right(PacketXi a) { return svasrd_n_s32_z(svptrue_b32(), a, N); } template EIGEN_STRONG_INLINE PacketXi plogical_shift_right(PacketXi a) { return svreinterpret_s32_u32(svlsr_u32_z(svptrue_b32(), svreinterpret_u32_s32(a), svdup_n_u32_z(svptrue_b32(), N))); } template EIGEN_STRONG_INLINE PacketXi plogical_shift_left(PacketXi a) { return svlsl_s32_z(svptrue_b32(), a, svdup_n_u32_z(svptrue_b32(), N)); } template <> EIGEN_STRONG_INLINE PacketXi pload(const numext::int32_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return svld1_s32(svptrue_b32(), from); } template <> EIGEN_STRONG_INLINE PacketXi ploadu(const numext::int32_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return svld1_s32(svptrue_b32(), from); } template <> EIGEN_STRONG_INLINE PacketXi ploaddup(const numext::int32_t* from) { svuint32_t indices = svindex_u32(0, 1); // index {base=0, base+step=1, base+step*2, ...} indices = svzip1_u32(indices, indices); // index in the format {a0, a0, a1, a1, a2, a2, ...} return svld1_gather_u32index_s32(svptrue_b32(), from, indices); } template <> EIGEN_STRONG_INLINE PacketXi ploadquad(const numext::int32_t* from) { svuint32_t indices = svindex_u32(0, 1); // index {base=0, base+step=1, base+step*2, ...} indices = svzip1_u32(indices, indices); // index in the format {a0, a0, a1, a1, a2, a2, ...} indices = svzip1_u32(indices, indices); // index in the format {a0, a0, a0, a0, a1, a1, a1, a1, ...} return svld1_gather_u32index_s32(svptrue_b32(), from, indices); } template <> EIGEN_STRONG_INLINE void pstore(numext::int32_t* to, const PacketXi& from) { EIGEN_DEBUG_ALIGNED_STORE svst1_s32(svptrue_b32(), to, from); } template <> EIGEN_STRONG_INLINE void pstoreu(numext::int32_t* to, const PacketXi& from) { EIGEN_DEBUG_UNALIGNED_STORE svst1_s32(svptrue_b32(), to, from); } template <> EIGEN_DEVICE_FUNC inline PacketXi pgather(const numext::int32_t* from, Index stride) { // Indice format: {base=0, base+stride, base+stride*2, base+stride*3, ...} svint32_t indices = svindex_s32(0, stride); return svld1_gather_s32index_s32(svptrue_b32(), from, indices); } template <> EIGEN_DEVICE_FUNC inline void pscatter(numext::int32_t* to, const PacketXi& from, Index stride) { // Indice format: {base=0, base+stride, base+stride*2, base+stride*3, ...} svint32_t indices = svindex_s32(0, stride); svst1_scatter_s32index_s32(svptrue_b32(), to, indices, from); } template <> EIGEN_STRONG_INLINE numext::int32_t pfirst(const PacketXi& a) { // svlasta returns the first element if all predicate bits are 0 return svlasta_s32(svpfalse_b(), a); } template <> EIGEN_STRONG_INLINE PacketXi preverse(const PacketXi& a) { return svrev_s32(a); } template <> EIGEN_STRONG_INLINE PacketXi pabs(const PacketXi& a) { return svabs_s32_z(svptrue_b32(), a); } template <> EIGEN_STRONG_INLINE numext::int32_t predux(const PacketXi& a) { return static_cast(svaddv_s32(svptrue_b32(), a)); } template <> EIGEN_STRONG_INLINE numext::int32_t predux_mul(const PacketXi& a) { EIGEN_STATIC_ASSERT((EIGEN_ARM64_SVE_VL % 128 == 0), EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT); // Multiply the vector by its reverse svint32_t prod = svmul_s32_z(svptrue_b32(), a, svrev_s32(a)); svint32_t half_prod; // Extract the high half of the vector. Depending on the VL more reductions need to be done if (EIGEN_ARM64_SVE_VL >= 2048) { half_prod = svtbl_s32(prod, svindex_u32(32, 1)); prod = svmul_s32_z(svptrue_b32(), prod, half_prod); } if (EIGEN_ARM64_SVE_VL >= 1024) { half_prod = svtbl_s32(prod, svindex_u32(16, 1)); prod = svmul_s32_z(svptrue_b32(), prod, half_prod); } if (EIGEN_ARM64_SVE_VL >= 512) { half_prod = svtbl_s32(prod, svindex_u32(8, 1)); prod = svmul_s32_z(svptrue_b32(), prod, half_prod); } if (EIGEN_ARM64_SVE_VL >= 256) { half_prod = svtbl_s32(prod, svindex_u32(4, 1)); prod = svmul_s32_z(svptrue_b32(), prod, half_prod); } // Last reduction half_prod = svtbl_s32(prod, svindex_u32(2, 1)); prod = svmul_s32_z(svptrue_b32(), prod, half_prod); // The reduction is done to the first element. return pfirst(prod); } template <> EIGEN_STRONG_INLINE numext::int32_t predux_min(const PacketXi& a) { return svminv_s32(svptrue_b32(), a); } template <> EIGEN_STRONG_INLINE numext::int32_t predux_max(const PacketXi& a) { return svmaxv_s32(svptrue_b32(), a); } template EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { int buffer[packet_traits::size * N] = {0}; int i = 0; PacketXi stride_index = svindex_s32(0, N); for (i = 0; i < N; i++) { svst1_scatter_s32index_s32(svptrue_b32(), buffer + i, stride_index, kernel.packet[i]); } for (i = 0; i < N; i++) { kernel.packet[i] = svld1_s32(svptrue_b32(), buffer + i * packet_traits::size); } } /********************************* float32 ************************************/ typedef svfloat32_t PacketXf __attribute__((arm_sve_vector_bits(EIGEN_ARM64_SVE_VL))); template <> struct packet_traits : default_packet_traits { typedef PacketXf type; typedef PacketXf half; enum { Vectorizable = 1, AlignedOnScalar = 1, size = sve_packet_size_selector::size, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasShift = 1, HasMul = 1, HasNegate = 1, HasAbs = 1, HasArg = 0, HasAbs2 = 1, HasMin = 1, HasMax = 1, HasConj = 1, HasSetLinear = 0, HasBlend = 0, HasReduxp = 0, // Not implemented in SVE HasDiv = 1, HasFloor = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasLog = 1, HasExp = 1, HasSqrt = 0, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH }; }; template <> struct unpacket_traits { typedef float type; typedef PacketXf half; // Half not yet implemented typedef PacketXi integer_packet; enum { size = sve_packet_size_selector::size, alignment = Aligned64, vectorizable = true, masked_load_available = false, masked_store_available = false }; }; template <> EIGEN_STRONG_INLINE PacketXf pset1(const float& from) { return svdup_n_f32(from); } template <> EIGEN_STRONG_INLINE PacketXf pset1frombits(numext::uint32_t from) { return svreinterpret_f32_u32(svdup_n_u32_z(svptrue_b32(), from)); } template <> EIGEN_STRONG_INLINE PacketXf plset(const float& a) { float c[packet_traits::size]; for (int i = 0; i < packet_traits::size; i++) c[i] = i; return svadd_f32_z(svptrue_b32(), pset1(a), svld1_f32(svptrue_b32(), c)); } template <> EIGEN_STRONG_INLINE PacketXf padd(const PacketXf& a, const PacketXf& b) { return svadd_f32_z(svptrue_b32(), a, b); } template <> EIGEN_STRONG_INLINE PacketXf psub(const PacketXf& a, const PacketXf& b) { return svsub_f32_z(svptrue_b32(), a, b); } template <> EIGEN_STRONG_INLINE PacketXf pnegate(const PacketXf& a) { return svneg_f32_z(svptrue_b32(), a); } template <> EIGEN_STRONG_INLINE PacketXf pconj(const PacketXf& a) { return a; } template <> EIGEN_STRONG_INLINE PacketXf pmul(const PacketXf& a, const PacketXf& b) { return svmul_f32_z(svptrue_b32(), a, b); } template <> EIGEN_STRONG_INLINE PacketXf pdiv(const PacketXf& a, const PacketXf& b) { return svdiv_f32_z(svptrue_b32(), a, b); } template <> EIGEN_STRONG_INLINE PacketXf pmadd(const PacketXf& a, const PacketXf& b, const PacketXf& c) { return svmla_f32_z(svptrue_b32(), c, a, b); } template <> EIGEN_STRONG_INLINE PacketXf pmin(const PacketXf& a, const PacketXf& b) { return svmin_f32_z(svptrue_b32(), a, b); } template <> EIGEN_STRONG_INLINE PacketXf pmin(const PacketXf& a, const PacketXf& b) { return pmin(a, b); } template <> EIGEN_STRONG_INLINE PacketXf pmin(const PacketXf& a, const PacketXf& b) { return svminnm_f32_z(svptrue_b32(), a, b); } template <> EIGEN_STRONG_INLINE PacketXf pmax(const PacketXf& a, const PacketXf& b) { return svmax_f32_z(svptrue_b32(), a, b); } template <> EIGEN_STRONG_INLINE PacketXf pmax(const PacketXf& a, const PacketXf& b) { return pmax(a, b); } template <> EIGEN_STRONG_INLINE PacketXf pmax(const PacketXf& a, const PacketXf& b) { return svmaxnm_f32_z(svptrue_b32(), a, b); } // Float comparisons in SVE return svbool (predicate). Use svdup to set active // lanes to 1 (0xffffffffu) and inactive lanes to 0. template <> EIGEN_STRONG_INLINE PacketXf pcmp_le(const PacketXf& a, const PacketXf& b) { return svreinterpret_f32_u32(svdup_n_u32_z(svcmplt_f32(svptrue_b32(), a, b), 0xffffffffu)); } template <> EIGEN_STRONG_INLINE PacketXf pcmp_lt(const PacketXf& a, const PacketXf& b) { return svreinterpret_f32_u32(svdup_n_u32_z(svcmplt_f32(svptrue_b32(), a, b), 0xffffffffu)); } template <> EIGEN_STRONG_INLINE PacketXf pcmp_eq(const PacketXf& a, const PacketXf& b) { return svreinterpret_f32_u32(svdup_n_u32_z(svcmpeq_f32(svptrue_b32(), a, b), 0xffffffffu)); } // Do a predicate inverse (svnot_b_z) on the predicate resulted from the // greater/equal comparison (svcmpge_f32). Then fill a float vector with the // active elements. template <> EIGEN_STRONG_INLINE PacketXf pcmp_lt_or_nan(const PacketXf& a, const PacketXf& b) { return svreinterpret_f32_u32(svdup_n_u32_z(svnot_b_z(svptrue_b32(), svcmpge_f32(svptrue_b32(), a, b)), 0xffffffffu)); } template <> EIGEN_STRONG_INLINE PacketXf pfloor(const PacketXf& a) { return svrintm_f32_z(svptrue_b32(), a); } template <> EIGEN_STRONG_INLINE PacketXf ptrue(const PacketXf& /*a*/) { return svreinterpret_f32_u32(svdup_n_u32_z(svptrue_b32(), 0xffffffffu)); } // Logical Operations are not supported for float, so reinterpret casts template <> EIGEN_STRONG_INLINE PacketXf pand(const PacketXf& a, const PacketXf& b) { return svreinterpret_f32_u32(svand_u32_z(svptrue_b32(), svreinterpret_u32_f32(a), svreinterpret_u32_f32(b))); } template <> EIGEN_STRONG_INLINE PacketXf por(const PacketXf& a, const PacketXf& b) { return svreinterpret_f32_u32(svorr_u32_z(svptrue_b32(), svreinterpret_u32_f32(a), svreinterpret_u32_f32(b))); } template <> EIGEN_STRONG_INLINE PacketXf pxor(const PacketXf& a, const PacketXf& b) { return svreinterpret_f32_u32(sveor_u32_z(svptrue_b32(), svreinterpret_u32_f32(a), svreinterpret_u32_f32(b))); } template <> EIGEN_STRONG_INLINE PacketXf pandnot(const PacketXf& a, const PacketXf& b) { return svreinterpret_f32_u32(svbic_u32_z(svptrue_b32(), svreinterpret_u32_f32(a), svreinterpret_u32_f32(b))); } template <> EIGEN_STRONG_INLINE PacketXf pload(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return svld1_f32(svptrue_b32(), from); } template <> EIGEN_STRONG_INLINE PacketXf ploadu(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD return svld1_f32(svptrue_b32(), from); } template <> EIGEN_STRONG_INLINE PacketXf ploaddup(const float* from) { svuint32_t indices = svindex_u32(0, 1); // index {base=0, base+step=1, base+step*2, ...} indices = svzip1_u32(indices, indices); // index in the format {a0, a0, a1, a1, a2, a2, ...} return svld1_gather_u32index_f32(svptrue_b32(), from, indices); } template <> EIGEN_STRONG_INLINE PacketXf ploadquad(const float* from) { svuint32_t indices = svindex_u32(0, 1); // index {base=0, base+step=1, base+step*2, ...} indices = svzip1_u32(indices, indices); // index in the format {a0, a0, a1, a1, a2, a2, ...} indices = svzip1_u32(indices, indices); // index in the format {a0, a0, a0, a0, a1, a1, a1, a1, ...} return svld1_gather_u32index_f32(svptrue_b32(), from, indices); } template <> EIGEN_STRONG_INLINE void pstore(float* to, const PacketXf& from) { EIGEN_DEBUG_ALIGNED_STORE svst1_f32(svptrue_b32(), to, from); } template <> EIGEN_STRONG_INLINE void pstoreu(float* to, const PacketXf& from) { EIGEN_DEBUG_UNALIGNED_STORE svst1_f32(svptrue_b32(), to, from); } template <> EIGEN_DEVICE_FUNC inline PacketXf pgather(const float* from, Index stride) { // Indice format: {base=0, base+stride, base+stride*2, base+stride*3, ...} svint32_t indices = svindex_s32(0, stride); return svld1_gather_s32index_f32(svptrue_b32(), from, indices); } template <> EIGEN_DEVICE_FUNC inline void pscatter(float* to, const PacketXf& from, Index stride) { // Indice format: {base=0, base+stride, base+stride*2, base+stride*3, ...} svint32_t indices = svindex_s32(0, stride); svst1_scatter_s32index_f32(svptrue_b32(), to, indices, from); } template <> EIGEN_STRONG_INLINE float pfirst(const PacketXf& a) { // svlasta returns the first element if all predicate bits are 0 return svlasta_f32(svpfalse_b(), a); } template <> EIGEN_STRONG_INLINE PacketXf preverse(const PacketXf& a) { return svrev_f32(a); } template <> EIGEN_STRONG_INLINE PacketXf pabs(const PacketXf& a) { return svabs_f32_z(svptrue_b32(), a); } // TODO(tellenbach): Should this go into MathFunctions.h? If so, change for // all vector extensions and the generic version. template <> EIGEN_STRONG_INLINE PacketXf pfrexp(const PacketXf& a, PacketXf& exponent) { return pfrexp_generic(a, exponent); } template <> EIGEN_STRONG_INLINE float predux(const PacketXf& a) { return svaddv_f32(svptrue_b32(), a); } // Other reduction functions: // mul // Only works for SVE Vls multiple of 128 template <> EIGEN_STRONG_INLINE float predux_mul(const PacketXf& a) { EIGEN_STATIC_ASSERT((EIGEN_ARM64_SVE_VL % 128 == 0), EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT); // Multiply the vector by its reverse svfloat32_t prod = svmul_f32_z(svptrue_b32(), a, svrev_f32(a)); svfloat32_t half_prod; // Extract the high half of the vector. Depending on the VL more reductions need to be done if (EIGEN_ARM64_SVE_VL >= 2048) { half_prod = svtbl_f32(prod, svindex_u32(32, 1)); prod = svmul_f32_z(svptrue_b32(), prod, half_prod); } if (EIGEN_ARM64_SVE_VL >= 1024) { half_prod = svtbl_f32(prod, svindex_u32(16, 1)); prod = svmul_f32_z(svptrue_b32(), prod, half_prod); } if (EIGEN_ARM64_SVE_VL >= 512) { half_prod = svtbl_f32(prod, svindex_u32(8, 1)); prod = svmul_f32_z(svptrue_b32(), prod, half_prod); } if (EIGEN_ARM64_SVE_VL >= 256) { half_prod = svtbl_f32(prod, svindex_u32(4, 1)); prod = svmul_f32_z(svptrue_b32(), prod, half_prod); } // Last reduction half_prod = svtbl_f32(prod, svindex_u32(2, 1)); prod = svmul_f32_z(svptrue_b32(), prod, half_prod); // The reduction is done to the first element. return pfirst(prod); } template <> EIGEN_STRONG_INLINE float predux_min(const PacketXf& a) { return svminv_f32(svptrue_b32(), a); } template <> EIGEN_STRONG_INLINE float predux_max(const PacketXf& a) { return svmaxv_f32(svptrue_b32(), a); } template EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { float buffer[packet_traits::size * N] = {0}; int i = 0; PacketXi stride_index = svindex_s32(0, N); for (i = 0; i < N; i++) { svst1_scatter_s32index_f32(svptrue_b32(), buffer + i, stride_index, kernel.packet[i]); } for (i = 0; i < N; i++) { kernel.packet[i] = svld1_f32(svptrue_b32(), buffer + i * packet_traits::size); } } template<> EIGEN_STRONG_INLINE PacketXf pldexp(const PacketXf& a, const PacketXf& exponent) { return pldexp_generic(a, exponent); } } // namespace internal } // namespace Eigen #endif // EIGEN_PACKET_MATH_SVE_H RcppEigen/inst/include/Eigen/src/Core/arch/SVE/MathFunctions.h0000644000176200001440000000225214567757725023573 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2020, Arm Limited and Contributors // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_MATH_FUNCTIONS_SVE_H #define EIGEN_MATH_FUNCTIONS_SVE_H namespace Eigen { namespace internal { template <> EIGEN_STRONG_INLINE EIGEN_UNUSED PacketXf pexp(const PacketXf& x) { return pexp_float(x); } template <> EIGEN_STRONG_INLINE EIGEN_UNUSED PacketXf plog(const PacketXf& x) { return plog_float(x); } template <> EIGEN_STRONG_INLINE EIGEN_UNUSED PacketXf psin(const PacketXf& x) { return psin_float(x); } template <> EIGEN_STRONG_INLINE EIGEN_UNUSED PacketXf pcos(const PacketXf& x) { return pcos_float(x); } // Hyperbolic Tangent function. template <> EIGEN_STRONG_INLINE EIGEN_UNUSED PacketXf ptanh(const PacketXf& x) { return internal::generic_fast_tanh_float(x); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_MATH_FUNCTIONS_SVE_H RcppEigen/inst/include/Eigen/src/Core/arch/GPU/0000755000176200001440000000000014567757725020635 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/arch/GPU/TypeCasting.h0000644000176200001440000000432014567757725023237 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2016 Benoit Steiner // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_TYPE_CASTING_GPU_H #define EIGEN_TYPE_CASTING_GPU_H namespace Eigen { namespace internal { #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; }; template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcast(const half2& a, const half2& b) { float2 r1 = __half22float2(a); float2 r2 = __half22float2(b); return make_float4(r1.x, r1.y, r2.x, r2.y); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pcast(const float4& a, const float4& b) { Packet4h2 r; half2* r_alias=reinterpret_cast(&r); r_alias[0]=__floats2half2_rn(a.x,a.y); r_alias[1]=__floats2half2_rn(a.z,a.w); r_alias[2]=__floats2half2_rn(b.x,b.y); r_alias[3]=__floats2half2_rn(b.z,b.w); return r; } template <> struct type_casting_traits { enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; }; template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcast(const Packet4h2& a) { // Simply discard the second half of the input float4 r; const half2* a_alias=reinterpret_cast(&a); float2 r1 = __half22float2(a_alias[0]); float2 r2 = __half22float2(a_alias[1]); r.x=static_cast(r1.x); r.y=static_cast(r1.y); r.z=static_cast(r2.x); r.w=static_cast(r2.y); return r; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pcast(const float4& a) { // Simply discard the second half of the input return __floats2half2_rn(a.x, a.y); } #endif } // end namespace internal } // end namespace Eigen #endif // EIGEN_TYPE_CASTING_GPU_H RcppEigen/inst/include/Eigen/src/Core/arch/GPU/PacketMath.h0000644000176200001440000015732714567757725023046 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2014 Benoit Steiner // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_PACKET_MATH_GPU_H #define EIGEN_PACKET_MATH_GPU_H namespace Eigen { namespace internal { // Read-only data cached load available. #if defined(EIGEN_HIP_DEVICE_COMPILE) || (defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 350) #define EIGEN_GPU_HAS_LDG 1 #endif // FP16 math available. #if (defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 530) #define EIGEN_CUDA_HAS_FP16_ARITHMETIC 1 #endif #if defined(EIGEN_HIP_DEVICE_COMPILE) || defined(EIGEN_CUDA_HAS_FP16_ARITHMETIC) #define EIGEN_GPU_HAS_FP16_ARITHMETIC 1 #endif // Make sure this is only available when targeting a GPU: we don't want to // introduce conflicts between these packet_traits definitions and the ones // we'll use on the host side (SSE, AVX, ...) #if defined(EIGEN_GPUCC) && defined(EIGEN_USE_GPU) template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct packet_traits : default_packet_traits { typedef float4 type; typedef float4 half; enum { Vectorizable = 1, AlignedOnScalar = 1, size=4, HasHalfPacket = 0, HasDiv = 1, HasSin = 0, HasCos = 0, HasLog = 1, HasExp = 1, HasSqrt = 1, HasRsqrt = 1, HasLGamma = 1, HasDiGamma = 1, HasZeta = 1, HasPolygamma = 1, HasErf = 1, HasErfc = 1, HasNdtri = 1, HasBessel = 1, HasIGamma = 1, HasIGammaDerA = 1, HasGammaSampleDerAlpha = 1, HasIGammac = 1, HasBetaInc = 1, HasBlend = 0, HasFloor = 1, }; }; template<> struct packet_traits : default_packet_traits { typedef double2 type; typedef double2 half; enum { Vectorizable = 1, AlignedOnScalar = 1, size=2, HasHalfPacket = 0, HasDiv = 1, HasLog = 1, HasExp = 1, HasSqrt = 1, HasRsqrt = 1, HasLGamma = 1, HasDiGamma = 1, HasZeta = 1, HasPolygamma = 1, HasErf = 1, HasErfc = 1, HasNdtri = 1, HasBessel = 1, HasIGamma = 1, HasIGammaDerA = 1, HasGammaSampleDerAlpha = 1, HasIGammac = 1, HasBetaInc = 1, HasBlend = 0, HasFloor = 1, }; }; template<> struct unpacket_traits { typedef float type; enum {size=4, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef float4 half; }; template<> struct unpacket_traits { typedef double type; enum {size=2, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef double2 half; }; template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pset1(const float& from) { return make_float4(from, from, from, from); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pset1(const double& from) { return make_double2(from, from); } // We need to distinguish ‘clang as the CUDA compiler’ from ‘clang as the host compiler, // invoked by NVCC’ (e.g. on MacOS). The former needs to see both host and device implementation // of the functions, while the latter can only deal with one of them. #if defined(EIGEN_CUDA_ARCH) || defined(EIGEN_HIPCC) || (defined(EIGEN_CUDACC) && EIGEN_COMP_CLANG && !EIGEN_COMP_NVCC) namespace { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float bitwise_and(const float& a, const float& b) { return __int_as_float(__float_as_int(a) & __float_as_int(b)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double bitwise_and(const double& a, const double& b) { return __longlong_as_double(__double_as_longlong(a) & __double_as_longlong(b)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float bitwise_or(const float& a, const float& b) { return __int_as_float(__float_as_int(a) | __float_as_int(b)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double bitwise_or(const double& a, const double& b) { return __longlong_as_double(__double_as_longlong(a) | __double_as_longlong(b)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float bitwise_xor(const float& a, const float& b) { return __int_as_float(__float_as_int(a) ^ __float_as_int(b)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double bitwise_xor(const double& a, const double& b) { return __longlong_as_double(__double_as_longlong(a) ^ __double_as_longlong(b)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float bitwise_andnot(const float& a, const float& b) { return __int_as_float(__float_as_int(a) & ~__float_as_int(b)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double bitwise_andnot(const double& a, const double& b) { return __longlong_as_double(__double_as_longlong(a) & ~__double_as_longlong(b)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float eq_mask(const float& a, const float& b) { return __int_as_float(a == b ? 0xffffffffu : 0u); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double eq_mask(const double& a, const double& b) { return __longlong_as_double(a == b ? 0xffffffffffffffffull : 0ull); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float lt_mask(const float& a, const float& b) { return __int_as_float(a < b ? 0xffffffffu : 0u); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double lt_mask(const double& a, const double& b) { return __longlong_as_double(a < b ? 0xffffffffffffffffull : 0ull); } } // namespace template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pand(const float4& a, const float4& b) { return make_float4(bitwise_and(a.x, b.x), bitwise_and(a.y, b.y), bitwise_and(a.z, b.z), bitwise_and(a.w, b.w)); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pand(const double2& a, const double2& b) { return make_double2(bitwise_and(a.x, b.x), bitwise_and(a.y, b.y)); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 por(const float4& a, const float4& b) { return make_float4(bitwise_or(a.x, b.x), bitwise_or(a.y, b.y), bitwise_or(a.z, b.z), bitwise_or(a.w, b.w)); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 por(const double2& a, const double2& b) { return make_double2(bitwise_or(a.x, b.x), bitwise_or(a.y, b.y)); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pxor(const float4& a, const float4& b) { return make_float4(bitwise_xor(a.x, b.x), bitwise_xor(a.y, b.y), bitwise_xor(a.z, b.z), bitwise_xor(a.w, b.w)); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pxor(const double2& a, const double2& b) { return make_double2(bitwise_xor(a.x, b.x), bitwise_xor(a.y, b.y)); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pandnot(const float4& a, const float4& b) { return make_float4(bitwise_andnot(a.x, b.x), bitwise_andnot(a.y, b.y), bitwise_andnot(a.z, b.z), bitwise_andnot(a.w, b.w)); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pandnot(const double2& a, const double2& b) { return make_double2(bitwise_andnot(a.x, b.x), bitwise_andnot(a.y, b.y)); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcmp_eq(const float4& a, const float4& b) { return make_float4(eq_mask(a.x, b.x), eq_mask(a.y, b.y), eq_mask(a.z, b.z), eq_mask(a.w, b.w)); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcmp_lt(const float4& a, const float4& b) { return make_float4(lt_mask(a.x, b.x), lt_mask(a.y, b.y), lt_mask(a.z, b.z), lt_mask(a.w, b.w)); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pcmp_eq(const double2& a, const double2& b) { return make_double2(eq_mask(a.x, b.x), eq_mask(a.y, b.y)); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pcmp_lt(const double2& a, const double2& b) { return make_double2(lt_mask(a.x, b.x), lt_mask(a.y, b.y)); } #endif // defined(EIGEN_CUDA_ARCH) || defined(EIGEN_HIPCC) || (defined(EIGEN_CUDACC) && EIGEN_COMP_CLANG && !EIGEN_COMP_NVCC) template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 plset(const float& a) { return make_float4(a, a+1, a+2, a+3); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 plset(const double& a) { return make_double2(a, a+1); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 padd(const float4& a, const float4& b) { return make_float4(a.x+b.x, a.y+b.y, a.z+b.z, a.w+b.w); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 padd(const double2& a, const double2& b) { return make_double2(a.x+b.x, a.y+b.y); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 psub(const float4& a, const float4& b) { return make_float4(a.x-b.x, a.y-b.y, a.z-b.z, a.w-b.w); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 psub(const double2& a, const double2& b) { return make_double2(a.x-b.x, a.y-b.y); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pnegate(const float4& a) { return make_float4(-a.x, -a.y, -a.z, -a.w); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pnegate(const double2& a) { return make_double2(-a.x, -a.y); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pconj(const float4& a) { return a; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pconj(const double2& a) { return a; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pmul(const float4& a, const float4& b) { return make_float4(a.x*b.x, a.y*b.y, a.z*b.z, a.w*b.w); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pmul(const double2& a, const double2& b) { return make_double2(a.x*b.x, a.y*b.y); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pdiv(const float4& a, const float4& b) { return make_float4(a.x/b.x, a.y/b.y, a.z/b.z, a.w/b.w); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pdiv(const double2& a, const double2& b) { return make_double2(a.x/b.x, a.y/b.y); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pmin(const float4& a, const float4& b) { return make_float4(fminf(a.x, b.x), fminf(a.y, b.y), fminf(a.z, b.z), fminf(a.w, b.w)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pmin(const double2& a, const double2& b) { return make_double2(fmin(a.x, b.x), fmin(a.y, b.y)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pmax(const float4& a, const float4& b) { return make_float4(fmaxf(a.x, b.x), fmaxf(a.y, b.y), fmaxf(a.z, b.z), fmaxf(a.w, b.w)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pmax(const double2& a, const double2& b) { return make_double2(fmax(a.x, b.x), fmax(a.y, b.y)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pload(const float* from) { return *reinterpret_cast(from); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pload(const double* from) { return *reinterpret_cast(from); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 ploadu(const float* from) { return make_float4(from[0], from[1], from[2], from[3]); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 ploadu(const double* from) { return make_double2(from[0], from[1]); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 ploaddup(const float* from) { return make_float4(from[0], from[0], from[1], from[1]); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 ploaddup(const double* from) { return make_double2(from[0], from[0]); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstore(float* to, const float4& from) { *reinterpret_cast(to) = from; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstore(double* to, const double2& from) { *reinterpret_cast(to) = from; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstoreu(float* to, const float4& from) { to[0] = from.x; to[1] = from.y; to[2] = from.z; to[3] = from.w; } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstoreu(double* to, const double2& from) { to[0] = from.x; to[1] = from.y; } template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float4 ploadt_ro(const float* from) { #if defined(EIGEN_GPU_HAS_LDG) return __ldg((const float4*)from); #else return make_float4(from[0], from[1], from[2], from[3]); #endif } template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double2 ploadt_ro(const double* from) { #if defined(EIGEN_GPU_HAS_LDG) return __ldg((const double2*)from); #else return make_double2(from[0], from[1]); #endif } template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float4 ploadt_ro(const float* from) { #if defined(EIGEN_GPU_HAS_LDG) return make_float4(__ldg(from+0), __ldg(from+1), __ldg(from+2), __ldg(from+3)); #else return make_float4(from[0], from[1], from[2], from[3]); #endif } template<> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double2 ploadt_ro(const double* from) { #if defined(EIGEN_GPU_HAS_LDG) return make_double2(__ldg(from+0), __ldg(from+1)); #else return make_double2(from[0], from[1]); #endif } template<> EIGEN_DEVICE_FUNC inline float4 pgather(const float* from, Index stride) { return make_float4(from[0*stride], from[1*stride], from[2*stride], from[3*stride]); } template<> EIGEN_DEVICE_FUNC inline double2 pgather(const double* from, Index stride) { return make_double2(from[0*stride], from[1*stride]); } template<> EIGEN_DEVICE_FUNC inline void pscatter(float* to, const float4& from, Index stride) { to[stride*0] = from.x; to[stride*1] = from.y; to[stride*2] = from.z; to[stride*3] = from.w; } template<> EIGEN_DEVICE_FUNC inline void pscatter(double* to, const double2& from, Index stride) { to[stride*0] = from.x; to[stride*1] = from.y; } template<> EIGEN_DEVICE_FUNC inline float pfirst(const float4& a) { return a.x; } template<> EIGEN_DEVICE_FUNC inline double pfirst(const double2& a) { return a.x; } template<> EIGEN_DEVICE_FUNC inline float predux(const float4& a) { return a.x + a.y + a.z + a.w; } template<> EIGEN_DEVICE_FUNC inline double predux(const double2& a) { return a.x + a.y; } template<> EIGEN_DEVICE_FUNC inline float predux_max(const float4& a) { return fmaxf(fmaxf(a.x, a.y), fmaxf(a.z, a.w)); } template<> EIGEN_DEVICE_FUNC inline double predux_max(const double2& a) { return fmax(a.x, a.y); } template<> EIGEN_DEVICE_FUNC inline float predux_min(const float4& a) { return fminf(fminf(a.x, a.y), fminf(a.z, a.w)); } template<> EIGEN_DEVICE_FUNC inline double predux_min(const double2& a) { return fmin(a.x, a.y); } template<> EIGEN_DEVICE_FUNC inline float predux_mul(const float4& a) { return a.x * a.y * a.z * a.w; } template<> EIGEN_DEVICE_FUNC inline double predux_mul(const double2& a) { return a.x * a.y; } template<> EIGEN_DEVICE_FUNC inline float4 pabs(const float4& a) { return make_float4(fabsf(a.x), fabsf(a.y), fabsf(a.z), fabsf(a.w)); } template<> EIGEN_DEVICE_FUNC inline double2 pabs(const double2& a) { return make_double2(fabs(a.x), fabs(a.y)); } template<> EIGEN_DEVICE_FUNC inline float4 pfloor(const float4& a) { return make_float4(floorf(a.x), floorf(a.y), floorf(a.z), floorf(a.w)); } template<> EIGEN_DEVICE_FUNC inline double2 pfloor(const double2& a) { return make_double2(floor(a.x), floor(a.y)); } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { float tmp = kernel.packet[0].y; kernel.packet[0].y = kernel.packet[1].x; kernel.packet[1].x = tmp; tmp = kernel.packet[0].z; kernel.packet[0].z = kernel.packet[2].x; kernel.packet[2].x = tmp; tmp = kernel.packet[0].w; kernel.packet[0].w = kernel.packet[3].x; kernel.packet[3].x = tmp; tmp = kernel.packet[1].z; kernel.packet[1].z = kernel.packet[2].y; kernel.packet[2].y = tmp; tmp = kernel.packet[1].w; kernel.packet[1].w = kernel.packet[3].y; kernel.packet[3].y = tmp; tmp = kernel.packet[2].w; kernel.packet[2].w = kernel.packet[3].z; kernel.packet[3].z = tmp; } EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { double tmp = kernel.packet[0].y; kernel.packet[0].y = kernel.packet[1].x; kernel.packet[1].x = tmp; } #endif // defined(EIGEN_GPUCC) && defined(EIGEN_USE_GPU) // Packet4h2 must be defined in the macro without EIGEN_CUDA_ARCH, meaning // its corresponding packet_traits must be visible on host. #if defined(EIGEN_HAS_CUDA_FP16) || defined(EIGEN_HAS_HIP_FP16) typedef ulonglong2 Packet4h2; template<> struct unpacket_traits { typedef Eigen::half type; enum {size=8, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet4h2 half; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct unpacket_traits { typedef Eigen::half type; enum {size=2, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef half2 half; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct packet_traits : default_packet_traits { typedef Packet4h2 type; typedef Packet4h2 half; enum { Vectorizable = 1, AlignedOnScalar = 1, size=8, HasHalfPacket = 0, HasAdd = 1, HasSub = 1, HasMul = 1, HasDiv = 1, HasSqrt = 1, HasRsqrt = 1, HasExp = 1, HasExpm1 = 1, HasLog = 1, HasLog1p = 1 }; }; namespace { // This is equivalent to make_half2, which is undocumented and doesn't seem to always exist. EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 combine_half(const __half& a, const __half& b) { #if defined(EIGEN_GPU_COMPILE_PHASE) return __halves2half2(a, b); #else // Round-about way since __halves2half2 is a __device__ function. return __floats2half2_rn(__half2float(a), __half2float(b)); #endif } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE __half get_half2_low(const half2& a) { #if defined(EIGEN_GPU_COMPILE_PHASE) return __low2half(a); #else return __float2half(__low2float(a)); #endif } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE __half get_half2_high(const half2& a) { #if defined(EIGEN_GPU_COMPILE_PHASE) return __high2half(a); #else return __float2half(__high2float(a)); #endif } } // namespace template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pset1(const Eigen::half& from) { #if defined(EIGEN_GPU_COMPILE_PHASE) return __half2half2(from); #else const float f = __half2float(from); return __floats2half2_rn(f, f); #endif } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pset1(const Eigen::half& from) { Packet4h2 r; half2* p_alias = reinterpret_cast(&r); p_alias[0] = pset1(from); p_alias[1] = pset1(from); p_alias[2] = pset1(from); p_alias[3] = pset1(from); return r; } // We now need this visible on both host and device. // #if defined(EIGEN_CUDA_ARCH) || defined(EIGEN_HIPCC) || (defined(EIGEN_CUDACC) && EIGEN_COMP_CLANG && !EIGEN_COMP_NVCC) namespace { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pload(const Eigen::half* from) { return *reinterpret_cast(from); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 ploadu(const Eigen::half* from) { return combine_half(from[0], from[1]); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 ploaddup(const Eigen::half* from) { return combine_half(from[0], from[0]); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstore(Eigen::half* to, const half2& from) { *reinterpret_cast(to) = from; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstoreu(Eigen::half* to, const half2& from) { to[0] = get_half2_low(from); to[1] = get_half2_high(from); } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE half2 ploadt_ro_aligned( const Eigen::half* from) { #if defined(EIGEN_GPU_HAS_LDG) // Input is guaranteed to be properly aligned. return __ldg(reinterpret_cast(from)); #else return combine_half(*(from+0), *(from+1)); #endif } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE half2 ploadt_ro_unaligned( const Eigen::half* from) { #if defined(EIGEN_GPU_HAS_LDG) return __halves2half2(__ldg(from+0), __ldg(from+1)); #else return combine_half(*(from+0), *(from+1)); #endif } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pgather(const Eigen::half* from, Index stride) { return combine_half(from[0*stride], from[1*stride]); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter( Eigen::half* to, const half2& from, Index stride) { to[stride*0] = get_half2_low(from); to[stride*1] = get_half2_high(from); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half pfirst(const half2& a) { return get_half2_low(a); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pabs(const half2& a) { half a1 = get_half2_low(a); half a2 = get_half2_high(a); half result1 = half_impl::raw_uint16_to_half(a1.x & 0x7FFF); half result2 = half_impl::raw_uint16_to_half(a2.x & 0x7FFF); return combine_half(result1, result2); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 ptrue(const half2& /*a*/) { half true_half = half_impl::raw_uint16_to_half(0xffffu); return pset1(true_half); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pzero(const half2& /*a*/) { half false_half = half_impl::raw_uint16_to_half(0x0000u); return pset1(false_half); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { __half a1 = get_half2_low(kernel.packet[0]); __half a2 = get_half2_high(kernel.packet[0]); __half b1 = get_half2_low(kernel.packet[1]); __half b2 = get_half2_high(kernel.packet[1]); kernel.packet[0] = combine_half(a1, b1); kernel.packet[1] = combine_half(a2, b2); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 plset(const Eigen::half& a) { #if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC) return __halves2half2(a, __hadd(a, __float2half(1.0f))); #else float f = __half2float(a) + 1.0f; return combine_half(a, __float2half(f)); #endif } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pselect(const half2& mask, const half2& a, const half2& b) { half mask_low = get_half2_low(mask); half mask_high = get_half2_high(mask); half result_low = mask_low == half(0) ? get_half2_low(b) : get_half2_low(a); half result_high = mask_high == half(0) ? get_half2_high(b) : get_half2_high(a); return combine_half(result_low, result_high); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pcmp_eq(const half2& a, const half2& b) { half true_half = half_impl::raw_uint16_to_half(0xffffu); half false_half = half_impl::raw_uint16_to_half(0x0000u); half a1 = get_half2_low(a); half a2 = get_half2_high(a); half b1 = get_half2_low(b); half b2 = get_half2_high(b); half eq1 = __half2float(a1) == __half2float(b1) ? true_half : false_half; half eq2 = __half2float(a2) == __half2float(b2) ? true_half : false_half; return combine_half(eq1, eq2); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pcmp_lt(const half2& a, const half2& b) { half true_half = half_impl::raw_uint16_to_half(0xffffu); half false_half = half_impl::raw_uint16_to_half(0x0000u); half a1 = get_half2_low(a); half a2 = get_half2_high(a); half b1 = get_half2_low(b); half b2 = get_half2_high(b); half eq1 = __half2float(a1) < __half2float(b1) ? true_half : false_half; half eq2 = __half2float(a2) < __half2float(b2) ? true_half : false_half; return combine_half(eq1, eq2); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pand(const half2& a, const half2& b) { half a1 = get_half2_low(a); half a2 = get_half2_high(a); half b1 = get_half2_low(b); half b2 = get_half2_high(b); half result1 = half_impl::raw_uint16_to_half(a1.x & b1.x); half result2 = half_impl::raw_uint16_to_half(a2.x & b2.x); return combine_half(result1, result2); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 por(const half2& a, const half2& b) { half a1 = get_half2_low(a); half a2 = get_half2_high(a); half b1 = get_half2_low(b); half b2 = get_half2_high(b); half result1 = half_impl::raw_uint16_to_half(a1.x | b1.x); half result2 = half_impl::raw_uint16_to_half(a2.x | b2.x); return combine_half(result1, result2); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pxor(const half2& a, const half2& b) { half a1 = get_half2_low(a); half a2 = get_half2_high(a); half b1 = get_half2_low(b); half b2 = get_half2_high(b); half result1 = half_impl::raw_uint16_to_half(a1.x ^ b1.x); half result2 = half_impl::raw_uint16_to_half(a2.x ^ b2.x); return combine_half(result1, result2); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pandnot(const half2& a, const half2& b) { half a1 = get_half2_low(a); half a2 = get_half2_high(a); half b1 = get_half2_low(b); half b2 = get_half2_high(b); half result1 = half_impl::raw_uint16_to_half(a1.x & ~b1.x); half result2 = half_impl::raw_uint16_to_half(a2.x & ~b2.x); return combine_half(result1, result2); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 padd(const half2& a, const half2& b) { #if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC) return __hadd2(a, b); #else float a1 = __low2float(a); float a2 = __high2float(a); float b1 = __low2float(b); float b2 = __high2float(b); float r1 = a1 + b1; float r2 = a2 + b2; return __floats2half2_rn(r1, r2); #endif } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 psub(const half2& a, const half2& b) { #if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC) return __hsub2(a, b); #else float a1 = __low2float(a); float a2 = __high2float(a); float b1 = __low2float(b); float b2 = __high2float(b); float r1 = a1 - b1; float r2 = a2 - b2; return __floats2half2_rn(r1, r2); #endif } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pnegate(const half2& a) { #if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC) return __hneg2(a); #else float a1 = __low2float(a); float a2 = __high2float(a); return __floats2half2_rn(-a1, -a2); #endif } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pconj(const half2& a) { return a; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmul(const half2& a, const half2& b) { #if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC) return __hmul2(a, b); #else float a1 = __low2float(a); float a2 = __high2float(a); float b1 = __low2float(b); float b2 = __high2float(b); float r1 = a1 * b1; float r2 = a2 * b2; return __floats2half2_rn(r1, r2); #endif } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmadd(const half2& a, const half2& b, const half2& c) { #if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC) return __hfma2(a, b, c); #else float a1 = __low2float(a); float a2 = __high2float(a); float b1 = __low2float(b); float b2 = __high2float(b); float c1 = __low2float(c); float c2 = __high2float(c); float r1 = a1 * b1 + c1; float r2 = a2 * b2 + c2; return __floats2half2_rn(r1, r2); #endif } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pdiv(const half2& a, const half2& b) { #if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC) return __h2div(a, b); #else float a1 = __low2float(a); float a2 = __high2float(a); float b1 = __low2float(b); float b2 = __high2float(b); float r1 = a1 / b1; float r2 = a2 / b2; return __floats2half2_rn(r1, r2); #endif } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmin(const half2& a, const half2& b) { float a1 = __low2float(a); float a2 = __high2float(a); float b1 = __low2float(b); float b2 = __high2float(b); __half r1 = a1 < b1 ? get_half2_low(a) : get_half2_low(b); __half r2 = a2 < b2 ? get_half2_high(a) : get_half2_high(b); return combine_half(r1, r2); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmax(const half2& a, const half2& b) { float a1 = __low2float(a); float a2 = __high2float(a); float b1 = __low2float(b); float b2 = __high2float(b); __half r1 = a1 > b1 ? get_half2_low(a) : get_half2_low(b); __half r2 = a2 > b2 ? get_half2_high(a) : get_half2_high(b); return combine_half(r1, r2); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux(const half2& a) { #if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC) return __hadd(__low2half(a), __high2half(a)); #else float a1 = __low2float(a); float a2 = __high2float(a); return Eigen::half(__float2half(a1 + a2)); #endif } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_max(const half2& a) { #if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC) __half first = __low2half(a); __half second = __high2half(a); return __hgt(first, second) ? first : second; #else float a1 = __low2float(a); float a2 = __high2float(a); return a1 > a2 ? get_half2_low(a) : get_half2_high(a); #endif } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_min(const half2& a) { #if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC) __half first = __low2half(a); __half second = __high2half(a); return __hlt(first, second) ? first : second; #else float a1 = __low2float(a); float a2 = __high2float(a); return a1 < a2 ? get_half2_low(a) : get_half2_high(a); #endif } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_mul(const half2& a) { #if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC) return __hmul(__low2half(a), __high2half(a)); #else float a1 = __low2float(a); float a2 = __high2float(a); return Eigen::half(__float2half(a1 * a2)); #endif } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 plog1p(const half2& a) { float a1 = __low2float(a); float a2 = __high2float(a); float r1 = log1pf(a1); float r2 = log1pf(a2); return __floats2half2_rn(r1, r2); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pexpm1(const half2& a) { float a1 = __low2float(a); float a2 = __high2float(a); float r1 = expm1f(a1); float r2 = expm1f(a2); return __floats2half2_rn(r1, r2); } #if (EIGEN_CUDA_SDK_VER >= 80000 && defined(EIGEN_CUDA_HAS_FP16_ARITHMETIC)) || \ defined(EIGEN_HIP_DEVICE_COMPILE) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 plog(const half2& a) { return h2log(a); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pexp(const half2& a) { return h2exp(a); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 psqrt(const half2& a) { return h2sqrt(a); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 prsqrt(const half2& a) { return h2rsqrt(a); } #else EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 plog(const half2& a) { float a1 = __low2float(a); float a2 = __high2float(a); float r1 = logf(a1); float r2 = logf(a2); return __floats2half2_rn(r1, r2); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pexp(const half2& a) { float a1 = __low2float(a); float a2 = __high2float(a); float r1 = expf(a1); float r2 = expf(a2); return __floats2half2_rn(r1, r2); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 psqrt(const half2& a) { float a1 = __low2float(a); float a2 = __high2float(a); float r1 = sqrtf(a1); float r2 = sqrtf(a2); return __floats2half2_rn(r1, r2); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 prsqrt(const half2& a) { float a1 = __low2float(a); float a2 = __high2float(a); float r1 = rsqrtf(a1); float r2 = rsqrtf(a2); return __floats2half2_rn(r1, r2); } #endif } // namespace template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pload(const Eigen::half* from) { return *reinterpret_cast(from); } // unaligned load; template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 ploadu(const Eigen::half* from) { Packet4h2 r; half2* p_alias = reinterpret_cast(&r); p_alias[0] = ploadu(from + 0); p_alias[1] = ploadu(from + 2); p_alias[2] = ploadu(from + 4); p_alias[3] = ploadu(from + 6); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 ploaddup(const Eigen::half* from) { Packet4h2 r; half2* p_alias = reinterpret_cast(&r); p_alias[0] = ploaddup(from + 0); p_alias[1] = ploaddup(from + 1); p_alias[2] = ploaddup(from + 2); p_alias[3] = ploaddup(from + 3); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstore( Eigen::half* to, const Packet4h2& from) { *reinterpret_cast(to) = from; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstoreu( Eigen::half* to, const Packet4h2& from) { const half2* from_alias = reinterpret_cast(&from); pstoreu(to + 0,from_alias[0]); pstoreu(to + 2,from_alias[1]); pstoreu(to + 4,from_alias[2]); pstoreu(to + 6,from_alias[3]); } template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4h2 ploadt_ro(const Eigen::half* from) { #if defined(EIGEN_GPU_HAS_LDG) Packet4h2 r; r = __ldg(reinterpret_cast(from)); return r; #else Packet4h2 r; half2* r_alias = reinterpret_cast(&r); r_alias[0] = ploadt_ro_aligned(from + 0); r_alias[1] = ploadt_ro_aligned(from + 2); r_alias[2] = ploadt_ro_aligned(from + 4); r_alias[3] = ploadt_ro_aligned(from + 6); return r; #endif } template <> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4h2 ploadt_ro(const Eigen::half* from) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); r_alias[0] = ploadt_ro_unaligned(from + 0); r_alias[1] = ploadt_ro_unaligned(from + 2); r_alias[2] = ploadt_ro_unaligned(from + 4); r_alias[3] = ploadt_ro_unaligned(from + 6); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pgather(const Eigen::half* from, Index stride) { Packet4h2 r; half2* p_alias = reinterpret_cast(&r); p_alias[0] = combine_half(from[0 * stride], from[1 * stride]); p_alias[1] = combine_half(from[2 * stride], from[3 * stride]); p_alias[2] = combine_half(from[4 * stride], from[5 * stride]); p_alias[3] = combine_half(from[6 * stride], from[7 * stride]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter( Eigen::half* to, const Packet4h2& from, Index stride) { const half2* from_alias = reinterpret_cast(&from); pscatter(to + stride * 0, from_alias[0], stride); pscatter(to + stride * 2, from_alias[1], stride); pscatter(to + stride * 4, from_alias[2], stride); pscatter(to + stride * 6, from_alias[3], stride); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half pfirst( const Packet4h2& a) { return pfirst(*(reinterpret_cast(&a))); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pabs( const Packet4h2& a) { Packet4h2 r; half2* p_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); p_alias[0] = pabs(a_alias[0]); p_alias[1] = pabs(a_alias[1]); p_alias[2] = pabs(a_alias[2]); p_alias[3] = pabs(a_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 ptrue( const Packet4h2& /*a*/) { half true_half = half_impl::raw_uint16_to_half(0xffffu); return pset1(true_half); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pzero(const Packet4h2& /*a*/) { half false_half = half_impl::raw_uint16_to_half(0x0000u); return pset1(false_half); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose_double( double* d_row0, double* d_row1, double* d_row2, double* d_row3, double* d_row4, double* d_row5, double* d_row6, double* d_row7) { double d_tmp; d_tmp = d_row0[1]; d_row0[1] = d_row4[0]; d_row4[0] = d_tmp; d_tmp = d_row1[1]; d_row1[1] = d_row5[0]; d_row5[0] = d_tmp; d_tmp = d_row2[1]; d_row2[1] = d_row6[0]; d_row6[0] = d_tmp; d_tmp = d_row3[1]; d_row3[1] = d_row7[0]; d_row7[0] = d_tmp; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose_half2( half2* f_row0, half2* f_row1, half2* f_row2, half2* f_row3) { half2 f_tmp; f_tmp = f_row0[1]; f_row0[1] = f_row2[0]; f_row2[0] = f_tmp; f_tmp = f_row1[1]; f_row1[1] = f_row3[0]; f_row3[0] = f_tmp; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose_half(half2& f0, half2& f1) { __half a1 = get_half2_low(f0); __half a2 = get_half2_high(f0); __half b1 = get_half2_low(f1); __half b2 = get_half2_high(f1); f0 = combine_half(a1, b1); f1 = combine_half(a2, b2); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { double* d_row0 = reinterpret_cast(&kernel.packet[0]); double* d_row1 = reinterpret_cast(&kernel.packet[1]); double* d_row2 = reinterpret_cast(&kernel.packet[2]); double* d_row3 = reinterpret_cast(&kernel.packet[3]); double* d_row4 = reinterpret_cast(&kernel.packet[4]); double* d_row5 = reinterpret_cast(&kernel.packet[5]); double* d_row6 = reinterpret_cast(&kernel.packet[6]); double* d_row7 = reinterpret_cast(&kernel.packet[7]); ptranspose_double(d_row0, d_row1, d_row2, d_row3, d_row4, d_row5, d_row6, d_row7); half2* f_row0 = reinterpret_cast(d_row0); half2* f_row1 = reinterpret_cast(d_row1); half2* f_row2 = reinterpret_cast(d_row2); half2* f_row3 = reinterpret_cast(d_row3); ptranspose_half2(f_row0, f_row1, f_row2, f_row3); ptranspose_half(f_row0[0], f_row1[0]); ptranspose_half(f_row0[1], f_row1[1]); ptranspose_half(f_row2[0], f_row3[0]); ptranspose_half(f_row2[1], f_row3[1]); f_row0 = reinterpret_cast(d_row0 + 1); f_row1 = reinterpret_cast(d_row1 + 1); f_row2 = reinterpret_cast(d_row2 + 1); f_row3 = reinterpret_cast(d_row3 + 1); ptranspose_half2(f_row0, f_row1, f_row2, f_row3); ptranspose_half(f_row0[0], f_row1[0]); ptranspose_half(f_row0[1], f_row1[1]); ptranspose_half(f_row2[0], f_row3[0]); ptranspose_half(f_row2[1], f_row3[1]); f_row0 = reinterpret_cast(d_row4); f_row1 = reinterpret_cast(d_row5); f_row2 = reinterpret_cast(d_row6); f_row3 = reinterpret_cast(d_row7); ptranspose_half2(f_row0, f_row1, f_row2, f_row3); ptranspose_half(f_row0[0], f_row1[0]); ptranspose_half(f_row0[1], f_row1[1]); ptranspose_half(f_row2[0], f_row3[0]); ptranspose_half(f_row2[1], f_row3[1]); f_row0 = reinterpret_cast(d_row4 + 1); f_row1 = reinterpret_cast(d_row5 + 1); f_row2 = reinterpret_cast(d_row6 + 1); f_row3 = reinterpret_cast(d_row7 + 1); ptranspose_half2(f_row0, f_row1, f_row2, f_row3); ptranspose_half(f_row0[0], f_row1[0]); ptranspose_half(f_row0[1], f_row1[1]); ptranspose_half(f_row2[0], f_row3[0]); ptranspose_half(f_row2[1], f_row3[1]); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 plset(const Eigen::half& a) { #if defined(EIGEN_HIP_DEVICE_COMPILE) Packet4h2 r; half2* p_alias = reinterpret_cast(&r); p_alias[0] = __halves2half2(a, __hadd(a, __float2half(1.0f))); p_alias[1] = __halves2half2(__hadd(a, __float2half(2.0f)), __hadd(a, __float2half(3.0f))); p_alias[2] = __halves2half2(__hadd(a, __float2half(4.0f)), __hadd(a, __float2half(5.0f))); p_alias[3] = __halves2half2(__hadd(a, __float2half(6.0f)), __hadd(a, __float2half(7.0f))); return r; #elif defined(EIGEN_CUDA_HAS_FP16_ARITHMETIC) Packet4h2 r; half2* r_alias = reinterpret_cast(&r); half2 b = pset1(a); half2 c; half2 half_offset0 = __halves2half2(__float2half(0.0f),__float2half(2.0f)); half2 half_offset1 = __halves2half2(__float2half(4.0f),__float2half(6.0f)); c = __hadd2(b, half_offset0); r_alias[0] = plset(__low2half(c)); r_alias[1] = plset(__high2half(c)); c = __hadd2(b, half_offset1); r_alias[2] = plset(__low2half(c)); r_alias[3] = plset(__high2half(c)); return r; #else float f = __half2float(a); Packet4h2 r; half2* p_alias = reinterpret_cast(&r); p_alias[0] = combine_half(a, __float2half(f + 1.0f)); p_alias[1] = combine_half(__float2half(f + 2.0f), __float2half(f + 3.0f)); p_alias[2] = combine_half(__float2half(f + 4.0f), __float2half(f + 5.0f)); p_alias[3] = combine_half(__float2half(f + 6.0f), __float2half(f + 7.0f)); return r; #endif } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pselect(const Packet4h2& mask, const Packet4h2& a, const Packet4h2& b) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* mask_alias = reinterpret_cast(&mask); const half2* a_alias = reinterpret_cast(&a); const half2* b_alias = reinterpret_cast(&b); r_alias[0] = pselect(mask_alias[0], a_alias[0], b_alias[0]); r_alias[1] = pselect(mask_alias[1], a_alias[1], b_alias[1]); r_alias[2] = pselect(mask_alias[2], a_alias[2], b_alias[2]); r_alias[3] = pselect(mask_alias[3], a_alias[3], b_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pcmp_eq(const Packet4h2& a, const Packet4h2& b) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); const half2* b_alias = reinterpret_cast(&b); r_alias[0] = pcmp_eq(a_alias[0], b_alias[0]); r_alias[1] = pcmp_eq(a_alias[1], b_alias[1]); r_alias[2] = pcmp_eq(a_alias[2], b_alias[2]); r_alias[3] = pcmp_eq(a_alias[3], b_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pand( const Packet4h2& a, const Packet4h2& b) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); const half2* b_alias = reinterpret_cast(&b); r_alias[0] = pand(a_alias[0], b_alias[0]); r_alias[1] = pand(a_alias[1], b_alias[1]); r_alias[2] = pand(a_alias[2], b_alias[2]); r_alias[3] = pand(a_alias[3], b_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 por( const Packet4h2& a, const Packet4h2& b) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); const half2* b_alias = reinterpret_cast(&b); r_alias[0] = por(a_alias[0], b_alias[0]); r_alias[1] = por(a_alias[1], b_alias[1]); r_alias[2] = por(a_alias[2], b_alias[2]); r_alias[3] = por(a_alias[3], b_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pxor( const Packet4h2& a, const Packet4h2& b) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); const half2* b_alias = reinterpret_cast(&b); r_alias[0] = pxor(a_alias[0], b_alias[0]); r_alias[1] = pxor(a_alias[1], b_alias[1]); r_alias[2] = pxor(a_alias[2], b_alias[2]); r_alias[3] = pxor(a_alias[3], b_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pandnot(const Packet4h2& a, const Packet4h2& b) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); const half2* b_alias = reinterpret_cast(&b); r_alias[0] = pandnot(a_alias[0], b_alias[0]); r_alias[1] = pandnot(a_alias[1], b_alias[1]); r_alias[2] = pandnot(a_alias[2], b_alias[2]); r_alias[3] = pandnot(a_alias[3], b_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 padd( const Packet4h2& a, const Packet4h2& b) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); const half2* b_alias = reinterpret_cast(&b); r_alias[0] = padd(a_alias[0], b_alias[0]); r_alias[1] = padd(a_alias[1], b_alias[1]); r_alias[2] = padd(a_alias[2], b_alias[2]); r_alias[3] = padd(a_alias[3], b_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 psub( const Packet4h2& a, const Packet4h2& b) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); const half2* b_alias = reinterpret_cast(&b); r_alias[0] = psub(a_alias[0], b_alias[0]); r_alias[1] = psub(a_alias[1], b_alias[1]); r_alias[2] = psub(a_alias[2], b_alias[2]); r_alias[3] = psub(a_alias[3], b_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pnegate(const Packet4h2& a) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); r_alias[0] = pnegate(a_alias[0]); r_alias[1] = pnegate(a_alias[1]); r_alias[2] = pnegate(a_alias[2]); r_alias[3] = pnegate(a_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pconj(const Packet4h2& a) { return a; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pmul( const Packet4h2& a, const Packet4h2& b) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); const half2* b_alias = reinterpret_cast(&b); r_alias[0] = pmul(a_alias[0], b_alias[0]); r_alias[1] = pmul(a_alias[1], b_alias[1]); r_alias[2] = pmul(a_alias[2], b_alias[2]); r_alias[3] = pmul(a_alias[3], b_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pmadd( const Packet4h2& a, const Packet4h2& b, const Packet4h2& c) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); const half2* b_alias = reinterpret_cast(&b); const half2* c_alias = reinterpret_cast(&c); r_alias[0] = pmadd(a_alias[0], b_alias[0], c_alias[0]); r_alias[1] = pmadd(a_alias[1], b_alias[1], c_alias[1]); r_alias[2] = pmadd(a_alias[2], b_alias[2], c_alias[2]); r_alias[3] = pmadd(a_alias[3], b_alias[3], c_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pdiv( const Packet4h2& a, const Packet4h2& b) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); const half2* b_alias = reinterpret_cast(&b); r_alias[0] = pdiv(a_alias[0], b_alias[0]); r_alias[1] = pdiv(a_alias[1], b_alias[1]); r_alias[2] = pdiv(a_alias[2], b_alias[2]); r_alias[3] = pdiv(a_alias[3], b_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pmin( const Packet4h2& a, const Packet4h2& b) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); const half2* b_alias = reinterpret_cast(&b); r_alias[0] = pmin(a_alias[0], b_alias[0]); r_alias[1] = pmin(a_alias[1], b_alias[1]); r_alias[2] = pmin(a_alias[2], b_alias[2]); r_alias[3] = pmin(a_alias[3], b_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pmax( const Packet4h2& a, const Packet4h2& b) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); const half2* b_alias = reinterpret_cast(&b); r_alias[0] = pmax(a_alias[0], b_alias[0]); r_alias[1] = pmax(a_alias[1], b_alias[1]); r_alias[2] = pmax(a_alias[2], b_alias[2]); r_alias[3] = pmax(a_alias[3], b_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux( const Packet4h2& a) { const half2* a_alias = reinterpret_cast(&a); return predux(a_alias[0]) + predux(a_alias[1]) + predux(a_alias[2]) + predux(a_alias[3]); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_max( const Packet4h2& a) { const half2* a_alias = reinterpret_cast(&a); half2 m0 = combine_half(predux_max(a_alias[0]), predux_max(a_alias[1])); half2 m1 = combine_half(predux_max(a_alias[2]), predux_max(a_alias[3])); __half first = predux_max(m0); __half second = predux_max(m1); #if defined(EIGEN_CUDA_HAS_FP16_ARITHMETIC) return (__hgt(first, second) ? first : second); #else float ffirst = __half2float(first); float fsecond = __half2float(second); return (ffirst > fsecond)? first: second; #endif } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_min( const Packet4h2& a) { const half2* a_alias = reinterpret_cast(&a); half2 m0 = combine_half(predux_min(a_alias[0]), predux_min(a_alias[1])); half2 m1 = combine_half(predux_min(a_alias[2]), predux_min(a_alias[3])); __half first = predux_min(m0); __half second = predux_min(m1); #if defined(EIGEN_CUDA_HAS_FP16_ARITHMETIC) return (__hlt(first, second) ? first : second); #else float ffirst = __half2float(first); float fsecond = __half2float(second); return (ffirst < fsecond)? first: second; #endif } // likely overflow/underflow template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_mul( const Packet4h2& a) { const half2* a_alias = reinterpret_cast(&a); return predux_mul(pmul(pmul(a_alias[0], a_alias[1]), pmul(a_alias[2], a_alias[3]))); } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 plog1p(const Packet4h2& a) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); r_alias[0] = plog1p(a_alias[0]); r_alias[1] = plog1p(a_alias[1]); r_alias[2] = plog1p(a_alias[2]); r_alias[3] = plog1p(a_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pexpm1(const Packet4h2& a) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); r_alias[0] = pexpm1(a_alias[0]); r_alias[1] = pexpm1(a_alias[1]); r_alias[2] = pexpm1(a_alias[2]); r_alias[3] = pexpm1(a_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 plog(const Packet4h2& a) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); r_alias[0] = plog(a_alias[0]); r_alias[1] = plog(a_alias[1]); r_alias[2] = plog(a_alias[2]); r_alias[3] = plog(a_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pexp(const Packet4h2& a) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); r_alias[0] = pexp(a_alias[0]); r_alias[1] = pexp(a_alias[1]); r_alias[2] = pexp(a_alias[2]); r_alias[3] = pexp(a_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 psqrt(const Packet4h2& a) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); r_alias[0] = psqrt(a_alias[0]); r_alias[1] = psqrt(a_alias[1]); r_alias[2] = psqrt(a_alias[2]); r_alias[3] = psqrt(a_alias[3]); return r; } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 prsqrt(const Packet4h2& a) { Packet4h2 r; half2* r_alias = reinterpret_cast(&r); const half2* a_alias = reinterpret_cast(&a); r_alias[0] = prsqrt(a_alias[0]); r_alias[1] = prsqrt(a_alias[1]); r_alias[2] = prsqrt(a_alias[2]); r_alias[3] = prsqrt(a_alias[3]); return r; } // The following specialized padd, pmul, pdiv, pmin, pmax, pset1 are needed for // the implementation of GPU half reduction. template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 padd(const half2& a, const half2& b) { #if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC) return __hadd2(a, b); #else float a1 = __low2float(a); float a2 = __high2float(a); float b1 = __low2float(b); float b2 = __high2float(b); float r1 = a1 + b1; float r2 = a2 + b2; return __floats2half2_rn(r1, r2); #endif } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmul(const half2& a, const half2& b) { #if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC) return __hmul2(a, b); #else float a1 = __low2float(a); float a2 = __high2float(a); float b1 = __low2float(b); float b2 = __high2float(b); float r1 = a1 * b1; float r2 = a2 * b2; return __floats2half2_rn(r1, r2); #endif } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pdiv(const half2& a, const half2& b) { #if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC) return __h2div(a, b); #else float a1 = __low2float(a); float a2 = __high2float(a); float b1 = __low2float(b); float b2 = __high2float(b); float r1 = a1 / b1; float r2 = a2 / b2; return __floats2half2_rn(r1, r2); #endif } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmin(const half2& a, const half2& b) { float a1 = __low2float(a); float a2 = __high2float(a); float b1 = __low2float(b); float b2 = __high2float(b); __half r1 = a1 < b1 ? get_half2_low(a) : get_half2_low(b); __half r2 = a2 < b2 ? get_half2_high(a) : get_half2_high(b); return combine_half(r1, r2); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmax(const half2& a, const half2& b) { float a1 = __low2float(a); float a2 = __high2float(a); float b1 = __low2float(b); float b2 = __high2float(b); __half r1 = a1 > b1 ? get_half2_low(a) : get_half2_low(b); __half r2 = a2 > b2 ? get_half2_high(a) : get_half2_high(b); return combine_half(r1, r2); } // #endif // defined(EIGEN_CUDA_ARCH) || defined(EIGEN_HIPCC) || (defined(EIGEN_CUDACC) && EIGEN_COMP_CLANG && !EIGEN_COMP_NVCC) #endif // defined(EIGEN_HAS_CUDA_FP16) || defined(EIGEN_HAS_HIP_FP16) #undef EIGEN_GPU_HAS_LDG #undef EIGEN_CUDA_HAS_FP16_ARITHMETIC #undef EIGEN_GPU_HAS_FP16_ARITHMETIC } // end namespace internal } // end namespace Eigen #endif // EIGEN_PACKET_MATH_GPU_H RcppEigen/inst/include/Eigen/src/Core/arch/GPU/MathFunctions.h0000644000176200001440000000520714567757725023574 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2014 Benoit Steiner // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_MATH_FUNCTIONS_GPU_H #define EIGEN_MATH_FUNCTIONS_GPU_H namespace Eigen { namespace internal { // Make sure this is only available when targeting a GPU: we don't want to // introduce conflicts between these packet_traits definitions and the ones // we'll use on the host side (SSE, AVX, ...) #if defined(EIGEN_GPUCC) && defined(EIGEN_USE_GPU) template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 plog(const float4& a) { return make_float4(logf(a.x), logf(a.y), logf(a.z), logf(a.w)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 plog(const double2& a) { using ::log; return make_double2(log(a.x), log(a.y)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 plog1p(const float4& a) { return make_float4(log1pf(a.x), log1pf(a.y), log1pf(a.z), log1pf(a.w)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 plog1p(const double2& a) { return make_double2(log1p(a.x), log1p(a.y)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pexp(const float4& a) { return make_float4(expf(a.x), expf(a.y), expf(a.z), expf(a.w)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pexp(const double2& a) { using ::exp; return make_double2(exp(a.x), exp(a.y)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pexpm1(const float4& a) { return make_float4(expm1f(a.x), expm1f(a.y), expm1f(a.z), expm1f(a.w)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pexpm1(const double2& a) { return make_double2(expm1(a.x), expm1(a.y)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 psqrt(const float4& a) { return make_float4(sqrtf(a.x), sqrtf(a.y), sqrtf(a.z), sqrtf(a.w)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 psqrt(const double2& a) { using ::sqrt; return make_double2(sqrt(a.x), sqrt(a.y)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 prsqrt(const float4& a) { return make_float4(rsqrtf(a.x), rsqrtf(a.y), rsqrtf(a.z), rsqrtf(a.w)); } template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 prsqrt(const double2& a) { return make_double2(rsqrt(a.x), rsqrt(a.y)); } #endif } // end namespace internal } // end namespace Eigen #endif // EIGEN_MATH_FUNCTIONS_GPU_H RcppEigen/inst/include/Eigen/src/Core/arch/CUDA/0000755000176200001440000000000014567757725020716 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/arch/CUDA/Complex.h0000644000176200001440000004164514567757725022510 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2014 Benoit Steiner // Copyright (C) 2021 C. Antonio Sanchez // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_COMPLEX_CUDA_H #define EIGEN_COMPLEX_CUDA_H // clang-format off // Many std::complex methods such as operator+, operator-, operator* and // operator/ are not constexpr. Due to this, GCC and older versions of clang do // not treat them as device functions and thus Eigen functors making use of // these operators fail to compile. Here, we manually specialize these // operators and functors for complex types when building for CUDA to enable // their use on-device. #if defined(EIGEN_CUDACC) && defined(EIGEN_GPU_COMPILE_PHASE) // ICC already specializes std::complex and std::complex // operators, preventing us from making them device functions here. // This will lead to silent runtime errors if the operators are used on device. // // To allow std::complex operator use on device, define _OVERRIDE_COMPLEX_SPECIALIZATION_ // prior to first inclusion of . This prevents ICC from adding // its own specializations, so our custom ones below can be used instead. #if !(defined(EIGEN_COMP_ICC) && defined(_USE_COMPLEX_SPECIALIZATION_)) // Import Eigen's internal operator specializations. #define EIGEN_USING_STD_COMPLEX_OPERATORS \ using Eigen::complex_operator_detail::operator+; \ using Eigen::complex_operator_detail::operator-; \ using Eigen::complex_operator_detail::operator*; \ using Eigen::complex_operator_detail::operator/; \ using Eigen::complex_operator_detail::operator+=; \ using Eigen::complex_operator_detail::operator-=; \ using Eigen::complex_operator_detail::operator*=; \ using Eigen::complex_operator_detail::operator/=; \ using Eigen::complex_operator_detail::operator==; \ using Eigen::complex_operator_detail::operator!=; namespace Eigen { // Specialized std::complex overloads. namespace complex_operator_detail { template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex complex_multiply(const std::complex& a, const std::complex& b) { const T a_real = numext::real(a); const T a_imag = numext::imag(a); const T b_real = numext::real(b); const T b_imag = numext::imag(b); return std::complex( a_real * b_real - a_imag * b_imag, a_imag * b_real + a_real * b_imag); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex complex_divide_fast(const std::complex& a, const std::complex& b) { const T a_real = numext::real(a); const T a_imag = numext::imag(a); const T b_real = numext::real(b); const T b_imag = numext::imag(b); const T norm = (b_real * b_real + b_imag * b_imag); return std::complex((a_real * b_real + a_imag * b_imag) / norm, (a_imag * b_real - a_real * b_imag) / norm); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex complex_divide_stable(const std::complex& a, const std::complex& b) { const T a_real = numext::real(a); const T a_imag = numext::imag(a); const T b_real = numext::real(b); const T b_imag = numext::imag(b); // Smith's complex division (https://arxiv.org/pdf/1210.4539.pdf), // guards against over/under-flow. const bool scale_imag = numext::abs(b_imag) <= numext::abs(b_real); const T rscale = scale_imag ? T(1) : b_real / b_imag; const T iscale = scale_imag ? b_imag / b_real : T(1); const T denominator = b_real * rscale + b_imag * iscale; return std::complex((a_real * rscale + a_imag * iscale) / denominator, (a_imag * rscale - a_real * iscale) / denominator); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex complex_divide(const std::complex& a, const std::complex& b) { #if EIGEN_FAST_MATH return complex_divide_fast(a, b); #else return complex_divide_stable(a, b); #endif } // NOTE: We cannot specialize compound assignment operators with Scalar T, // (i.e. operator@=(const T&), for @=+,-,*,/) // since they are already specialized for float/double/long double within // the standard header. We also do not specialize the stream // operators. #define EIGEN_CREATE_STD_COMPLEX_OPERATOR_SPECIALIZATIONS(T) \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex operator+(const std::complex& a) { return a; } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex operator-(const std::complex& a) { \ return std::complex(-numext::real(a), -numext::imag(a)); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex operator+(const std::complex& a, const std::complex& b) { \ return std::complex(numext::real(a) + numext::real(b), numext::imag(a) + numext::imag(b)); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex operator+(const std::complex& a, const T& b) { \ return std::complex(numext::real(a) + b, numext::imag(a)); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex operator+(const T& a, const std::complex& b) { \ return std::complex(a + numext::real(b), numext::imag(b)); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex operator-(const std::complex& a, const std::complex& b) { \ return std::complex(numext::real(a) - numext::real(b), numext::imag(a) - numext::imag(b)); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex operator-(const std::complex& a, const T& b) { \ return std::complex(numext::real(a) - b, numext::imag(a)); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex operator-(const T& a, const std::complex& b) { \ return std::complex(a - numext::real(b), -numext::imag(b)); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex operator*(const std::complex& a, const std::complex& b) { \ return complex_multiply(a, b); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex operator*(const std::complex& a, const T& b) { \ return std::complex(numext::real(a) * b, numext::imag(a) * b); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex operator*(const T& a, const std::complex& b) { \ return std::complex(a * numext::real(b), a * numext::imag(b)); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex operator/(const std::complex& a, const std::complex& b) { \ return complex_divide(a, b); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex operator/(const std::complex& a, const T& b) { \ return std::complex(numext::real(a) / b, numext::imag(a) / b); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex operator/(const T& a, const std::complex& b) { \ return complex_divide(std::complex(a, 0), b); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex& operator+=(std::complex& a, const std::complex& b) { \ numext::real_ref(a) += numext::real(b); \ numext::imag_ref(a) += numext::imag(b); \ return a; \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex& operator-=(std::complex& a, const std::complex& b) { \ numext::real_ref(a) -= numext::real(b); \ numext::imag_ref(a) -= numext::imag(b); \ return a; \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex& operator*=(std::complex& a, const std::complex& b) { \ a = complex_multiply(a, b); \ return a; \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ std::complex& operator/=(std::complex& a, const std::complex& b) { \ a = complex_divide(a, b); \ return a; \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ bool operator==(const std::complex& a, const std::complex& b) { \ return numext::real(a) == numext::real(b) && numext::imag(a) == numext::imag(b); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ bool operator==(const std::complex& a, const T& b) { \ return numext::real(a) == b && numext::imag(a) == 0; \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ bool operator==(const T& a, const std::complex& b) { \ return a == numext::real(b) && 0 == numext::imag(b); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ bool operator!=(const std::complex& a, const std::complex& b) { \ return !(a == b); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ bool operator!=(const std::complex& a, const T& b) { \ return !(a == b); \ } \ \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ bool operator!=(const T& a, const std::complex& b) { \ return !(a == b); \ } // Do not specialize for long double, since that reduces to double on device. EIGEN_CREATE_STD_COMPLEX_OPERATOR_SPECIALIZATIONS(float) EIGEN_CREATE_STD_COMPLEX_OPERATOR_SPECIALIZATIONS(double) #undef EIGEN_CREATE_STD_COMPLEX_OPERATOR_SPECIALIZATIONS } // namespace complex_operator_detail EIGEN_USING_STD_COMPLEX_OPERATORS namespace numext { EIGEN_USING_STD_COMPLEX_OPERATORS } // namespace numext namespace internal { EIGEN_USING_STD_COMPLEX_OPERATORS } // namespace internal } // namespace Eigen #endif // !(EIGEN_COMP_ICC && _USE_COMPLEX_SPECIALIZATION_) #endif // EIGEN_CUDACC && EIGEN_GPU_COMPILE_PHASE #endif // EIGEN_COMPLEX_CUDA_H RcppEigen/inst/include/Eigen/src/Core/Random.h0000644000176200001440000001711414567757725020662 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_RANDOM_H #define EIGEN_RANDOM_H namespace Eigen { namespace internal { template struct scalar_random_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_random_op) inline const Scalar operator() () const { return random(); } }; template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = false, IsRepeatable = false }; }; } // end namespace internal /** \returns a random matrix expression * * Numbers are uniformly spread through their whole definition range for integer types, * and in the [-1:1] range for floating point scalar types. * * The parameters \a rows and \a cols are the number of rows and of columns of * the returned matrix. Must be compatible with this MatrixBase type. * * \not_reentrant * * This variant is meant to be used for dynamic-size matrix types. For fixed-size types, * it is redundant to pass \a rows and \a cols as arguments, so Random() should be used * instead. * * * Example: \include MatrixBase_random_int_int.cpp * Output: \verbinclude MatrixBase_random_int_int.out * * This expression has the "evaluate before nesting" flag so that it will be evaluated into * a temporary matrix whenever it is nested in a larger expression. This prevents unexpected * behavior with expressions involving random matrices. * * See DenseBase::NullaryExpr(Index, const CustomNullaryOp&) for an example using C++11 random generators. * * \sa DenseBase::setRandom(), DenseBase::Random(Index), DenseBase::Random() */ template inline const typename DenseBase::RandomReturnType DenseBase::Random(Index rows, Index cols) { return NullaryExpr(rows, cols, internal::scalar_random_op()); } /** \returns a random vector expression * * Numbers are uniformly spread through their whole definition range for integer types, * and in the [-1:1] range for floating point scalar types. * * The parameter \a size is the size of the returned vector. * Must be compatible with this MatrixBase type. * * \only_for_vectors * \not_reentrant * * This variant is meant to be used for dynamic-size vector types. For fixed-size types, * it is redundant to pass \a size as argument, so Random() should be used * instead. * * Example: \include MatrixBase_random_int.cpp * Output: \verbinclude MatrixBase_random_int.out * * This expression has the "evaluate before nesting" flag so that it will be evaluated into * a temporary vector whenever it is nested in a larger expression. This prevents unexpected * behavior with expressions involving random matrices. * * \sa DenseBase::setRandom(), DenseBase::Random(Index,Index), DenseBase::Random() */ template inline const typename DenseBase::RandomReturnType DenseBase::Random(Index size) { return NullaryExpr(size, internal::scalar_random_op()); } /** \returns a fixed-size random matrix or vector expression * * Numbers are uniformly spread through their whole definition range for integer types, * and in the [-1:1] range for floating point scalar types. * * This variant is only for fixed-size MatrixBase types. For dynamic-size types, you * need to use the variants taking size arguments. * * Example: \include MatrixBase_random.cpp * Output: \verbinclude MatrixBase_random.out * * This expression has the "evaluate before nesting" flag so that it will be evaluated into * a temporary matrix whenever it is nested in a larger expression. This prevents unexpected * behavior with expressions involving random matrices. * * \not_reentrant * * \sa DenseBase::setRandom(), DenseBase::Random(Index,Index), DenseBase::Random(Index) */ template inline const typename DenseBase::RandomReturnType DenseBase::Random() { return NullaryExpr(RowsAtCompileTime, ColsAtCompileTime, internal::scalar_random_op()); } /** Sets all coefficients in this expression to random values. * * Numbers are uniformly spread through their whole definition range for integer types, * and in the [-1:1] range for floating point scalar types. * * \not_reentrant * * Example: \include MatrixBase_setRandom.cpp * Output: \verbinclude MatrixBase_setRandom.out * * \sa class CwiseNullaryOp, setRandom(Index), setRandom(Index,Index) */ template EIGEN_DEVICE_FUNC inline Derived& DenseBase::setRandom() { return *this = Random(rows(), cols()); } /** Resizes to the given \a newSize, and sets all coefficients in this expression to random values. * * Numbers are uniformly spread through their whole definition range for integer types, * and in the [-1:1] range for floating point scalar types. * * \only_for_vectors * \not_reentrant * * Example: \include Matrix_setRandom_int.cpp * Output: \verbinclude Matrix_setRandom_int.out * * \sa DenseBase::setRandom(), setRandom(Index,Index), class CwiseNullaryOp, DenseBase::Random() */ template EIGEN_STRONG_INLINE Derived& PlainObjectBase::setRandom(Index newSize) { resize(newSize); return setRandom(); } /** Resizes to the given size, and sets all coefficients in this expression to random values. * * Numbers are uniformly spread through their whole definition range for integer types, * and in the [-1:1] range for floating point scalar types. * * \not_reentrant * * \param rows the new number of rows * \param cols the new number of columns * * Example: \include Matrix_setRandom_int_int.cpp * Output: \verbinclude Matrix_setRandom_int_int.out * * \sa DenseBase::setRandom(), setRandom(Index), class CwiseNullaryOp, DenseBase::Random() */ template EIGEN_STRONG_INLINE Derived& PlainObjectBase::setRandom(Index rows, Index cols) { resize(rows, cols); return setRandom(); } /** Resizes to the given size, changing only the number of columns, and sets all * coefficients in this expression to random values. For the parameter of type * NoChange_t, just pass the special value \c NoChange. * * Numbers are uniformly spread through their whole definition range for integer types, * and in the [-1:1] range for floating point scalar types. * * \not_reentrant * * \sa DenseBase::setRandom(), setRandom(Index), setRandom(Index, NoChange_t), class CwiseNullaryOp, DenseBase::Random() */ template EIGEN_STRONG_INLINE Derived& PlainObjectBase::setRandom(NoChange_t, Index cols) { return setRandom(rows(), cols); } /** Resizes to the given size, changing only the number of rows, and sets all * coefficients in this expression to random values. For the parameter of type * NoChange_t, just pass the special value \c NoChange. * * Numbers are uniformly spread through their whole definition range for integer types, * and in the [-1:1] range for floating point scalar types. * * \not_reentrant * * \sa DenseBase::setRandom(), setRandom(Index), setRandom(NoChange_t, Index), class CwiseNullaryOp, DenseBase::Random() */ template EIGEN_STRONG_INLINE Derived& PlainObjectBase::setRandom(Index rows, NoChange_t) { return setRandom(rows, cols()); } } // end namespace Eigen #endif // EIGEN_RANDOM_H RcppEigen/inst/include/Eigen/src/Core/MathFunctionsImpl.h0000644000176200001440000001576414567757725023057 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com) // Copyright (C) 2016 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_MATHFUNCTIONSIMPL_H #define EIGEN_MATHFUNCTIONSIMPL_H namespace Eigen { namespace internal { /** \internal \returns the hyperbolic tan of \a a (coeff-wise) Doesn't do anything fancy, just a 13/6-degree rational interpolant which is accurate up to a couple of ulps in the (approximate) range [-8, 8], outside of which tanh(x) = +/-1 in single precision. The input is clamped to the range [-c, c]. The value c is chosen as the smallest value where the approximation evaluates to exactly 1. In the reange [-0.0004, 0.0004] the approxmation tanh(x) ~= x is used for better accuracy as x tends to zero. This implementation works on both scalars and packets. */ template T generic_fast_tanh_float(const T& a_x) { // Clamp the inputs to the range [-c, c] #ifdef EIGEN_VECTORIZE_FMA const T plus_clamp = pset1(7.99881172180175781f); const T minus_clamp = pset1(-7.99881172180175781f); #else const T plus_clamp = pset1(7.90531110763549805f); const T minus_clamp = pset1(-7.90531110763549805f); #endif const T tiny = pset1(0.0004f); const T x = pmax(pmin(a_x, plus_clamp), minus_clamp); const T tiny_mask = pcmp_lt(pabs(a_x), tiny); // The monomial coefficients of the numerator polynomial (odd). const T alpha_1 = pset1(4.89352455891786e-03f); const T alpha_3 = pset1(6.37261928875436e-04f); const T alpha_5 = pset1(1.48572235717979e-05f); const T alpha_7 = pset1(5.12229709037114e-08f); const T alpha_9 = pset1(-8.60467152213735e-11f); const T alpha_11 = pset1(2.00018790482477e-13f); const T alpha_13 = pset1(-2.76076847742355e-16f); // The monomial coefficients of the denominator polynomial (even). const T beta_0 = pset1(4.89352518554385e-03f); const T beta_2 = pset1(2.26843463243900e-03f); const T beta_4 = pset1(1.18534705686654e-04f); const T beta_6 = pset1(1.19825839466702e-06f); // Since the polynomials are odd/even, we need x^2. const T x2 = pmul(x, x); // Evaluate the numerator polynomial p. T p = pmadd(x2, alpha_13, alpha_11); p = pmadd(x2, p, alpha_9); p = pmadd(x2, p, alpha_7); p = pmadd(x2, p, alpha_5); p = pmadd(x2, p, alpha_3); p = pmadd(x2, p, alpha_1); p = pmul(x, p); // Evaluate the denominator polynomial q. T q = pmadd(x2, beta_6, beta_4); q = pmadd(x2, q, beta_2); q = pmadd(x2, q, beta_0); // Divide the numerator by the denominator. return pselect(tiny_mask, x, pdiv(p, q)); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE RealScalar positive_real_hypot(const RealScalar& x, const RealScalar& y) { // IEEE IEC 6059 special cases. if ((numext::isinf)(x) || (numext::isinf)(y)) return NumTraits::infinity(); if ((numext::isnan)(x) || (numext::isnan)(y)) return NumTraits::quiet_NaN(); EIGEN_USING_STD(sqrt); RealScalar p, qp; p = numext::maxi(x,y); if(p==RealScalar(0)) return RealScalar(0); qp = numext::mini(y,x) / p; return p * sqrt(RealScalar(1) + qp*qp); } template struct hypot_impl { typedef typename NumTraits::Real RealScalar; static EIGEN_DEVICE_FUNC inline RealScalar run(const Scalar& x, const Scalar& y) { EIGEN_USING_STD(abs); return positive_real_hypot(abs(x), abs(y)); } }; // Generic complex sqrt implementation that correctly handles corner cases // according to https://en.cppreference.com/w/cpp/numeric/complex/sqrt template EIGEN_DEVICE_FUNC std::complex complex_sqrt(const std::complex& z) { // Computes the principal sqrt of the input. // // For a complex square root of the number x + i*y. We want to find real // numbers u and v such that // (u + i*v)^2 = x + i*y <=> // u^2 - v^2 + i*2*u*v = x + i*v. // By equating the real and imaginary parts we get: // u^2 - v^2 = x // 2*u*v = y. // // For x >= 0, this has the numerically stable solution // u = sqrt(0.5 * (x + sqrt(x^2 + y^2))) // v = y / (2 * u) // and for x < 0, // v = sign(y) * sqrt(0.5 * (-x + sqrt(x^2 + y^2))) // u = y / (2 * v) // // Letting w = sqrt(0.5 * (|x| + |z|)), // if x == 0: u = w, v = sign(y) * w // if x > 0: u = w, v = y / (2 * w) // if x < 0: u = |y| / (2 * w), v = sign(y) * w const T x = numext::real(z); const T y = numext::imag(z); const T zero = T(0); const T w = numext::sqrt(T(0.5) * (numext::abs(x) + numext::hypot(x, y))); return (numext::isinf)(y) ? std::complex(NumTraits::infinity(), y) : x == zero ? std::complex(w, y < zero ? -w : w) : x > zero ? std::complex(w, y / (2 * w)) : std::complex(numext::abs(y) / (2 * w), y < zero ? -w : w ); } // Generic complex rsqrt implementation. template EIGEN_DEVICE_FUNC std::complex complex_rsqrt(const std::complex& z) { // Computes the principal reciprocal sqrt of the input. // // For a complex reciprocal square root of the number z = x + i*y. We want to // find real numbers u and v such that // (u + i*v)^2 = 1 / (x + i*y) <=> // u^2 - v^2 + i*2*u*v = x/|z|^2 - i*v/|z|^2. // By equating the real and imaginary parts we get: // u^2 - v^2 = x/|z|^2 // 2*u*v = y/|z|^2. // // For x >= 0, this has the numerically stable solution // u = sqrt(0.5 * (x + |z|)) / |z| // v = -y / (2 * u * |z|) // and for x < 0, // v = -sign(y) * sqrt(0.5 * (-x + |z|)) / |z| // u = -y / (2 * v * |z|) // // Letting w = sqrt(0.5 * (|x| + |z|)), // if x == 0: u = w / |z|, v = -sign(y) * w / |z| // if x > 0: u = w / |z|, v = -y / (2 * w * |z|) // if x < 0: u = |y| / (2 * w * |z|), v = -sign(y) * w / |z| const T x = numext::real(z); const T y = numext::imag(z); const T zero = T(0); const T abs_z = numext::hypot(x, y); const T w = numext::sqrt(T(0.5) * (numext::abs(x) + abs_z)); const T woz = w / abs_z; // Corner cases consistent with 1/sqrt(z) on gcc/clang. return abs_z == zero ? std::complex(NumTraits::infinity(), NumTraits::quiet_NaN()) : ((numext::isinf)(x) || (numext::isinf)(y)) ? std::complex(zero, zero) : x == zero ? std::complex(woz, y < zero ? woz : -woz) : x > zero ? std::complex(woz, -y / (2 * w * abs_z)) : std::complex(numext::abs(y) / (2 * w * abs_z), y < zero ? woz : -woz ); } template EIGEN_DEVICE_FUNC std::complex complex_log(const std::complex& z) { // Computes complex log. T a = numext::abs(z); EIGEN_USING_STD(atan2); T b = atan2(z.imag(), z.real()); return std::complex(numext::log(a), b); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_MATHFUNCTIONSIMPL_H RcppEigen/inst/include/Eigen/src/Core/SelfAdjointView.h0000644000176200001440000003522714567757725022504 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2009 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_SELFADJOINTMATRIX_H #define EIGEN_SELFADJOINTMATRIX_H namespace Eigen { /** \class SelfAdjointView * \ingroup Core_Module * * * \brief Expression of a selfadjoint matrix from a triangular part of a dense matrix * * \param MatrixType the type of the dense matrix storing the coefficients * \param TriangularPart can be either \c #Lower or \c #Upper * * This class is an expression of a sefladjoint matrix from a triangular part of a matrix * with given dense storage of the coefficients. It is the return type of MatrixBase::selfadjointView() * and most of the time this is the only way that it is used. * * \sa class TriangularBase, MatrixBase::selfadjointView() */ namespace internal { template struct traits > : traits { typedef typename ref_selector::non_const_type MatrixTypeNested; typedef typename remove_all::type MatrixTypeNestedCleaned; typedef MatrixType ExpressionType; typedef typename MatrixType::PlainObject FullMatrixType; enum { Mode = UpLo | SelfAdjoint, FlagsLvalueBit = is_lvalue::value ? LvalueBit : 0, Flags = MatrixTypeNestedCleaned::Flags & (HereditaryBits|FlagsLvalueBit) & (~(PacketAccessBit | DirectAccessBit | LinearAccessBit)) // FIXME these flags should be preserved }; }; } template class SelfAdjointView : public TriangularBase > { public: typedef _MatrixType MatrixType; typedef TriangularBase Base; typedef typename internal::traits::MatrixTypeNested MatrixTypeNested; typedef typename internal::traits::MatrixTypeNestedCleaned MatrixTypeNestedCleaned; typedef MatrixTypeNestedCleaned NestedExpression; /** \brief The type of coefficients in this matrix */ typedef typename internal::traits::Scalar Scalar; typedef typename MatrixType::StorageIndex StorageIndex; typedef typename internal::remove_all::type MatrixConjugateReturnType; typedef SelfAdjointView::type, UpLo> ConstSelfAdjointView; enum { Mode = internal::traits::Mode, Flags = internal::traits::Flags, TransposeMode = ((int(Mode) & int(Upper)) ? Lower : 0) | ((int(Mode) & int(Lower)) ? Upper : 0) }; typedef typename MatrixType::PlainObject PlainObject; EIGEN_DEVICE_FUNC explicit inline SelfAdjointView(MatrixType& matrix) : m_matrix(matrix) { EIGEN_STATIC_ASSERT(UpLo==Lower || UpLo==Upper,SELFADJOINTVIEW_ACCEPTS_UPPER_AND_LOWER_MODE_ONLY); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return m_matrix.rows(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return m_matrix.cols(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT { return m_matrix.outerStride(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT { return m_matrix.innerStride(); } /** \sa MatrixBase::coeff() * \warning the coordinates must fit into the referenced triangular part */ EIGEN_DEVICE_FUNC inline Scalar coeff(Index row, Index col) const { Base::check_coordinates_internal(row, col); return m_matrix.coeff(row, col); } /** \sa MatrixBase::coeffRef() * \warning the coordinates must fit into the referenced triangular part */ EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index row, Index col) { EIGEN_STATIC_ASSERT_LVALUE(SelfAdjointView); Base::check_coordinates_internal(row, col); return m_matrix.coeffRef(row, col); } /** \internal */ EIGEN_DEVICE_FUNC const MatrixTypeNestedCleaned& _expression() const { return m_matrix; } EIGEN_DEVICE_FUNC const MatrixTypeNestedCleaned& nestedExpression() const { return m_matrix; } EIGEN_DEVICE_FUNC MatrixTypeNestedCleaned& nestedExpression() { return m_matrix; } /** Efficient triangular matrix times vector/matrix product */ template EIGEN_DEVICE_FUNC const Product operator*(const MatrixBase& rhs) const { return Product(*this, rhs.derived()); } /** Efficient vector/matrix times triangular matrix product */ template friend EIGEN_DEVICE_FUNC const Product operator*(const MatrixBase& lhs, const SelfAdjointView& rhs) { return Product(lhs.derived(),rhs); } friend EIGEN_DEVICE_FUNC const SelfAdjointView operator*(const Scalar& s, const SelfAdjointView& mat) { return (s*mat.nestedExpression()).template selfadjointView(); } /** Perform a symmetric rank 2 update of the selfadjoint matrix \c *this: * \f$ this = this + \alpha u v^* + conj(\alpha) v u^* \f$ * \returns a reference to \c *this * * The vectors \a u and \c v \b must be column vectors, however they can be * a adjoint expression without any overhead. Only the meaningful triangular * part of the matrix is updated, the rest is left unchanged. * * \sa rankUpdate(const MatrixBase&, Scalar) */ template EIGEN_DEVICE_FUNC SelfAdjointView& rankUpdate(const MatrixBase& u, const MatrixBase& v, const Scalar& alpha = Scalar(1)); /** Perform a symmetric rank K update of the selfadjoint matrix \c *this: * \f$ this = this + \alpha ( u u^* ) \f$ where \a u is a vector or matrix. * * \returns a reference to \c *this * * Note that to perform \f$ this = this + \alpha ( u^* u ) \f$ you can simply * call this function with u.adjoint(). * * \sa rankUpdate(const MatrixBase&, const MatrixBase&, Scalar) */ template EIGEN_DEVICE_FUNC SelfAdjointView& rankUpdate(const MatrixBase& u, const Scalar& alpha = Scalar(1)); /** \returns an expression of a triangular view extracted from the current selfadjoint view of a given triangular part * * The parameter \a TriMode can have the following values: \c #Upper, \c #StrictlyUpper, \c #UnitUpper, * \c #Lower, \c #StrictlyLower, \c #UnitLower. * * If \c TriMode references the same triangular part than \c *this, then this method simply return a \c TriangularView of the nested expression, * otherwise, the nested expression is first transposed, thus returning a \c TriangularView> object. * * \sa MatrixBase::triangularView(), class TriangularView */ template EIGEN_DEVICE_FUNC typename internal::conditional<(TriMode&(Upper|Lower))==(UpLo&(Upper|Lower)), TriangularView, TriangularView >::type triangularView() const { typename internal::conditional<(TriMode&(Upper|Lower))==(UpLo&(Upper|Lower)), MatrixType&, typename MatrixType::ConstTransposeReturnType>::type tmp1(m_matrix); typename internal::conditional<(TriMode&(Upper|Lower))==(UpLo&(Upper|Lower)), MatrixType&, typename MatrixType::AdjointReturnType>::type tmp2(tmp1); return typename internal::conditional<(TriMode&(Upper|Lower))==(UpLo&(Upper|Lower)), TriangularView, TriangularView >::type(tmp2); } typedef SelfAdjointView ConjugateReturnType; /** \sa MatrixBase::conjugate() const */ EIGEN_DEVICE_FUNC inline const ConjugateReturnType conjugate() const { return ConjugateReturnType(m_matrix.conjugate()); } /** \returns an expression of the complex conjugate of \c *this if Cond==true, * returns \c *this otherwise. */ template EIGEN_DEVICE_FUNC inline typename internal::conditional::type conjugateIf() const { typedef typename internal::conditional::type ReturnType; return ReturnType(m_matrix.template conjugateIf()); } typedef SelfAdjointView AdjointReturnType; /** \sa MatrixBase::adjoint() const */ EIGEN_DEVICE_FUNC inline const AdjointReturnType adjoint() const { return AdjointReturnType(m_matrix.adjoint()); } typedef SelfAdjointView TransposeReturnType; /** \sa MatrixBase::transpose() */ EIGEN_DEVICE_FUNC inline TransposeReturnType transpose() { EIGEN_STATIC_ASSERT_LVALUE(MatrixType) typename MatrixType::TransposeReturnType tmp(m_matrix); return TransposeReturnType(tmp); } typedef SelfAdjointView ConstTransposeReturnType; /** \sa MatrixBase::transpose() const */ EIGEN_DEVICE_FUNC inline const ConstTransposeReturnType transpose() const { return ConstTransposeReturnType(m_matrix.transpose()); } /** \returns a const expression of the main diagonal of the matrix \c *this * * This method simply returns the diagonal of the nested expression, thus by-passing the SelfAdjointView decorator. * * \sa MatrixBase::diagonal(), class Diagonal */ EIGEN_DEVICE_FUNC typename MatrixType::ConstDiagonalReturnType diagonal() const { return typename MatrixType::ConstDiagonalReturnType(m_matrix); } /////////// Cholesky module /////////// const LLT llt() const; const LDLT ldlt() const; /////////// Eigenvalue module /////////// /** Real part of #Scalar */ typedef typename NumTraits::Real RealScalar; /** Return type of eigenvalues() */ typedef Matrix::ColsAtCompileTime, 1> EigenvaluesReturnType; EIGEN_DEVICE_FUNC EigenvaluesReturnType eigenvalues() const; EIGEN_DEVICE_FUNC RealScalar operatorNorm() const; protected: MatrixTypeNested m_matrix; }; // template // internal::selfadjoint_matrix_product_returntype > // operator*(const MatrixBase& lhs, const SelfAdjointView& rhs) // { // return internal::matrix_selfadjoint_product_returntype >(lhs.derived(),rhs); // } // selfadjoint to dense matrix namespace internal { // TODO currently a selfadjoint expression has the form SelfAdjointView<.,.> // in the future selfadjoint-ness should be defined by the expression traits // such that Transpose > is valid. (currently TriangularBase::transpose() is overloaded to make it work) template struct evaluator_traits > { typedef typename storage_kind_to_evaluator_kind::Kind Kind; typedef SelfAdjointShape Shape; }; template class triangular_dense_assignment_kernel : public generic_dense_assignment_kernel { protected: typedef generic_dense_assignment_kernel Base; typedef typename Base::DstXprType DstXprType; typedef typename Base::SrcXprType SrcXprType; using Base::m_dst; using Base::m_src; using Base::m_functor; public: typedef typename Base::DstEvaluatorType DstEvaluatorType; typedef typename Base::SrcEvaluatorType SrcEvaluatorType; typedef typename Base::Scalar Scalar; typedef typename Base::AssignmentTraits AssignmentTraits; EIGEN_DEVICE_FUNC triangular_dense_assignment_kernel(DstEvaluatorType &dst, const SrcEvaluatorType &src, const Functor &func, DstXprType& dstExpr) : Base(dst, src, func, dstExpr) {} EIGEN_DEVICE_FUNC void assignCoeff(Index row, Index col) { eigen_internal_assert(row!=col); Scalar tmp = m_src.coeff(row,col); m_functor.assignCoeff(m_dst.coeffRef(row,col), tmp); m_functor.assignCoeff(m_dst.coeffRef(col,row), numext::conj(tmp)); } EIGEN_DEVICE_FUNC void assignDiagonalCoeff(Index id) { Base::assignCoeff(id,id); } EIGEN_DEVICE_FUNC void assignOppositeCoeff(Index, Index) { eigen_internal_assert(false && "should never be called"); } }; } // end namespace internal /*************************************************************************** * Implementation of MatrixBase methods ***************************************************************************/ /** This is the const version of MatrixBase::selfadjointView() */ template template EIGEN_DEVICE_FUNC typename MatrixBase::template ConstSelfAdjointViewReturnType::Type MatrixBase::selfadjointView() const { return typename ConstSelfAdjointViewReturnType::Type(derived()); } /** \returns an expression of a symmetric/self-adjoint view extracted from the upper or lower triangular part of the current matrix * * The parameter \a UpLo can be either \c #Upper or \c #Lower * * Example: \include MatrixBase_selfadjointView.cpp * Output: \verbinclude MatrixBase_selfadjointView.out * * \sa class SelfAdjointView */ template template EIGEN_DEVICE_FUNC typename MatrixBase::template SelfAdjointViewReturnType::Type MatrixBase::selfadjointView() { return typename SelfAdjointViewReturnType::Type(derived()); } } // end namespace Eigen #endif // EIGEN_SELFADJOINTMATRIX_H RcppEigen/inst/include/Eigen/src/Core/NestByValue.h0000644000176200001440000000473014567757725021643 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008 Gael Guennebaud // Copyright (C) 2006-2008 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_NESTBYVALUE_H #define EIGEN_NESTBYVALUE_H namespace Eigen { namespace internal { template struct traits > : public traits { enum { Flags = traits::Flags & ~NestByRefBit }; }; } /** \class NestByValue * \ingroup Core_Module * * \brief Expression which must be nested by value * * \tparam ExpressionType the type of the object of which we are requiring nesting-by-value * * This class is the return type of MatrixBase::nestByValue() * and most of the time this is the only way it is used. * * \sa MatrixBase::nestByValue() */ template class NestByValue : public internal::dense_xpr_base< NestByValue >::type { public: typedef typename internal::dense_xpr_base::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(NestByValue) EIGEN_DEVICE_FUNC explicit inline NestByValue(const ExpressionType& matrix) : m_expression(matrix) {} EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return m_expression.rows(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return m_expression.cols(); } EIGEN_DEVICE_FUNC operator const ExpressionType&() const { return m_expression; } EIGEN_DEVICE_FUNC const ExpressionType& nestedExpression() const { return m_expression; } protected: const ExpressionType m_expression; }; /** \returns an expression of the temporary version of *this. */ template EIGEN_DEVICE_FUNC inline const NestByValue DenseBase::nestByValue() const { return NestByValue(derived()); } namespace internal { // Evaluator of Solve -> eval into a temporary template struct evaluator > : public evaluator { typedef evaluator Base; EIGEN_DEVICE_FUNC explicit evaluator(const NestByValue& xpr) : Base(xpr.nestedExpression()) {} }; } } // end namespace Eigen #endif // EIGEN_NESTBYVALUE_H RcppEigen/inst/include/Eigen/src/Core/Dot.h0000644000176200001440000002660614567757725020176 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2006-2008, 2010 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_DOT_H #define EIGEN_DOT_H namespace Eigen { namespace internal { // helper function for dot(). The problem is that if we put that in the body of dot(), then upon calling dot // with mismatched types, the compiler emits errors about failing to instantiate cwiseProduct BEFORE // looking at the static assertions. Thus this is a trick to get better compile errors. template struct dot_nocheck { typedef scalar_conj_product_op::Scalar,typename traits::Scalar> conj_prod; typedef typename conj_prod::result_type ResScalar; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static ResScalar run(const MatrixBase& a, const MatrixBase& b) { return a.template binaryExpr(b).sum(); } }; template struct dot_nocheck { typedef scalar_conj_product_op::Scalar,typename traits::Scalar> conj_prod; typedef typename conj_prod::result_type ResScalar; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static ResScalar run(const MatrixBase& a, const MatrixBase& b) { return a.transpose().template binaryExpr(b).sum(); } }; } // end namespace internal /** \fn MatrixBase::dot * \returns the dot product of *this with other. * * \only_for_vectors * * \note If the scalar type is complex numbers, then this function returns the hermitian * (sesquilinear) dot product, conjugate-linear in the first variable and linear in the * second variable. * * \sa squaredNorm(), norm() */ template template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename ScalarBinaryOpTraits::Scalar,typename internal::traits::Scalar>::ReturnType MatrixBase::dot(const MatrixBase& other) const { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) EIGEN_STATIC_ASSERT_SAME_VECTOR_SIZE(Derived,OtherDerived) #if !(defined(EIGEN_NO_STATIC_ASSERT) && defined(EIGEN_NO_DEBUG)) typedef internal::scalar_conj_product_op func; EIGEN_CHECK_BINARY_COMPATIBILIY(func,Scalar,typename OtherDerived::Scalar); #endif eigen_assert(size() == other.size()); return internal::dot_nocheck::run(*this, other); } //---------- implementation of L2 norm and related functions ---------- /** \returns, for vectors, the squared \em l2 norm of \c *this, and for matrices the squared Frobenius norm. * In both cases, it consists in the sum of the square of all the matrix entries. * For vectors, this is also equals to the dot product of \c *this with itself. * * \sa dot(), norm(), lpNorm() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename NumTraits::Scalar>::Real MatrixBase::squaredNorm() const { return numext::real((*this).cwiseAbs2().sum()); } /** \returns, for vectors, the \em l2 norm of \c *this, and for matrices the Frobenius norm. * In both cases, it consists in the square root of the sum of the square of all the matrix entries. * For vectors, this is also equals to the square root of the dot product of \c *this with itself. * * \sa lpNorm(), dot(), squaredNorm() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename NumTraits::Scalar>::Real MatrixBase::norm() const { return numext::sqrt(squaredNorm()); } /** \returns an expression of the quotient of \c *this by its own norm. * * \warning If the input vector is too small (i.e., this->norm()==0), * then this function returns a copy of the input. * * \only_for_vectors * * \sa norm(), normalize() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase::PlainObject MatrixBase::normalized() const { typedef typename internal::nested_eval::type _Nested; _Nested n(derived()); RealScalar z = n.squaredNorm(); // NOTE: after extensive benchmarking, this conditional does not impact performance, at least on recent x86 CPU if(z>RealScalar(0)) return n / numext::sqrt(z); else return n; } /** Normalizes the vector, i.e. divides it by its own norm. * * \only_for_vectors * * \warning If the input vector is too small (i.e., this->norm()==0), then \c *this is left unchanged. * * \sa norm(), normalized() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void MatrixBase::normalize() { RealScalar z = squaredNorm(); // NOTE: after extensive benchmarking, this conditional does not impact performance, at least on recent x86 CPU if(z>RealScalar(0)) derived() /= numext::sqrt(z); } /** \returns an expression of the quotient of \c *this by its own norm while avoiding underflow and overflow. * * \only_for_vectors * * This method is analogue to the normalized() method, but it reduces the risk of * underflow and overflow when computing the norm. * * \warning If the input vector is too small (i.e., this->norm()==0), * then this function returns a copy of the input. * * \sa stableNorm(), stableNormalize(), normalized() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase::PlainObject MatrixBase::stableNormalized() const { typedef typename internal::nested_eval::type _Nested; _Nested n(derived()); RealScalar w = n.cwiseAbs().maxCoeff(); RealScalar z = (n/w).squaredNorm(); if(z>RealScalar(0)) return n / (numext::sqrt(z)*w); else return n; } /** Normalizes the vector while avoid underflow and overflow * * \only_for_vectors * * This method is analogue to the normalize() method, but it reduces the risk of * underflow and overflow when computing the norm. * * \warning If the input vector is too small (i.e., this->norm()==0), then \c *this is left unchanged. * * \sa stableNorm(), stableNormalized(), normalize() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void MatrixBase::stableNormalize() { RealScalar w = cwiseAbs().maxCoeff(); RealScalar z = (derived()/w).squaredNorm(); if(z>RealScalar(0)) derived() /= numext::sqrt(z)*w; } //---------- implementation of other norms ---------- namespace internal { template struct lpNorm_selector { typedef typename NumTraits::Scalar>::Real RealScalar; EIGEN_DEVICE_FUNC static inline RealScalar run(const MatrixBase& m) { EIGEN_USING_STD(pow) return pow(m.cwiseAbs().array().pow(p).sum(), RealScalar(1)/p); } }; template struct lpNorm_selector { EIGEN_DEVICE_FUNC static inline typename NumTraits::Scalar>::Real run(const MatrixBase& m) { return m.cwiseAbs().sum(); } }; template struct lpNorm_selector { EIGEN_DEVICE_FUNC static inline typename NumTraits::Scalar>::Real run(const MatrixBase& m) { return m.norm(); } }; template struct lpNorm_selector { typedef typename NumTraits::Scalar>::Real RealScalar; EIGEN_DEVICE_FUNC static inline RealScalar run(const MatrixBase& m) { if(Derived::SizeAtCompileTime==0 || (Derived::SizeAtCompileTime==Dynamic && m.size()==0)) return RealScalar(0); return m.cwiseAbs().maxCoeff(); } }; } // end namespace internal /** \returns the \b coefficient-wise \f$ \ell^p \f$ norm of \c *this, that is, returns the p-th root of the sum of the p-th powers of the absolute values * of the coefficients of \c *this. If \a p is the special value \a Eigen::Infinity, this function returns the \f$ \ell^\infty \f$ * norm, that is the maximum of the absolute values of the coefficients of \c *this. * * In all cases, if \c *this is empty, then the value 0 is returned. * * \note For matrices, this function does not compute the operator-norm. That is, if \c *this is a matrix, then its coefficients are interpreted as a 1D vector. Nonetheless, you can easily compute the 1-norm and \f$\infty\f$-norm matrix operator norms using \link TutorialReductionsVisitorsBroadcastingReductionsNorm partial reductions \endlink. * * \sa norm() */ template template #ifndef EIGEN_PARSED_BY_DOXYGEN EIGEN_DEVICE_FUNC inline typename NumTraits::Scalar>::Real #else EIGEN_DEVICE_FUNC MatrixBase::RealScalar #endif MatrixBase::lpNorm() const { return internal::lpNorm_selector::run(*this); } //---------- implementation of isOrthogonal / isUnitary ---------- /** \returns true if *this is approximately orthogonal to \a other, * within the precision given by \a prec. * * Example: \include MatrixBase_isOrthogonal.cpp * Output: \verbinclude MatrixBase_isOrthogonal.out */ template template bool MatrixBase::isOrthogonal (const MatrixBase& other, const RealScalar& prec) const { typename internal::nested_eval::type nested(derived()); typename internal::nested_eval::type otherNested(other.derived()); return numext::abs2(nested.dot(otherNested)) <= prec * prec * nested.squaredNorm() * otherNested.squaredNorm(); } /** \returns true if *this is approximately an unitary matrix, * within the precision given by \a prec. In the case where the \a Scalar * type is real numbers, a unitary matrix is an orthogonal matrix, whence the name. * * \note This can be used to check whether a family of vectors forms an orthonormal basis. * Indeed, \c m.isUnitary() returns true if and only if the columns (equivalently, the rows) of m form an * orthonormal basis. * * Example: \include MatrixBase_isUnitary.cpp * Output: \verbinclude MatrixBase_isUnitary.out */ template bool MatrixBase::isUnitary(const RealScalar& prec) const { typename internal::nested_eval::type self(derived()); for(Index i = 0; i < cols(); ++i) { if(!internal::isApprox(self.col(i).squaredNorm(), static_cast(1), prec)) return false; for(Index j = 0; j < i; ++j) if(!internal::isMuchSmallerThan(self.col(i).dot(self.col(j)), static_cast(1), prec)) return false; } return true; } } // end namespace Eigen #endif // EIGEN_DOT_H RcppEigen/inst/include/Eigen/src/Core/CwiseUnaryOp.h0000644000176200001440000000754114567757725022035 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2014 Gael Guennebaud // Copyright (C) 2006-2008 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_CWISE_UNARY_OP_H #define EIGEN_CWISE_UNARY_OP_H namespace Eigen { namespace internal { template struct traits > : traits { typedef typename result_of< UnaryOp(const typename XprType::Scalar&) >::type Scalar; typedef typename XprType::Nested XprTypeNested; typedef typename remove_reference::type _XprTypeNested; enum { Flags = _XprTypeNested::Flags & RowMajorBit }; }; } template class CwiseUnaryOpImpl; /** \class CwiseUnaryOp * \ingroup Core_Module * * \brief Generic expression where a coefficient-wise unary operator is applied to an expression * * \tparam UnaryOp template functor implementing the operator * \tparam XprType the type of the expression to which we are applying the unary operator * * This class represents an expression where a unary operator is applied to an expression. * It is the return type of all operations taking exactly 1 input expression, regardless of the * presence of other inputs such as scalars. For example, the operator* in the expression 3*matrix * is considered unary, because only the right-hand side is an expression, and its * return type is a specialization of CwiseUnaryOp. * * Most of the time, this is the only way that it is used, so you typically don't have to name * CwiseUnaryOp types explicitly. * * \sa MatrixBase::unaryExpr(const CustomUnaryOp &) const, class CwiseBinaryOp, class CwiseNullaryOp */ template class CwiseUnaryOp : public CwiseUnaryOpImpl::StorageKind>, internal::no_assignment_operator { public: typedef typename CwiseUnaryOpImpl::StorageKind>::Base Base; EIGEN_GENERIC_PUBLIC_INTERFACE(CwiseUnaryOp) typedef typename internal::ref_selector::type XprTypeNested; typedef typename internal::remove_all::type NestedExpression; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit CwiseUnaryOp(const XprType& xpr, const UnaryOp& func = UnaryOp()) : m_xpr(xpr), m_functor(func) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_xpr.rows(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_xpr.cols(); } /** \returns the functor representing the unary operation */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const UnaryOp& functor() const { return m_functor; } /** \returns the nested expression */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename internal::remove_all::type& nestedExpression() const { return m_xpr; } /** \returns the nested expression */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::remove_all::type& nestedExpression() { return m_xpr; } protected: XprTypeNested m_xpr; const UnaryOp m_functor; }; // Generic API dispatcher template class CwiseUnaryOpImpl : public internal::generic_xpr_base >::type { public: typedef typename internal::generic_xpr_base >::type Base; }; } // end namespace Eigen #endif // EIGEN_CWISE_UNARY_OP_H RcppEigen/inst/include/Eigen/src/Core/ArithmeticSequence.h0000644000176200001440000004541614567757725023232 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2017 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_ARITHMETIC_SEQUENCE_H #define EIGEN_ARITHMETIC_SEQUENCE_H namespace Eigen { namespace internal { #if (!EIGEN_HAS_CXX11) || !((!EIGEN_COMP_GNUC) || EIGEN_COMP_GNUC>=48) template struct aseq_negate {}; template<> struct aseq_negate { typedef Index type; }; template struct aseq_negate > { typedef FixedInt<-N> type; }; // Compilation error in the following case: template<> struct aseq_negate > {}; template::value, bool SizeIsSymbolic =symbolic::is_symbolic::value> struct aseq_reverse_first_type { typedef Index type; }; template struct aseq_reverse_first_type { typedef symbolic::AddExpr > >, symbolic::ValueExpr > > type; }; template struct aseq_reverse_first_type_aux { typedef Index type; }; template struct aseq_reverse_first_type_aux::type> { typedef FixedInt<(SizeType::value-1)*IncrType::value> type; }; template struct aseq_reverse_first_type { typedef typename aseq_reverse_first_type_aux::type Aux; typedef symbolic::AddExpr > type; }; template struct aseq_reverse_first_type { typedef symbolic::AddExpr > >, symbolic::ValueExpr >, symbolic::ValueExpr<> > type; }; #endif // Helper to cleanup the type of the increment: template struct cleanup_seq_incr { typedef typename cleanup_index_type::type type; }; } //-------------------------------------------------------------------------------- // seq(first,last,incr) and seqN(first,size,incr) //-------------------------------------------------------------------------------- template > class ArithmeticSequence; template ArithmeticSequence::type, typename internal::cleanup_index_type::type, typename internal::cleanup_seq_incr::type > seqN(FirstType first, SizeType size, IncrType incr); /** \class ArithmeticSequence * \ingroup Core_Module * * This class represents an arithmetic progression \f$ a_0, a_1, a_2, ..., a_{n-1}\f$ defined by * its \em first value \f$ a_0 \f$, its \em size (aka length) \em n, and the \em increment (aka stride) * that is equal to \f$ a_{i+1}-a_{i}\f$ for any \em i. * * It is internally used as the return type of the Eigen::seq and Eigen::seqN functions, and as the input arguments * of DenseBase::operator()(const RowIndices&, const ColIndices&), and most of the time this is the * only way it is used. * * \tparam FirstType type of the first element, usually an Index, * but internally it can be a symbolic expression * \tparam SizeType type representing the size of the sequence, usually an Index * or a compile time integral constant. Internally, it can also be a symbolic expression * \tparam IncrType type of the increment, can be a runtime Index, or a compile time integral constant (default is compile-time 1) * * \sa Eigen::seq, Eigen::seqN, DenseBase::operator()(const RowIndices&, const ColIndices&), class IndexedView */ template class ArithmeticSequence { public: ArithmeticSequence(FirstType first, SizeType size) : m_first(first), m_size(size) {} ArithmeticSequence(FirstType first, SizeType size, IncrType incr) : m_first(first), m_size(size), m_incr(incr) {} enum { SizeAtCompileTime = internal::get_fixed_value::value, IncrAtCompileTime = internal::get_fixed_value::value }; /** \returns the size, i.e., number of elements, of the sequence */ Index size() const { return m_size; } /** \returns the first element \f$ a_0 \f$ in the sequence */ Index first() const { return m_first; } /** \returns the value \f$ a_i \f$ at index \a i in the sequence. */ Index operator[](Index i) const { return m_first + i * m_incr; } const FirstType& firstObject() const { return m_first; } const SizeType& sizeObject() const { return m_size; } const IncrType& incrObject() const { return m_incr; } protected: FirstType m_first; SizeType m_size; IncrType m_incr; public: #if EIGEN_HAS_CXX11 && ((!EIGEN_COMP_GNUC) || EIGEN_COMP_GNUC>=48) auto reverse() const -> decltype(Eigen::seqN(m_first+(m_size+fix<-1>())*m_incr,m_size,-m_incr)) { return seqN(m_first+(m_size+fix<-1>())*m_incr,m_size,-m_incr); } #else protected: typedef typename internal::aseq_negate::type ReverseIncrType; typedef typename internal::aseq_reverse_first_type::type ReverseFirstType; public: ArithmeticSequence reverse() const { return seqN(m_first+(m_size+fix<-1>())*m_incr,m_size,-m_incr); } #endif }; /** \returns an ArithmeticSequence starting at \a first, of length \a size, and increment \a incr * * \sa seqN(FirstType,SizeType), seq(FirstType,LastType,IncrType) */ template ArithmeticSequence::type,typename internal::cleanup_index_type::type,typename internal::cleanup_seq_incr::type > seqN(FirstType first, SizeType size, IncrType incr) { return ArithmeticSequence::type,typename internal::cleanup_index_type::type,typename internal::cleanup_seq_incr::type>(first,size,incr); } /** \returns an ArithmeticSequence starting at \a first, of length \a size, and unit increment * * \sa seqN(FirstType,SizeType,IncrType), seq(FirstType,LastType) */ template ArithmeticSequence::type,typename internal::cleanup_index_type::type > seqN(FirstType first, SizeType size) { return ArithmeticSequence::type,typename internal::cleanup_index_type::type>(first,size); } #ifdef EIGEN_PARSED_BY_DOXYGEN /** \returns an ArithmeticSequence starting at \a f, up (or down) to \a l, and with positive (or negative) increment \a incr * * It is essentially an alias to: * \code * seqN(f, (l-f+incr)/incr, incr); * \endcode * * \sa seqN(FirstType,SizeType,IncrType), seq(FirstType,LastType) */ template auto seq(FirstType f, LastType l, IncrType incr); /** \returns an ArithmeticSequence starting at \a f, up (or down) to \a l, and unit increment * * It is essentially an alias to: * \code * seqN(f,l-f+1); * \endcode * * \sa seqN(FirstType,SizeType), seq(FirstType,LastType,IncrType) */ template auto seq(FirstType f, LastType l); #else // EIGEN_PARSED_BY_DOXYGEN #if EIGEN_HAS_CXX11 template auto seq(FirstType f, LastType l) -> decltype(seqN(typename internal::cleanup_index_type::type(f), ( typename internal::cleanup_index_type::type(l) - typename internal::cleanup_index_type::type(f)+fix<1>()))) { return seqN(typename internal::cleanup_index_type::type(f), (typename internal::cleanup_index_type::type(l) -typename internal::cleanup_index_type::type(f)+fix<1>())); } template auto seq(FirstType f, LastType l, IncrType incr) -> decltype(seqN(typename internal::cleanup_index_type::type(f), ( typename internal::cleanup_index_type::type(l) - typename internal::cleanup_index_type::type(f)+typename internal::cleanup_seq_incr::type(incr) ) / typename internal::cleanup_seq_incr::type(incr), typename internal::cleanup_seq_incr::type(incr))) { typedef typename internal::cleanup_seq_incr::type CleanedIncrType; return seqN(typename internal::cleanup_index_type::type(f), ( typename internal::cleanup_index_type::type(l) -typename internal::cleanup_index_type::type(f)+CleanedIncrType(incr)) / CleanedIncrType(incr), CleanedIncrType(incr)); } #else // EIGEN_HAS_CXX11 template typename internal::enable_if::value || symbolic::is_symbolic::value), ArithmeticSequence::type,Index> >::type seq(FirstType f, LastType l) { return seqN(typename internal::cleanup_index_type::type(f), Index((typename internal::cleanup_index_type::type(l)-typename internal::cleanup_index_type::type(f)+fix<1>()))); } template typename internal::enable_if::value, ArithmeticSequence,symbolic::ValueExpr<> >, symbolic::ValueExpr > > > >::type seq(const symbolic::BaseExpr &f, LastType l) { return seqN(f.derived(),(typename internal::cleanup_index_type::type(l)-f.derived()+fix<1>())); } template typename internal::enable_if::value, ArithmeticSequence::type, symbolic::AddExpr >, symbolic::ValueExpr > > > >::type seq(FirstType f, const symbolic::BaseExpr &l) { return seqN(typename internal::cleanup_index_type::type(f),(l.derived()-typename internal::cleanup_index_type::type(f)+fix<1>())); } template ArithmeticSequence >,symbolic::ValueExpr > > > seq(const symbolic::BaseExpr &f, const symbolic::BaseExpr &l) { return seqN(f.derived(),(l.derived()-f.derived()+fix<1>())); } template typename internal::enable_if::value || symbolic::is_symbolic::value), ArithmeticSequence::type,Index,typename internal::cleanup_seq_incr::type> >::type seq(FirstType f, LastType l, IncrType incr) { typedef typename internal::cleanup_seq_incr::type CleanedIncrType; return seqN(typename internal::cleanup_index_type::type(f), Index((typename internal::cleanup_index_type::type(l)-typename internal::cleanup_index_type::type(f)+CleanedIncrType(incr))/CleanedIncrType(incr)), incr); } template typename internal::enable_if::value, ArithmeticSequence, symbolic::ValueExpr<> >, symbolic::ValueExpr::type> >, symbolic::ValueExpr::type> >, typename internal::cleanup_seq_incr::type> >::type seq(const symbolic::BaseExpr &f, LastType l, IncrType incr) { typedef typename internal::cleanup_seq_incr::type CleanedIncrType; return seqN(f.derived(),(typename internal::cleanup_index_type::type(l)-f.derived()+CleanedIncrType(incr))/CleanedIncrType(incr), incr); } template typename internal::enable_if::value, ArithmeticSequence::type, symbolic::QuotientExpr >, symbolic::ValueExpr::type> >, symbolic::ValueExpr::type> >, typename internal::cleanup_seq_incr::type> >::type seq(FirstType f, const symbolic::BaseExpr &l, IncrType incr) { typedef typename internal::cleanup_seq_incr::type CleanedIncrType; return seqN(typename internal::cleanup_index_type::type(f), (l.derived()-typename internal::cleanup_index_type::type(f)+CleanedIncrType(incr))/CleanedIncrType(incr), incr); } template ArithmeticSequence >, symbolic::ValueExpr::type> >, symbolic::ValueExpr::type> >, typename internal::cleanup_seq_incr::type> seq(const symbolic::BaseExpr &f, const symbolic::BaseExpr &l, IncrType incr) { typedef typename internal::cleanup_seq_incr::type CleanedIncrType; return seqN(f.derived(),(l.derived()-f.derived()+CleanedIncrType(incr))/CleanedIncrType(incr), incr); } #endif // EIGEN_HAS_CXX11 #endif // EIGEN_PARSED_BY_DOXYGEN #if EIGEN_HAS_CXX11 || defined(EIGEN_PARSED_BY_DOXYGEN) /** \cpp11 * \returns a symbolic ArithmeticSequence representing the last \a size elements with increment \a incr. * * It is a shortcut for: \code seqN(last-(size-fix<1>)*incr, size, incr) \endcode * * \sa lastN(SizeType), seqN(FirstType,SizeType), seq(FirstType,LastType,IncrType) */ template auto lastN(SizeType size, IncrType incr) -> decltype(seqN(Eigen::last-(size-fix<1>())*incr, size, incr)) { return seqN(Eigen::last-(size-fix<1>())*incr, size, incr); } /** \cpp11 * \returns a symbolic ArithmeticSequence representing the last \a size elements with a unit increment. * * It is a shortcut for: \code seq(last+fix<1>-size, last) \endcode * * \sa lastN(SizeType,IncrType, seqN(FirstType,SizeType), seq(FirstType,LastType) */ template auto lastN(SizeType size) -> decltype(seqN(Eigen::last+fix<1>()-size, size)) { return seqN(Eigen::last+fix<1>()-size, size); } #endif namespace internal { // Convert a symbolic span into a usable one (i.e., remove last/end "keywords") template struct make_size_type { typedef typename internal::conditional::value, Index, T>::type type; }; template struct IndexedViewCompatibleType, XprSize> { typedef ArithmeticSequence::type,IncrType> type; }; template ArithmeticSequence::type,IncrType> makeIndexedViewCompatible(const ArithmeticSequence& ids, Index size,SpecializedType) { return ArithmeticSequence::type,IncrType>( eval_expr_given_size(ids.firstObject(),size),eval_expr_given_size(ids.sizeObject(),size),ids.incrObject()); } template struct get_compile_time_incr > { enum { value = get_fixed_value::value }; }; } // end namespace internal /** \namespace Eigen::indexing * \ingroup Core_Module * * The sole purpose of this namespace is to be able to import all functions * and symbols that are expected to be used within operator() for indexing * and slicing. If you already imported the whole Eigen namespace: * \code using namespace Eigen; \endcode * then you are already all set. Otherwise, if you don't want/cannot import * the whole Eigen namespace, the following line: * \code using namespace Eigen::indexing; \endcode * is equivalent to: * \code using Eigen::all; using Eigen::seq; using Eigen::seqN; using Eigen::lastN; // c++11 only using Eigen::last; using Eigen::lastp1; using Eigen::fix; \endcode */ namespace indexing { using Eigen::all; using Eigen::seq; using Eigen::seqN; #if EIGEN_HAS_CXX11 using Eigen::lastN; #endif using Eigen::last; using Eigen::lastp1; using Eigen::fix; } } // end namespace Eigen #endif // EIGEN_ARITHMETIC_SEQUENCE_H RcppEigen/inst/include/Eigen/src/Core/Visitor.h0000644000176200001440000002733514567757725021107 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_VISITOR_H #define EIGEN_VISITOR_H namespace Eigen { namespace internal { template struct visitor_impl { enum { col = (UnrollCount-1) / Derived::RowsAtCompileTime, row = (UnrollCount-1) % Derived::RowsAtCompileTime }; EIGEN_DEVICE_FUNC static inline void run(const Derived &mat, Visitor& visitor) { visitor_impl::run(mat, visitor); visitor(mat.coeff(row, col), row, col); } }; template struct visitor_impl { EIGEN_DEVICE_FUNC static inline void run(const Derived &mat, Visitor& visitor) { return visitor.init(mat.coeff(0, 0), 0, 0); } }; // This specialization enables visitors on empty matrices at compile-time template struct visitor_impl { EIGEN_DEVICE_FUNC static inline void run(const Derived &/*mat*/, Visitor& /*visitor*/) {} }; template struct visitor_impl { EIGEN_DEVICE_FUNC static inline void run(const Derived& mat, Visitor& visitor) { visitor.init(mat.coeff(0,0), 0, 0); for(Index i = 1; i < mat.rows(); ++i) visitor(mat.coeff(i, 0), i, 0); for(Index j = 1; j < mat.cols(); ++j) for(Index i = 0; i < mat.rows(); ++i) visitor(mat.coeff(i, j), i, j); } }; // evaluator adaptor template class visitor_evaluator { public: EIGEN_DEVICE_FUNC explicit visitor_evaluator(const XprType &xpr) : m_evaluator(xpr), m_xpr(xpr) {} typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; enum { RowsAtCompileTime = XprType::RowsAtCompileTime, CoeffReadCost = internal::evaluator::CoeffReadCost }; EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_xpr.rows(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_xpr.cols(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index size() const EIGEN_NOEXCEPT { return m_xpr.size(); } EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index row, Index col) const { return m_evaluator.coeff(row, col); } protected: internal::evaluator m_evaluator; const XprType &m_xpr; }; } // end namespace internal /** Applies the visitor \a visitor to the whole coefficients of the matrix or vector. * * The template parameter \a Visitor is the type of the visitor and provides the following interface: * \code * struct MyVisitor { * // called for the first coefficient * void init(const Scalar& value, Index i, Index j); * // called for all other coefficients * void operator() (const Scalar& value, Index i, Index j); * }; * \endcode * * \note compared to one or two \em for \em loops, visitors offer automatic * unrolling for small fixed size matrix. * * \note if the matrix is empty, then the visitor is left unchanged. * * \sa minCoeff(Index*,Index*), maxCoeff(Index*,Index*), DenseBase::redux() */ template template EIGEN_DEVICE_FUNC void DenseBase::visit(Visitor& visitor) const { if(size()==0) return; typedef typename internal::visitor_evaluator ThisEvaluator; ThisEvaluator thisEval(derived()); enum { unroll = SizeAtCompileTime != Dynamic && SizeAtCompileTime * int(ThisEvaluator::CoeffReadCost) + (SizeAtCompileTime-1) * int(internal::functor_traits::Cost) <= EIGEN_UNROLLING_LIMIT }; return internal::visitor_impl::run(thisEval, visitor); } namespace internal { /** \internal * \brief Base class to implement min and max visitors */ template struct coeff_visitor { // default initialization to avoid countless invalid maybe-uninitialized warnings by gcc EIGEN_DEVICE_FUNC coeff_visitor() : row(-1), col(-1), res(0) {} typedef typename Derived::Scalar Scalar; Index row, col; Scalar res; EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index i, Index j) { res = value; row = i; col = j; } }; /** \internal * \brief Visitor computing the min coefficient with its value and coordinates * * \sa DenseBase::minCoeff(Index*, Index*) */ template struct min_coeff_visitor : coeff_visitor { typedef typename Derived::Scalar Scalar; EIGEN_DEVICE_FUNC void operator() (const Scalar& value, Index i, Index j) { if(value < this->res) { this->res = value; this->row = i; this->col = j; } } }; template struct min_coeff_visitor : coeff_visitor { typedef typename Derived::Scalar Scalar; EIGEN_DEVICE_FUNC void operator() (const Scalar& value, Index i, Index j) { if((numext::isnan)(this->res) || (!(numext::isnan)(value) && value < this->res)) { this->res = value; this->row = i; this->col = j; } } }; template struct min_coeff_visitor : coeff_visitor { typedef typename Derived::Scalar Scalar; EIGEN_DEVICE_FUNC void operator() (const Scalar& value, Index i, Index j) { if((numext::isnan)(value) || value < this->res) { this->res = value; this->row = i; this->col = j; } } }; template struct functor_traits > { enum { Cost = NumTraits::AddCost }; }; /** \internal * \brief Visitor computing the max coefficient with its value and coordinates * * \sa DenseBase::maxCoeff(Index*, Index*) */ template struct max_coeff_visitor : coeff_visitor { typedef typename Derived::Scalar Scalar; EIGEN_DEVICE_FUNC void operator() (const Scalar& value, Index i, Index j) { if(value > this->res) { this->res = value; this->row = i; this->col = j; } } }; template struct max_coeff_visitor : coeff_visitor { typedef typename Derived::Scalar Scalar; EIGEN_DEVICE_FUNC void operator() (const Scalar& value, Index i, Index j) { if((numext::isnan)(this->res) || (!(numext::isnan)(value) && value > this->res)) { this->res = value; this->row = i; this->col = j; } } }; template struct max_coeff_visitor : coeff_visitor { typedef typename Derived::Scalar Scalar; EIGEN_DEVICE_FUNC void operator() (const Scalar& value, Index i, Index j) { if((numext::isnan)(value) || value > this->res) { this->res = value; this->row = i; this->col = j; } } }; template struct functor_traits > { enum { Cost = NumTraits::AddCost }; }; } // end namespace internal /** \fn DenseBase::minCoeff(IndexType* rowId, IndexType* colId) const * \returns the minimum of all coefficients of *this and puts in *row and *col its location. * * In case \c *this contains NaN, NaNPropagation determines the behavior: * NaNPropagation == PropagateFast : undefined * NaNPropagation == PropagateNaN : result is NaN * NaNPropagation == PropagateNumbers : result is maximum of elements that are not NaN * \warning the matrix must be not empty, otherwise an assertion is triggered. * * \sa DenseBase::minCoeff(Index*), DenseBase::maxCoeff(Index*,Index*), DenseBase::visit(), DenseBase::minCoeff() */ template template EIGEN_DEVICE_FUNC typename internal::traits::Scalar DenseBase::minCoeff(IndexType* rowId, IndexType* colId) const { eigen_assert(this->rows()>0 && this->cols()>0 && "you are using an empty matrix"); internal::min_coeff_visitor minVisitor; this->visit(minVisitor); *rowId = minVisitor.row; if (colId) *colId = minVisitor.col; return minVisitor.res; } /** \returns the minimum of all coefficients of *this and puts in *index its location. * * In case \c *this contains NaN, NaNPropagation determines the behavior: * NaNPropagation == PropagateFast : undefined * NaNPropagation == PropagateNaN : result is NaN * NaNPropagation == PropagateNumbers : result is maximum of elements that are not NaN * \warning the matrix must be not empty, otherwise an assertion is triggered. * * \sa DenseBase::minCoeff(IndexType*,IndexType*), DenseBase::maxCoeff(IndexType*,IndexType*), DenseBase::visit(), DenseBase::minCoeff() */ template template EIGEN_DEVICE_FUNC typename internal::traits::Scalar DenseBase::minCoeff(IndexType* index) const { eigen_assert(this->rows()>0 && this->cols()>0 && "you are using an empty matrix"); EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) internal::min_coeff_visitor minVisitor; this->visit(minVisitor); *index = IndexType((RowsAtCompileTime==1) ? minVisitor.col : minVisitor.row); return minVisitor.res; } /** \fn DenseBase::maxCoeff(IndexType* rowId, IndexType* colId) const * \returns the maximum of all coefficients of *this and puts in *row and *col its location. * * In case \c *this contains NaN, NaNPropagation determines the behavior: * NaNPropagation == PropagateFast : undefined * NaNPropagation == PropagateNaN : result is NaN * NaNPropagation == PropagateNumbers : result is maximum of elements that are not NaN * \warning the matrix must be not empty, otherwise an assertion is triggered. * * \sa DenseBase::minCoeff(IndexType*,IndexType*), DenseBase::visit(), DenseBase::maxCoeff() */ template template EIGEN_DEVICE_FUNC typename internal::traits::Scalar DenseBase::maxCoeff(IndexType* rowPtr, IndexType* colPtr) const { eigen_assert(this->rows()>0 && this->cols()>0 && "you are using an empty matrix"); internal::max_coeff_visitor maxVisitor; this->visit(maxVisitor); *rowPtr = maxVisitor.row; if (colPtr) *colPtr = maxVisitor.col; return maxVisitor.res; } /** \returns the maximum of all coefficients of *this and puts in *index its location. * * In case \c *this contains NaN, NaNPropagation determines the behavior: * NaNPropagation == PropagateFast : undefined * NaNPropagation == PropagateNaN : result is NaN * NaNPropagation == PropagateNumbers : result is maximum of elements that are not NaN * \warning the matrix must be not empty, otherwise an assertion is triggered. * * \sa DenseBase::maxCoeff(IndexType*,IndexType*), DenseBase::minCoeff(IndexType*,IndexType*), DenseBase::visitor(), DenseBase::maxCoeff() */ template template EIGEN_DEVICE_FUNC typename internal::traits::Scalar DenseBase::maxCoeff(IndexType* index) const { eigen_assert(this->rows()>0 && this->cols()>0 && "you are using an empty matrix"); EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) internal::max_coeff_visitor maxVisitor; this->visit(maxVisitor); *index = (RowsAtCompileTime==1) ? maxVisitor.col : maxVisitor.row; return maxVisitor.res; } } // end namespace Eigen #endif // EIGEN_VISITOR_H RcppEigen/inst/include/Eigen/src/Core/VectorBlock.h0000644000176200001440000000664014567757725021661 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2010 Gael Guennebaud // Copyright (C) 2006-2008 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_VECTORBLOCK_H #define EIGEN_VECTORBLOCK_H namespace Eigen { namespace internal { template struct traits > : public traits::Flags & RowMajorBit ? 1 : Size, traits::Flags & RowMajorBit ? Size : 1> > { }; } /** \class VectorBlock * \ingroup Core_Module * * \brief Expression of a fixed-size or dynamic-size sub-vector * * \tparam VectorType the type of the object in which we are taking a sub-vector * \tparam Size size of the sub-vector we are taking at compile time (optional) * * This class represents an expression of either a fixed-size or dynamic-size sub-vector. * It is the return type of DenseBase::segment(Index,Index) and DenseBase::segment(Index) and * most of the time this is the only way it is used. * * However, if you want to directly manipulate sub-vector expressions, * for instance if you want to write a function returning such an expression, you * will need to use this class. * * Here is an example illustrating the dynamic case: * \include class_VectorBlock.cpp * Output: \verbinclude class_VectorBlock.out * * \note Even though this expression has dynamic size, in the case where \a VectorType * has fixed size, this expression inherits a fixed maximal size which means that evaluating * it does not cause a dynamic memory allocation. * * Here is an example illustrating the fixed-size case: * \include class_FixedVectorBlock.cpp * Output: \verbinclude class_FixedVectorBlock.out * * \sa class Block, DenseBase::segment(Index,Index,Index,Index), DenseBase::segment(Index,Index) */ template class VectorBlock : public Block::Flags & RowMajorBit ? 1 : Size, internal::traits::Flags & RowMajorBit ? Size : 1> { typedef Block::Flags & RowMajorBit ? 1 : Size, internal::traits::Flags & RowMajorBit ? Size : 1> Base; enum { IsColVector = !(internal::traits::Flags & RowMajorBit) }; public: EIGEN_DENSE_PUBLIC_INTERFACE(VectorBlock) using Base::operator=; /** Dynamic-size constructor */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE VectorBlock(VectorType& vector, Index start, Index size) : Base(vector, IsColVector ? start : 0, IsColVector ? 0 : start, IsColVector ? size : 1, IsColVector ? 1 : size) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(VectorBlock); } /** Fixed-size constructor */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE VectorBlock(VectorType& vector, Index start) : Base(vector, IsColVector ? start : 0, IsColVector ? 0 : start) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(VectorBlock); } }; } // end namespace Eigen #endif // EIGEN_VECTORBLOCK_H RcppEigen/inst/include/Eigen/src/Core/IndexedView.h0000644000176200001440000002262414567757725021657 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2017 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_INDEXED_VIEW_H #define EIGEN_INDEXED_VIEW_H namespace Eigen { namespace internal { template struct traits > : traits { enum { RowsAtCompileTime = int(array_size::value), ColsAtCompileTime = int(array_size::value), MaxRowsAtCompileTime = RowsAtCompileTime != Dynamic ? int(RowsAtCompileTime) : Dynamic, MaxColsAtCompileTime = ColsAtCompileTime != Dynamic ? int(ColsAtCompileTime) : Dynamic, XprTypeIsRowMajor = (int(traits::Flags)&RowMajorBit) != 0, IsRowMajor = (MaxRowsAtCompileTime==1&&MaxColsAtCompileTime!=1) ? 1 : (MaxColsAtCompileTime==1&&MaxRowsAtCompileTime!=1) ? 0 : XprTypeIsRowMajor, RowIncr = int(get_compile_time_incr::value), ColIncr = int(get_compile_time_incr::value), InnerIncr = IsRowMajor ? ColIncr : RowIncr, OuterIncr = IsRowMajor ? RowIncr : ColIncr, HasSameStorageOrderAsXprType = (IsRowMajor == XprTypeIsRowMajor), XprInnerStride = HasSameStorageOrderAsXprType ? int(inner_stride_at_compile_time::ret) : int(outer_stride_at_compile_time::ret), XprOuterstride = HasSameStorageOrderAsXprType ? int(outer_stride_at_compile_time::ret) : int(inner_stride_at_compile_time::ret), InnerSize = XprTypeIsRowMajor ? ColsAtCompileTime : RowsAtCompileTime, IsBlockAlike = InnerIncr==1 && OuterIncr==1, IsInnerPannel = HasSameStorageOrderAsXprType && is_same,typename conditional::type>::value, InnerStrideAtCompileTime = InnerIncr<0 || InnerIncr==DynamicIndex || XprInnerStride==Dynamic ? Dynamic : XprInnerStride * InnerIncr, OuterStrideAtCompileTime = OuterIncr<0 || OuterIncr==DynamicIndex || XprOuterstride==Dynamic ? Dynamic : XprOuterstride * OuterIncr, ReturnAsScalar = is_same::value && is_same::value, ReturnAsBlock = (!ReturnAsScalar) && IsBlockAlike, ReturnAsIndexedView = (!ReturnAsScalar) && (!ReturnAsBlock), // FIXME we deal with compile-time strides if and only if we have DirectAccessBit flag, // but this is too strict regarding negative strides... DirectAccessMask = (int(InnerIncr)!=UndefinedIncr && int(OuterIncr)!=UndefinedIncr && InnerIncr>=0 && OuterIncr>=0) ? DirectAccessBit : 0, FlagsRowMajorBit = IsRowMajor ? RowMajorBit : 0, FlagsLvalueBit = is_lvalue::value ? LvalueBit : 0, FlagsLinearAccessBit = (RowsAtCompileTime == 1 || ColsAtCompileTime == 1) ? LinearAccessBit : 0, Flags = (traits::Flags & (HereditaryBits | DirectAccessMask )) | FlagsLvalueBit | FlagsRowMajorBit | FlagsLinearAccessBit }; typedef Block BlockType; }; } template class IndexedViewImpl; /** \class IndexedView * \ingroup Core_Module * * \brief Expression of a non-sequential sub-matrix defined by arbitrary sequences of row and column indices * * \tparam XprType the type of the expression in which we are taking the intersections of sub-rows and sub-columns * \tparam RowIndices the type of the object defining the sequence of row indices * \tparam ColIndices the type of the object defining the sequence of column indices * * This class represents an expression of a sub-matrix (or sub-vector) defined as the intersection * of sub-sets of rows and columns, that are themself defined by generic sequences of row indices \f$ \{r_0,r_1,..r_{m-1}\} \f$ * and column indices \f$ \{c_0,c_1,..c_{n-1} \}\f$. Let \f$ A \f$ be the nested matrix, then the resulting matrix \f$ B \f$ has \c m * rows and \c n columns, and its entries are given by: \f$ B(i,j) = A(r_i,c_j) \f$. * * The \c RowIndices and \c ColIndices types must be compatible with the following API: * \code * operator[](Index) const; * Index size() const; * \endcode * * Typical supported types thus include: * - std::vector * - std::valarray * - std::array * - Plain C arrays: int[N] * - Eigen::ArrayXi * - decltype(ArrayXi::LinSpaced(...)) * - Any view/expressions of the previous types * - Eigen::ArithmeticSequence * - Eigen::internal::AllRange (helper for Eigen::all) * - Eigen::internal::SingleRange (helper for single index) * - etc. * * In typical usages of %Eigen, this class should never be used directly. It is the return type of * DenseBase::operator()(const RowIndices&, const ColIndices&). * * \sa class Block */ template class IndexedView : public IndexedViewImpl::StorageKind> { public: typedef typename IndexedViewImpl::StorageKind>::Base Base; EIGEN_GENERIC_PUBLIC_INTERFACE(IndexedView) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(IndexedView) typedef typename internal::ref_selector::non_const_type MatrixTypeNested; typedef typename internal::remove_all::type NestedExpression; template IndexedView(XprType& xpr, const T0& rowIndices, const T1& colIndices) : m_xpr(xpr), m_rowIndices(rowIndices), m_colIndices(colIndices) {} /** \returns number of rows */ Index rows() const { return internal::size(m_rowIndices); } /** \returns number of columns */ Index cols() const { return internal::size(m_colIndices); } /** \returns the nested expression */ const typename internal::remove_all::type& nestedExpression() const { return m_xpr; } /** \returns the nested expression */ typename internal::remove_reference::type& nestedExpression() { return m_xpr; } /** \returns a const reference to the object storing/generating the row indices */ const RowIndices& rowIndices() const { return m_rowIndices; } /** \returns a const reference to the object storing/generating the column indices */ const ColIndices& colIndices() const { return m_colIndices; } protected: MatrixTypeNested m_xpr; RowIndices m_rowIndices; ColIndices m_colIndices; }; // Generic API dispatcher template class IndexedViewImpl : public internal::generic_xpr_base >::type { public: typedef typename internal::generic_xpr_base >::type Base; }; namespace internal { template struct unary_evaluator, IndexBased> : evaluator_base > { typedef IndexedView XprType; enum { CoeffReadCost = evaluator::CoeffReadCost /* TODO + cost of row/col index */, FlagsLinearAccessBit = (traits::RowsAtCompileTime == 1 || traits::ColsAtCompileTime == 1) ? LinearAccessBit : 0, FlagsRowMajorBit = traits::FlagsRowMajorBit, Flags = (evaluator::Flags & (HereditaryBits & ~RowMajorBit /*| LinearAccessBit | DirectAccessBit*/)) | FlagsLinearAccessBit | FlagsRowMajorBit, Alignment = 0 }; EIGEN_DEVICE_FUNC explicit unary_evaluator(const XprType& xpr) : m_argImpl(xpr.nestedExpression()), m_xpr(xpr) { EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { return m_argImpl.coeff(m_xpr.rowIndices()[row], m_xpr.colIndices()[col]); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { return m_argImpl.coeffRef(m_xpr.rowIndices()[row], m_xpr.colIndices()[col]); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { EIGEN_STATIC_ASSERT_LVALUE(XprType) Index row = XprType::RowsAtCompileTime == 1 ? 0 : index; Index col = XprType::RowsAtCompileTime == 1 ? index : 0; return m_argImpl.coeffRef( m_xpr.rowIndices()[row], m_xpr.colIndices()[col]); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeffRef(Index index) const { Index row = XprType::RowsAtCompileTime == 1 ? 0 : index; Index col = XprType::RowsAtCompileTime == 1 ? index : 0; return m_argImpl.coeffRef( m_xpr.rowIndices()[row], m_xpr.colIndices()[col]); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CoeffReturnType coeff(Index index) const { Index row = XprType::RowsAtCompileTime == 1 ? 0 : index; Index col = XprType::RowsAtCompileTime == 1 ? index : 0; return m_argImpl.coeff( m_xpr.rowIndices()[row], m_xpr.colIndices()[col]); } protected: evaluator m_argImpl; const XprType& m_xpr; }; } // end namespace internal } // end namespace Eigen #endif // EIGEN_INDEXED_VIEW_H RcppEigen/inst/include/Eigen/src/Core/CommaInitializer.h0000644000176200001440000001353514567757725022705 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008 Gael Guennebaud // Copyright (C) 2006-2008 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_COMMAINITIALIZER_H #define EIGEN_COMMAINITIALIZER_H namespace Eigen { /** \class CommaInitializer * \ingroup Core_Module * * \brief Helper class used by the comma initializer operator * * This class is internally used to implement the comma initializer feature. It is * the return type of MatrixBase::operator<<, and most of the time this is the only * way it is used. * * \sa \blank \ref MatrixBaseCommaInitRef "MatrixBase::operator<<", CommaInitializer::finished() */ template struct CommaInitializer { typedef typename XprType::Scalar Scalar; EIGEN_DEVICE_FUNC inline CommaInitializer(XprType& xpr, const Scalar& s) : m_xpr(xpr), m_row(0), m_col(1), m_currentBlockRows(1) { eigen_assert(m_xpr.rows() > 0 && m_xpr.cols() > 0 && "Cannot comma-initialize a 0x0 matrix (operator<<)"); m_xpr.coeffRef(0,0) = s; } template EIGEN_DEVICE_FUNC inline CommaInitializer(XprType& xpr, const DenseBase& other) : m_xpr(xpr), m_row(0), m_col(other.cols()), m_currentBlockRows(other.rows()) { eigen_assert(m_xpr.rows() >= other.rows() && m_xpr.cols() >= other.cols() && "Cannot comma-initialize a 0x0 matrix (operator<<)"); m_xpr.block(0, 0, other.rows(), other.cols()) = other; } /* Copy/Move constructor which transfers ownership. This is crucial in * absence of return value optimization to avoid assertions during destruction. */ // FIXME in C++11 mode this could be replaced by a proper RValue constructor EIGEN_DEVICE_FUNC inline CommaInitializer(const CommaInitializer& o) : m_xpr(o.m_xpr), m_row(o.m_row), m_col(o.m_col), m_currentBlockRows(o.m_currentBlockRows) { // Mark original object as finished. In absence of R-value references we need to const_cast: const_cast(o).m_row = m_xpr.rows(); const_cast(o).m_col = m_xpr.cols(); const_cast(o).m_currentBlockRows = 0; } /* inserts a scalar value in the target matrix */ EIGEN_DEVICE_FUNC CommaInitializer& operator,(const Scalar& s) { if (m_col==m_xpr.cols()) { m_row+=m_currentBlockRows; m_col = 0; m_currentBlockRows = 1; eigen_assert(m_row EIGEN_DEVICE_FUNC CommaInitializer& operator,(const DenseBase& other) { if (m_col==m_xpr.cols() && (other.cols()!=0 || other.rows()!=m_currentBlockRows)) { m_row+=m_currentBlockRows; m_col = 0; m_currentBlockRows = other.rows(); eigen_assert(m_row+m_currentBlockRows<=m_xpr.rows() && "Too many rows passed to comma initializer (operator<<)"); } eigen_assert((m_col + other.cols() <= m_xpr.cols()) && "Too many coefficients passed to comma initializer (operator<<)"); eigen_assert(m_currentBlockRows==other.rows()); m_xpr.template block (m_row, m_col, other.rows(), other.cols()) = other; m_col += other.cols(); return *this; } EIGEN_DEVICE_FUNC inline ~CommaInitializer() #if defined VERIFY_RAISES_ASSERT && (!defined EIGEN_NO_ASSERTION_CHECKING) && defined EIGEN_EXCEPTIONS EIGEN_EXCEPTION_SPEC(Eigen::eigen_assert_exception) #endif { finished(); } /** \returns the built matrix once all its coefficients have been set. * Calling finished is 100% optional. Its purpose is to write expressions * like this: * \code * quaternion.fromRotationMatrix((Matrix3f() << axis0, axis1, axis2).finished()); * \endcode */ EIGEN_DEVICE_FUNC inline XprType& finished() { eigen_assert(((m_row+m_currentBlockRows) == m_xpr.rows() || m_xpr.cols() == 0) && m_col == m_xpr.cols() && "Too few coefficients passed to comma initializer (operator<<)"); return m_xpr; } XprType& m_xpr; // target expression Index m_row; // current row id Index m_col; // current col id Index m_currentBlockRows; // current block height }; /** \anchor MatrixBaseCommaInitRef * Convenient operator to set the coefficients of a matrix. * * The coefficients must be provided in a row major order and exactly match * the size of the matrix. Otherwise an assertion is raised. * * Example: \include MatrixBase_set.cpp * Output: \verbinclude MatrixBase_set.out * * \note According the c++ standard, the argument expressions of this comma initializer are evaluated in arbitrary order. * * \sa CommaInitializer::finished(), class CommaInitializer */ template EIGEN_DEVICE_FUNC inline CommaInitializer DenseBase::operator<< (const Scalar& s) { return CommaInitializer(*static_cast(this), s); } /** \sa operator<<(const Scalar&) */ template template EIGEN_DEVICE_FUNC inline CommaInitializer DenseBase::operator<<(const DenseBase& other) { return CommaInitializer(*static_cast(this), other); } } // end namespace Eigen #endif // EIGEN_COMMAINITIALIZER_H RcppEigen/inst/include/Eigen/src/Core/CoreIterators.h0000644000176200001440000001121114567757725022217 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2014 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_COREITERATORS_H #define EIGEN_COREITERATORS_H namespace Eigen { /* This file contains the respective InnerIterator definition of the expressions defined in Eigen/Core */ namespace internal { template class inner_iterator_selector; } /** \class InnerIterator * \brief An InnerIterator allows to loop over the element of any matrix expression. * * \warning To be used with care because an evaluator is constructed every time an InnerIterator iterator is constructed. * * TODO: add a usage example */ template class InnerIterator { protected: typedef internal::inner_iterator_selector::Kind> IteratorType; typedef internal::evaluator EvaluatorType; typedef typename internal::traits::Scalar Scalar; public: /** Construct an iterator over the \a outerId -th row or column of \a xpr */ InnerIterator(const XprType &xpr, const Index &outerId) : m_eval(xpr), m_iter(m_eval, outerId, xpr.innerSize()) {} /// \returns the value of the current coefficient. EIGEN_STRONG_INLINE Scalar value() const { return m_iter.value(); } /** Increment the iterator \c *this to the next non-zero coefficient. * Explicit zeros are not skipped over. To skip explicit zeros, see class SparseView */ EIGEN_STRONG_INLINE InnerIterator& operator++() { m_iter.operator++(); return *this; } EIGEN_STRONG_INLINE InnerIterator& operator+=(Index i) { m_iter.operator+=(i); return *this; } EIGEN_STRONG_INLINE InnerIterator operator+(Index i) { InnerIterator result(*this); result+=i; return result; } /// \returns the column or row index of the current coefficient. EIGEN_STRONG_INLINE Index index() const { return m_iter.index(); } /// \returns the row index of the current coefficient. EIGEN_STRONG_INLINE Index row() const { return m_iter.row(); } /// \returns the column index of the current coefficient. EIGEN_STRONG_INLINE Index col() const { return m_iter.col(); } /// \returns \c true if the iterator \c *this still references a valid coefficient. EIGEN_STRONG_INLINE operator bool() const { return m_iter; } protected: EvaluatorType m_eval; IteratorType m_iter; private: // If you get here, then you're not using the right InnerIterator type, e.g.: // SparseMatrix A; // SparseMatrix::InnerIterator it(A,0); template InnerIterator(const EigenBase&,Index outer); }; namespace internal { // Generic inner iterator implementation for dense objects template class inner_iterator_selector { protected: typedef evaluator EvaluatorType; typedef typename traits::Scalar Scalar; enum { IsRowMajor = (XprType::Flags&RowMajorBit)==RowMajorBit }; public: EIGEN_STRONG_INLINE inner_iterator_selector(const EvaluatorType &eval, const Index &outerId, const Index &innerSize) : m_eval(eval), m_inner(0), m_outer(outerId), m_end(innerSize) {} EIGEN_STRONG_INLINE Scalar value() const { return (IsRowMajor) ? m_eval.coeff(m_outer, m_inner) : m_eval.coeff(m_inner, m_outer); } EIGEN_STRONG_INLINE inner_iterator_selector& operator++() { m_inner++; return *this; } EIGEN_STRONG_INLINE Index index() const { return m_inner; } inline Index row() const { return IsRowMajor ? m_outer : index(); } inline Index col() const { return IsRowMajor ? index() : m_outer; } EIGEN_STRONG_INLINE operator bool() const { return m_inner < m_end && m_inner>=0; } protected: const EvaluatorType& m_eval; Index m_inner; const Index m_outer; const Index m_end; }; // For iterator-based evaluator, inner-iterator is already implemented as // evaluator<>::InnerIterator template class inner_iterator_selector : public evaluator::InnerIterator { protected: typedef typename evaluator::InnerIterator Base; typedef evaluator EvaluatorType; public: EIGEN_STRONG_INLINE inner_iterator_selector(const EvaluatorType &eval, const Index &outerId, const Index &/*innerSize*/) : Base(eval, outerId) {} }; } // end namespace internal } // end namespace Eigen #endif // EIGEN_COREITERATORS_H RcppEigen/inst/include/Eigen/src/Core/Swap.h0000644000176200001440000000531514567757725020354 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2006-2008 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_SWAP_H #define EIGEN_SWAP_H namespace Eigen { namespace internal { // Overload default assignPacket behavior for swapping them template class generic_dense_assignment_kernel, Specialized> : public generic_dense_assignment_kernel, BuiltIn> { protected: typedef generic_dense_assignment_kernel, BuiltIn> Base; using Base::m_dst; using Base::m_src; using Base::m_functor; public: typedef typename Base::Scalar Scalar; typedef typename Base::DstXprType DstXprType; typedef swap_assign_op Functor; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE generic_dense_assignment_kernel(DstEvaluatorTypeT &dst, const SrcEvaluatorTypeT &src, const Functor &func, DstXprType& dstExpr) : Base(dst, src, func, dstExpr) {} template EIGEN_STRONG_INLINE void assignPacket(Index row, Index col) { PacketType tmp = m_src.template packet(row,col); const_cast(m_src).template writePacket(row,col, m_dst.template packet(row,col)); m_dst.template writePacket(row,col,tmp); } template EIGEN_STRONG_INLINE void assignPacket(Index index) { PacketType tmp = m_src.template packet(index); const_cast(m_src).template writePacket(index, m_dst.template packet(index)); m_dst.template writePacket(index,tmp); } // TODO find a simple way not to have to copy/paste this function from generic_dense_assignment_kernel, by simple I mean no CRTP (Gael) template EIGEN_STRONG_INLINE void assignPacketByOuterInner(Index outer, Index inner) { Index row = Base::rowIndexByOuterInner(outer, inner); Index col = Base::colIndexByOuterInner(outer, inner); assignPacket(row, col); } }; } // namespace internal } // end namespace Eigen #endif // EIGEN_SWAP_H RcppEigen/inst/include/Eigen/src/Core/util/0000755000176200001440000000000014567757725020242 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/util/Macros.h0000644000176200001440000014725514567757725021655 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2015 Gael Guennebaud // Copyright (C) 2006-2008 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_MACROS_H #define EIGEN_MACROS_H //------------------------------------------------------------------------------------------ // Eigen version and basic defaults //------------------------------------------------------------------------------------------ #define EIGEN_WORLD_VERSION 3 #define EIGEN_MAJOR_VERSION 4 #define EIGEN_MINOR_VERSION 0 #define EIGEN_VERSION_AT_LEAST(x,y,z) (EIGEN_WORLD_VERSION>x || (EIGEN_WORLD_VERSION>=x && \ (EIGEN_MAJOR_VERSION>y || (EIGEN_MAJOR_VERSION>=y && \ EIGEN_MINOR_VERSION>=z)))) #ifdef EIGEN_DEFAULT_TO_ROW_MAJOR #define EIGEN_DEFAULT_MATRIX_STORAGE_ORDER_OPTION Eigen::RowMajor #else #define EIGEN_DEFAULT_MATRIX_STORAGE_ORDER_OPTION Eigen::ColMajor #endif #ifndef EIGEN_DEFAULT_DENSE_INDEX_TYPE #define EIGEN_DEFAULT_DENSE_INDEX_TYPE std::ptrdiff_t #endif // Upperbound on the C++ version to use. // Expected values are 03, 11, 14, 17, etc. // By default, let's use an arbitrarily large C++ version. #ifndef EIGEN_MAX_CPP_VER #define EIGEN_MAX_CPP_VER 99 #endif /** Allows to disable some optimizations which might affect the accuracy of the result. * Such optimization are enabled by default, and set EIGEN_FAST_MATH to 0 to disable them. * They currently include: * - single precision ArrayBase::sin() and ArrayBase::cos() for SSE and AVX vectorization. */ #ifndef EIGEN_FAST_MATH #define EIGEN_FAST_MATH 1 #endif #ifndef EIGEN_STACK_ALLOCATION_LIMIT // 131072 == 128 KB #define EIGEN_STACK_ALLOCATION_LIMIT 131072 #endif //------------------------------------------------------------------------------------------ // Compiler identification, EIGEN_COMP_* //------------------------------------------------------------------------------------------ /// \internal EIGEN_COMP_GNUC set to 1 for all compilers compatible with GCC #ifdef __GNUC__ #define EIGEN_COMP_GNUC (__GNUC__*10+__GNUC_MINOR__) #else #define EIGEN_COMP_GNUC 0 #endif /// \internal EIGEN_COMP_CLANG set to major+minor version (e.g., 307 for clang 3.7) if the compiler is clang #if defined(__clang__) #define EIGEN_COMP_CLANG (__clang_major__*100+__clang_minor__) #else #define EIGEN_COMP_CLANG 0 #endif /// \internal EIGEN_COMP_CASTXML set to 1 if being preprocessed by CastXML #if defined(__castxml__) #define EIGEN_COMP_CASTXML 1 #else #define EIGEN_COMP_CASTXML 0 #endif /// \internal EIGEN_COMP_LLVM set to 1 if the compiler backend is llvm #if defined(__llvm__) #define EIGEN_COMP_LLVM 1 #else #define EIGEN_COMP_LLVM 0 #endif /// \internal EIGEN_COMP_ICC set to __INTEL_COMPILER if the compiler is Intel compiler, 0 otherwise #if defined(__INTEL_COMPILER) #define EIGEN_COMP_ICC __INTEL_COMPILER #else #define EIGEN_COMP_ICC 0 #endif /// \internal EIGEN_COMP_MINGW set to 1 if the compiler is mingw #if defined(__MINGW32__) #define EIGEN_COMP_MINGW 1 #else #define EIGEN_COMP_MINGW 0 #endif /// \internal EIGEN_COMP_SUNCC set to 1 if the compiler is Solaris Studio #if defined(__SUNPRO_CC) #define EIGEN_COMP_SUNCC 1 #else #define EIGEN_COMP_SUNCC 0 #endif /// \internal EIGEN_COMP_MSVC set to _MSC_VER if the compiler is Microsoft Visual C++, 0 otherwise. #if defined(_MSC_VER) #define EIGEN_COMP_MSVC _MSC_VER #else #define EIGEN_COMP_MSVC 0 #endif #if defined(__NVCC__) #if defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 9) #define EIGEN_COMP_NVCC ((__CUDACC_VER_MAJOR__ * 10000) + (__CUDACC_VER_MINOR__ * 100)) #elif defined(__CUDACC_VER__) #define EIGEN_COMP_NVCC __CUDACC_VER__ #else #error "NVCC did not define compiler version." #endif #else #define EIGEN_COMP_NVCC 0 #endif // For the record, here is a table summarizing the possible values for EIGEN_COMP_MSVC: // name ver MSC_VER // 2008 9 1500 // 2010 10 1600 // 2012 11 1700 // 2013 12 1800 // 2015 14 1900 // "15" 15 1900 // 2017-14.1 15.0 1910 // 2017-14.11 15.3 1911 // 2017-14.12 15.5 1912 // 2017-14.13 15.6 1913 // 2017-14.14 15.7 1914 /// \internal EIGEN_COMP_MSVC_LANG set to _MSVC_LANG if the compiler is Microsoft Visual C++, 0 otherwise. #if defined(_MSVC_LANG) #define EIGEN_COMP_MSVC_LANG _MSVC_LANG #else #define EIGEN_COMP_MSVC_LANG 0 #endif // For the record, here is a table summarizing the possible values for EIGEN_COMP_MSVC_LANG: // MSVC option Standard MSVC_LANG // /std:c++14 (default as of VS 2019) C++14 201402L // /std:c++17 C++17 201703L // /std:c++latest >C++17 >201703L /// \internal EIGEN_COMP_MSVC_STRICT set to 1 if the compiler is really Microsoft Visual C++ and not ,e.g., ICC or clang-cl #if EIGEN_COMP_MSVC && !(EIGEN_COMP_ICC || EIGEN_COMP_LLVM || EIGEN_COMP_CLANG) #define EIGEN_COMP_MSVC_STRICT _MSC_VER #else #define EIGEN_COMP_MSVC_STRICT 0 #endif /// \internal EIGEN_COMP_IBM set to xlc version if the compiler is IBM XL C++ // XLC version // 3.1 0x0301 // 4.5 0x0405 // 5.0 0x0500 // 12.1 0x0C01 #if defined(__IBMCPP__) || defined(__xlc__) || defined(__ibmxl__) #define EIGEN_COMP_IBM __xlC__ #else #define EIGEN_COMP_IBM 0 #endif /// \internal EIGEN_COMP_PGI set to PGI version if the compiler is Portland Group Compiler #if defined(__PGI) #define EIGEN_COMP_PGI (__PGIC__*100+__PGIC_MINOR__) #else #define EIGEN_COMP_PGI 0 #endif /// \internal EIGEN_COMP_ARM set to 1 if the compiler is ARM Compiler #if defined(__CC_ARM) || defined(__ARMCC_VERSION) #define EIGEN_COMP_ARM 1 #else #define EIGEN_COMP_ARM 0 #endif /// \internal EIGEN_COMP_EMSCRIPTEN set to 1 if the compiler is Emscripten Compiler #if defined(__EMSCRIPTEN__) #define EIGEN_COMP_EMSCRIPTEN 1 #else #define EIGEN_COMP_EMSCRIPTEN 0 #endif /// \internal EIGEN_GNUC_STRICT set to 1 if the compiler is really GCC and not a compatible compiler (e.g., ICC, clang, mingw, etc.) #if EIGEN_COMP_GNUC && !(EIGEN_COMP_CLANG || EIGEN_COMP_ICC || EIGEN_COMP_MINGW || EIGEN_COMP_PGI || EIGEN_COMP_IBM || EIGEN_COMP_ARM || EIGEN_COMP_EMSCRIPTEN) #define EIGEN_COMP_GNUC_STRICT 1 #else #define EIGEN_COMP_GNUC_STRICT 0 #endif #if EIGEN_COMP_GNUC #define EIGEN_GNUC_AT_LEAST(x,y) ((__GNUC__==x && __GNUC_MINOR__>=y) || __GNUC__>x) #define EIGEN_GNUC_AT_MOST(x,y) ((__GNUC__==x && __GNUC_MINOR__<=y) || __GNUC__= 8 #define EIGEN_ARCH_ARMV8 1 #else #define EIGEN_ARCH_ARMV8 0 #endif /// \internal EIGEN_HAS_ARM64_FP16 set to 1 if the architecture provides an IEEE /// compliant Arm fp16 type #if EIGEN_ARCH_ARM64 #ifndef EIGEN_HAS_ARM64_FP16 #if defined(__ARM_FP16_FORMAT_IEEE) #define EIGEN_HAS_ARM64_FP16 1 #else #define EIGEN_HAS_ARM64_FP16 0 #endif #endif #endif /// \internal EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC set to 1 if the architecture /// supports Neon vector intrinsics for fp16. #if EIGEN_ARCH_ARM64 #ifndef EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) #define EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC 1 #else #define EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC 0 #endif #endif #endif /// \internal EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC set to 1 if the architecture /// supports Neon scalar intrinsics for fp16. #if EIGEN_ARCH_ARM64 #ifndef EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC #if defined(__ARM_FEATURE_FP16_SCALAR_ARITHMETIC) #define EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC 1 #endif #endif #endif /// \internal EIGEN_ARCH_MIPS set to 1 if the architecture is MIPS #if defined(__mips__) || defined(__mips) #define EIGEN_ARCH_MIPS 1 #else #define EIGEN_ARCH_MIPS 0 #endif /// \internal EIGEN_ARCH_SPARC set to 1 if the architecture is SPARC #if defined(__sparc__) || defined(__sparc) #define EIGEN_ARCH_SPARC 1 #else #define EIGEN_ARCH_SPARC 0 #endif /// \internal EIGEN_ARCH_IA64 set to 1 if the architecture is Intel Itanium #if defined(__ia64__) #define EIGEN_ARCH_IA64 1 #else #define EIGEN_ARCH_IA64 0 #endif /// \internal EIGEN_ARCH_PPC set to 1 if the architecture is PowerPC #if defined(__powerpc__) || defined(__ppc__) || defined(_M_PPC) #define EIGEN_ARCH_PPC 1 #else #define EIGEN_ARCH_PPC 0 #endif //------------------------------------------------------------------------------------------ // Operating system identification, EIGEN_OS_* //------------------------------------------------------------------------------------------ /// \internal EIGEN_OS_UNIX set to 1 if the OS is a unix variant #if defined(__unix__) || defined(__unix) #define EIGEN_OS_UNIX 1 #else #define EIGEN_OS_UNIX 0 #endif /// \internal EIGEN_OS_LINUX set to 1 if the OS is based on Linux kernel #if defined(__linux__) #define EIGEN_OS_LINUX 1 #else #define EIGEN_OS_LINUX 0 #endif /// \internal EIGEN_OS_ANDROID set to 1 if the OS is Android // note: ANDROID is defined when using ndk_build, __ANDROID__ is defined when using a standalone toolchain. #if defined(__ANDROID__) || defined(ANDROID) #define EIGEN_OS_ANDROID 1 #else #define EIGEN_OS_ANDROID 0 #endif /// \internal EIGEN_OS_GNULINUX set to 1 if the OS is GNU Linux and not Linux-based OS (e.g., not android) #if defined(__gnu_linux__) && !(EIGEN_OS_ANDROID) #define EIGEN_OS_GNULINUX 1 #else #define EIGEN_OS_GNULINUX 0 #endif /// \internal EIGEN_OS_BSD set to 1 if the OS is a BSD variant #if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__bsdi__) || defined(__DragonFly__) #define EIGEN_OS_BSD 1 #else #define EIGEN_OS_BSD 0 #endif /// \internal EIGEN_OS_MAC set to 1 if the OS is MacOS #if defined(__APPLE__) #define EIGEN_OS_MAC 1 #else #define EIGEN_OS_MAC 0 #endif /// \internal EIGEN_OS_QNX set to 1 if the OS is QNX #if defined(__QNX__) #define EIGEN_OS_QNX 1 #else #define EIGEN_OS_QNX 0 #endif /// \internal EIGEN_OS_WIN set to 1 if the OS is Windows based #if defined(_WIN32) #define EIGEN_OS_WIN 1 #else #define EIGEN_OS_WIN 0 #endif /// \internal EIGEN_OS_WIN64 set to 1 if the OS is Windows 64bits #if defined(_WIN64) #define EIGEN_OS_WIN64 1 #else #define EIGEN_OS_WIN64 0 #endif /// \internal EIGEN_OS_WINCE set to 1 if the OS is Windows CE #if defined(_WIN32_WCE) #define EIGEN_OS_WINCE 1 #else #define EIGEN_OS_WINCE 0 #endif /// \internal EIGEN_OS_CYGWIN set to 1 if the OS is Windows/Cygwin #if defined(__CYGWIN__) #define EIGEN_OS_CYGWIN 1 #else #define EIGEN_OS_CYGWIN 0 #endif /// \internal EIGEN_OS_WIN_STRICT set to 1 if the OS is really Windows and not some variants #if EIGEN_OS_WIN && !( EIGEN_OS_WINCE || EIGEN_OS_CYGWIN ) #define EIGEN_OS_WIN_STRICT 1 #else #define EIGEN_OS_WIN_STRICT 0 #endif /// \internal EIGEN_OS_SUN set to __SUNPRO_C if the OS is SUN // compiler solaris __SUNPRO_C // version studio // 5.7 10 0x570 // 5.8 11 0x580 // 5.9 12 0x590 // 5.10 12.1 0x5100 // 5.11 12.2 0x5110 // 5.12 12.3 0x5120 #if (defined(sun) || defined(__sun)) && !(defined(__SVR4) || defined(__svr4__)) #define EIGEN_OS_SUN __SUNPRO_C #else #define EIGEN_OS_SUN 0 #endif /// \internal EIGEN_OS_SOLARIS set to 1 if the OS is Solaris #if (defined(sun) || defined(__sun)) && (defined(__SVR4) || defined(__svr4__)) #define EIGEN_OS_SOLARIS 1 #else #define EIGEN_OS_SOLARIS 0 #endif //------------------------------------------------------------------------------------------ // Detect GPU compilers and architectures //------------------------------------------------------------------------------------------ // NVCC is not supported as the target platform for HIPCC // Note that this also makes EIGEN_CUDACC and EIGEN_HIPCC mutually exclusive #if defined(__NVCC__) && defined(__HIPCC__) #error "NVCC as the target platform for HIPCC is currently not supported." #endif #if defined(__CUDACC__) && !defined(EIGEN_NO_CUDA) // Means the compiler is either nvcc or clang with CUDA enabled #define EIGEN_CUDACC __CUDACC__ #endif #if defined(__CUDA_ARCH__) && !defined(EIGEN_NO_CUDA) // Means we are generating code for the device #define EIGEN_CUDA_ARCH __CUDA_ARCH__ #endif #if defined(EIGEN_CUDACC) #include #define EIGEN_CUDA_SDK_VER (CUDA_VERSION * 10) #else #define EIGEN_CUDA_SDK_VER 0 #endif #if defined(__HIPCC__) && !defined(EIGEN_NO_HIP) // Means the compiler is HIPCC (analogous to EIGEN_CUDACC, but for HIP) #define EIGEN_HIPCC __HIPCC__ // We need to include hip_runtime.h here because it pulls in // ++ hip_common.h which contains the define for __HIP_DEVICE_COMPILE__ // ++ host_defines.h which contains the defines for the __host__ and __device__ macros #include #if defined(__HIP_DEVICE_COMPILE__) // analogous to EIGEN_CUDA_ARCH, but for HIP #define EIGEN_HIP_DEVICE_COMPILE __HIP_DEVICE_COMPILE__ #endif // For HIP (ROCm 3.5 and higher), we need to explicitly set the launch_bounds attribute // value to 1024. The compiler assigns a default value of 256 when the attribute is not // specified. This results in failures on the HIP platform, for cases when a GPU kernel // without an explicit launch_bounds attribute is called with a threads_per_block value // greater than 256. // // This is a regression in functioanlity and is expected to be fixed within the next // couple of ROCm releases (compiler will go back to using 1024 value as the default) // // In the meantime, we will use a "only enabled for HIP" macro to set the launch_bounds // attribute. #define EIGEN_HIP_LAUNCH_BOUNDS_1024 __launch_bounds__(1024) #endif #if !defined(EIGEN_HIP_LAUNCH_BOUNDS_1024) #define EIGEN_HIP_LAUNCH_BOUNDS_1024 #endif // !defined(EIGEN_HIP_LAUNCH_BOUNDS_1024) // Unify CUDA/HIPCC #if defined(EIGEN_CUDACC) || defined(EIGEN_HIPCC) // // If either EIGEN_CUDACC or EIGEN_HIPCC is defined, then define EIGEN_GPUCC // #define EIGEN_GPUCC // // EIGEN_HIPCC implies the HIP compiler and is used to tweak Eigen code for use in HIP kernels // EIGEN_CUDACC implies the CUDA compiler and is used to tweak Eigen code for use in CUDA kernels // // In most cases the same tweaks are required to the Eigen code to enable in both the HIP and CUDA kernels. // For those cases, the corresponding code should be guarded with // #if defined(EIGEN_GPUCC) // instead of // #if defined(EIGEN_CUDACC) || defined(EIGEN_HIPCC) // // For cases where the tweak is specific to HIP, the code should be guarded with // #if defined(EIGEN_HIPCC) // // For cases where the tweak is specific to CUDA, the code should be guarded with // #if defined(EIGEN_CUDACC) // #endif #if defined(EIGEN_CUDA_ARCH) || defined(EIGEN_HIP_DEVICE_COMPILE) // // If either EIGEN_CUDA_ARCH or EIGEN_HIP_DEVICE_COMPILE is defined, then define EIGEN_GPU_COMPILE_PHASE // #define EIGEN_GPU_COMPILE_PHASE // // GPU compilers (HIPCC, NVCC) typically do two passes over the source code, // + one to compile the source for the "host" (ie CPU) // + another to compile the source for the "device" (ie. GPU) // // Code that needs to enabled only during the either the "host" or "device" compilation phase // needs to be guarded with a macro that indicates the current compilation phase // // EIGEN_HIP_DEVICE_COMPILE implies the device compilation phase in HIP // EIGEN_CUDA_ARCH implies the device compilation phase in CUDA // // In most cases, the "host" / "device" specific code is the same for both HIP and CUDA // For those cases, the code should be guarded with // #if defined(EIGEN_GPU_COMPILE_PHASE) // instead of // #if defined(EIGEN_CUDA_ARCH) || defined(EIGEN_HIP_DEVICE_COMPILE) // // For cases where the tweak is specific to HIP, the code should be guarded with // #if defined(EIGEN_HIP_DEVICE_COMPILE) // // For cases where the tweak is specific to CUDA, the code should be guarded with // #if defined(EIGEN_CUDA_ARCH) // #endif #if defined(EIGEN_USE_SYCL) && defined(__SYCL_DEVICE_ONLY__) // EIGEN_USE_SYCL is a user-defined macro while __SYCL_DEVICE_ONLY__ is a compiler-defined macro. // In most cases we want to check if both macros are defined which can be done using the define below. #define SYCL_DEVICE_ONLY #endif //------------------------------------------------------------------------------------------ // Detect Compiler/Architecture/OS specific features //------------------------------------------------------------------------------------------ #if EIGEN_GNUC_AT_MOST(4,3) && !EIGEN_COMP_CLANG // see bug 89 #define EIGEN_SAFE_TO_USE_STANDARD_ASSERT_MACRO 0 #else #define EIGEN_SAFE_TO_USE_STANDARD_ASSERT_MACRO 1 #endif // Cross compiler wrapper around LLVM's __has_builtin #ifdef __has_builtin # define EIGEN_HAS_BUILTIN(x) __has_builtin(x) #else # define EIGEN_HAS_BUILTIN(x) 0 #endif // A Clang feature extension to determine compiler features. // We use it to determine 'cxx_rvalue_references' #ifndef __has_feature # define __has_feature(x) 0 #endif // Some old compilers do not support template specializations like: // template void foo(const T x[N]); #if !( EIGEN_COMP_CLANG && ( (EIGEN_COMP_CLANG<309) \ || (defined(__apple_build_version__) && (__apple_build_version__ < 9000000))) \ || EIGEN_COMP_GNUC_STRICT && EIGEN_COMP_GNUC<49) #define EIGEN_HAS_STATIC_ARRAY_TEMPLATE 1 #else #define EIGEN_HAS_STATIC_ARRAY_TEMPLATE 0 #endif // The macro EIGEN_CPLUSPLUS is a replacement for __cplusplus/_MSVC_LANG that // works for both platforms, indicating the C++ standard version number. // // With MSVC, without defining /Zc:__cplusplus, the __cplusplus macro will // report 199711L regardless of the language standard specified via /std. // We need to rely on _MSVC_LANG instead, which is only available after // VS2015.3. #if EIGEN_COMP_MSVC_LANG > 0 #define EIGEN_CPLUSPLUS EIGEN_COMP_MSVC_LANG #elif EIGEN_COMP_MSVC >= 1900 #define EIGEN_CPLUSPLUS 201103L #elif defined(__cplusplus) #define EIGEN_CPLUSPLUS __cplusplus #else #define EIGEN_CPLUSPLUS 0 #endif // The macro EIGEN_COMP_CXXVER defines the c++ verson expected by the compiler. // For instance, if compiling with gcc and -std=c++17, then EIGEN_COMP_CXXVER // is defined to 17. #if EIGEN_CPLUSPLUS > 201703L #define EIGEN_COMP_CXXVER 20 #elif EIGEN_CPLUSPLUS > 201402L #define EIGEN_COMP_CXXVER 17 #elif EIGEN_CPLUSPLUS > 201103L #define EIGEN_COMP_CXXVER 14 #elif EIGEN_CPLUSPLUS >= 201103L #define EIGEN_COMP_CXXVER 11 #else #define EIGEN_COMP_CXXVER 03 #endif #ifndef EIGEN_HAS_CXX14_VARIABLE_TEMPLATES #if defined(__cpp_variable_templates) && __cpp_variable_templates >= 201304 && EIGEN_MAX_CPP_VER>=14 #define EIGEN_HAS_CXX14_VARIABLE_TEMPLATES 1 #else #define EIGEN_HAS_CXX14_VARIABLE_TEMPLATES 0 #endif #endif // The macros EIGEN_HAS_CXX?? defines a rough estimate of available c++ features // but in practice we should not rely on them but rather on the availabilty of // individual features as defined later. // This is why there is no EIGEN_HAS_CXX17. // FIXME: get rid of EIGEN_HAS_CXX14 and maybe even EIGEN_HAS_CXX11. #if EIGEN_MAX_CPP_VER>=11 && EIGEN_COMP_CXXVER>=11 #define EIGEN_HAS_CXX11 1 #else #define EIGEN_HAS_CXX11 0 #endif #if EIGEN_MAX_CPP_VER>=14 && EIGEN_COMP_CXXVER>=14 #define EIGEN_HAS_CXX14 1 #else #define EIGEN_HAS_CXX14 0 #endif // Do we support r-value references? #ifndef EIGEN_HAS_RVALUE_REFERENCES #if EIGEN_MAX_CPP_VER>=11 && \ (__has_feature(cxx_rvalue_references) || \ (EIGEN_COMP_CXXVER >= 11) || (EIGEN_COMP_MSVC >= 1600)) #define EIGEN_HAS_RVALUE_REFERENCES 1 #else #define EIGEN_HAS_RVALUE_REFERENCES 0 #endif #endif // Does the compiler support C99? // Need to include to make sure _GLIBCXX_USE_C99 gets defined #include #ifndef EIGEN_HAS_C99_MATH #if EIGEN_MAX_CPP_VER>=11 && \ ((defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901)) \ || (defined(__GNUC__) && defined(_GLIBCXX_USE_C99)) \ || (defined(_LIBCPP_VERSION) && !defined(_MSC_VER)) \ || (EIGEN_COMP_MSVC >= 1900) || defined(SYCL_DEVICE_ONLY)) #define EIGEN_HAS_C99_MATH 1 #else #define EIGEN_HAS_C99_MATH 0 #endif #endif // Does the compiler support result_of? // result_of was deprecated in c++17 and removed in c++ 20 #ifndef EIGEN_HAS_STD_RESULT_OF #if EIGEN_HAS_CXX11 && EIGEN_COMP_CXXVER < 17 #define EIGEN_HAS_STD_RESULT_OF 1 #else #define EIGEN_HAS_STD_RESULT_OF 0 #endif #endif // Does the compiler support std::hash? #ifndef EIGEN_HAS_STD_HASH // The std::hash struct is defined in C++11 but is not labelled as a __device__ // function and is not constexpr, so cannot be used on device. #if EIGEN_HAS_CXX11 && !defined(EIGEN_GPU_COMPILE_PHASE) #define EIGEN_HAS_STD_HASH 1 #else #define EIGEN_HAS_STD_HASH 0 #endif #endif // EIGEN_HAS_STD_HASH #ifndef EIGEN_HAS_STD_INVOKE_RESULT #if EIGEN_MAX_CPP_VER >= 17 && EIGEN_COMP_CXXVER >= 17 #define EIGEN_HAS_STD_INVOKE_RESULT 1 #else #define EIGEN_HAS_STD_INVOKE_RESULT 0 #endif #endif #ifndef EIGEN_HAS_ALIGNAS #if EIGEN_MAX_CPP_VER>=11 && EIGEN_HAS_CXX11 && \ ( __has_feature(cxx_alignas) \ || EIGEN_HAS_CXX14 \ || (EIGEN_COMP_MSVC >= 1800) \ || (EIGEN_GNUC_AT_LEAST(4,8)) \ || (EIGEN_COMP_CLANG>=305) \ || (EIGEN_COMP_ICC>=1500) \ || (EIGEN_COMP_PGI>=1500) \ || (EIGEN_COMP_SUNCC>=0x5130)) #define EIGEN_HAS_ALIGNAS 1 #else #define EIGEN_HAS_ALIGNAS 0 #endif #endif // Does the compiler support type_traits? // - full support of type traits was added only to GCC 5.1.0. // - 20150626 corresponds to the last release of 4.x libstdc++ #ifndef EIGEN_HAS_TYPE_TRAITS #if EIGEN_MAX_CPP_VER>=11 && (EIGEN_HAS_CXX11 || EIGEN_COMP_MSVC >= 1700) \ && ((!EIGEN_COMP_GNUC_STRICT) || EIGEN_GNUC_AT_LEAST(5, 1)) \ && ((!defined(__GLIBCXX__)) || __GLIBCXX__ > 20150626) #define EIGEN_HAS_TYPE_TRAITS 1 #define EIGEN_INCLUDE_TYPE_TRAITS #else #define EIGEN_HAS_TYPE_TRAITS 0 #endif #endif // Does the compiler support variadic templates? #ifndef EIGEN_HAS_VARIADIC_TEMPLATES #if EIGEN_MAX_CPP_VER>=11 && (EIGEN_COMP_CXXVER >= 11) \ && (!defined(__NVCC__) || !EIGEN_ARCH_ARM_OR_ARM64 || (EIGEN_COMP_NVCC >= 80000) ) // ^^ Disable the use of variadic templates when compiling with versions of nvcc older than 8.0 on ARM devices: // this prevents nvcc from crashing when compiling Eigen on Tegra X1 #define EIGEN_HAS_VARIADIC_TEMPLATES 1 #elif EIGEN_MAX_CPP_VER>=11 && (EIGEN_COMP_CXXVER >= 11) && defined(SYCL_DEVICE_ONLY) #define EIGEN_HAS_VARIADIC_TEMPLATES 1 #else #define EIGEN_HAS_VARIADIC_TEMPLATES 0 #endif #endif // Does the compiler fully support const expressions? (as in c++14) #ifndef EIGEN_HAS_CONSTEXPR #if defined(EIGEN_CUDACC) // Const expressions are supported provided that c++11 is enabled and we're using either clang or nvcc 7.5 or above #if EIGEN_MAX_CPP_VER>=14 && (EIGEN_COMP_CXXVER >= 11 && (EIGEN_COMP_CLANG || EIGEN_COMP_NVCC >= 70500)) #define EIGEN_HAS_CONSTEXPR 1 #endif #elif EIGEN_MAX_CPP_VER>=14 && (__has_feature(cxx_relaxed_constexpr) || (EIGEN_COMP_CXXVER >= 14) || \ (EIGEN_GNUC_AT_LEAST(4,8) && (EIGEN_COMP_CXXVER >= 11)) || \ (EIGEN_COMP_CLANG >= 306 && (EIGEN_COMP_CXXVER >= 11))) #define EIGEN_HAS_CONSTEXPR 1 #endif #ifndef EIGEN_HAS_CONSTEXPR #define EIGEN_HAS_CONSTEXPR 0 #endif #endif // EIGEN_HAS_CONSTEXPR #if EIGEN_HAS_CONSTEXPR #define EIGEN_CONSTEXPR constexpr #else #define EIGEN_CONSTEXPR #endif // Does the compiler support C++11 math? // Let's be conservative and enable the default C++11 implementation only if we are sure it exists #ifndef EIGEN_HAS_CXX11_MATH #if EIGEN_MAX_CPP_VER>=11 && ((EIGEN_COMP_CXXVER > 11) || (EIGEN_COMP_CXXVER == 11) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_CLANG || EIGEN_COMP_MSVC || EIGEN_COMP_ICC) \ && (EIGEN_ARCH_i386_OR_x86_64) && (EIGEN_OS_GNULINUX || EIGEN_OS_WIN_STRICT || EIGEN_OS_MAC)) #define EIGEN_HAS_CXX11_MATH 1 #else #define EIGEN_HAS_CXX11_MATH 0 #endif #endif // Does the compiler support proper C++11 containers? #ifndef EIGEN_HAS_CXX11_CONTAINERS #if EIGEN_MAX_CPP_VER>=11 && \ ((EIGEN_COMP_CXXVER > 11) \ || ((EIGEN_COMP_CXXVER == 11) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_CLANG || EIGEN_COMP_MSVC || EIGEN_COMP_ICC>=1400))) #define EIGEN_HAS_CXX11_CONTAINERS 1 #else #define EIGEN_HAS_CXX11_CONTAINERS 0 #endif #endif // Does the compiler support C++11 noexcept? #ifndef EIGEN_HAS_CXX11_NOEXCEPT #if EIGEN_MAX_CPP_VER>=11 && \ (__has_feature(cxx_noexcept) \ || (EIGEN_COMP_CXXVER > 11) \ || ((EIGEN_COMP_CXXVER == 11) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_CLANG || EIGEN_COMP_MSVC || EIGEN_COMP_ICC>=1400))) #define EIGEN_HAS_CXX11_NOEXCEPT 1 #else #define EIGEN_HAS_CXX11_NOEXCEPT 0 #endif #endif #ifndef EIGEN_HAS_CXX11_ATOMIC #if EIGEN_MAX_CPP_VER>=11 && \ (__has_feature(cxx_atomic) \ || (EIGEN_COMP_CXXVER > 11) \ || ((EIGEN_COMP_CXXVER == 11) && (EIGEN_COMP_MSVC==0 || EIGEN_COMP_MSVC >= 1700))) #define EIGEN_HAS_CXX11_ATOMIC 1 #else #define EIGEN_HAS_CXX11_ATOMIC 0 #endif #endif #ifndef EIGEN_HAS_CXX11_OVERRIDE_FINAL #if EIGEN_MAX_CPP_VER>=11 && \ (EIGEN_COMP_CXXVER >= 11 || EIGEN_COMP_MSVC >= 1700) #define EIGEN_HAS_CXX11_OVERRIDE_FINAL 1 #else #define EIGEN_HAS_CXX11_OVERRIDE_FINAL 0 #endif #endif // NOTE: the required Apple's clang version is very conservative // and it could be that XCode 9 works just fine. // NOTE: the MSVC version is based on https://en.cppreference.com/w/cpp/compiler_support // and not tested. #ifndef EIGEN_HAS_CXX17_OVERALIGN #if EIGEN_MAX_CPP_VER>=17 && EIGEN_COMP_CXXVER>=17 && ( \ (EIGEN_COMP_MSVC >= 1912) \ || (EIGEN_GNUC_AT_LEAST(7,0)) \ || ((!defined(__apple_build_version__)) && (EIGEN_COMP_CLANG>=500)) \ || (( defined(__apple_build_version__)) && (__apple_build_version__>=10000000)) \ ) #define EIGEN_HAS_CXX17_OVERALIGN 1 #else #define EIGEN_HAS_CXX17_OVERALIGN 0 #endif #endif #if defined(EIGEN_CUDACC) && EIGEN_HAS_CONSTEXPR // While available already with c++11, this is useful mostly starting with c++14 and relaxed constexpr rules #if defined(__NVCC__) // nvcc considers constexpr functions as __host__ __device__ with the option --expt-relaxed-constexpr #ifdef __CUDACC_RELAXED_CONSTEXPR__ #define EIGEN_CONSTEXPR_ARE_DEVICE_FUNC #endif #elif defined(__clang__) && defined(__CUDA__) && __has_feature(cxx_relaxed_constexpr) // clang++ always considers constexpr functions as implicitly __host__ __device__ #define EIGEN_CONSTEXPR_ARE_DEVICE_FUNC #endif #endif // Does the compiler support the __int128 and __uint128_t extensions for 128-bit // integer arithmetic? // // Clang and GCC define __SIZEOF_INT128__ when these extensions are supported, // but we avoid using them in certain cases: // // * Building using Clang for Windows, where the Clang runtime library has // 128-bit support only on LP64 architectures, but Windows is LLP64. #ifndef EIGEN_HAS_BUILTIN_INT128 #if defined(__SIZEOF_INT128__) && !(EIGEN_OS_WIN && EIGEN_COMP_CLANG) #define EIGEN_HAS_BUILTIN_INT128 1 #else #define EIGEN_HAS_BUILTIN_INT128 0 #endif #endif //------------------------------------------------------------------------------------------ // Preprocessor programming helpers //------------------------------------------------------------------------------------------ // This macro can be used to prevent from macro expansion, e.g.: // std::max EIGEN_NOT_A_MACRO(a,b) #define EIGEN_NOT_A_MACRO #define EIGEN_DEBUG_VAR(x) std::cerr << #x << " = " << x << std::endl; // concatenate two tokens #define EIGEN_CAT2(a,b) a ## b #define EIGEN_CAT(a,b) EIGEN_CAT2(a,b) #define EIGEN_COMMA , // convert a token to a string #define EIGEN_MAKESTRING2(a) #a #define EIGEN_MAKESTRING(a) EIGEN_MAKESTRING2(a) // EIGEN_STRONG_INLINE is a stronger version of the inline, using __forceinline on MSVC, // but it still doesn't use GCC's always_inline. This is useful in (common) situations where MSVC needs forceinline // but GCC is still doing fine with just inline. #ifndef EIGEN_STRONG_INLINE #if (EIGEN_COMP_MSVC || EIGEN_COMP_ICC) && !defined(EIGEN_GPUCC) #define EIGEN_STRONG_INLINE __forceinline #else #define EIGEN_STRONG_INLINE inline #endif #endif // EIGEN_ALWAYS_INLINE is the stronget, it has the effect of making the function inline and adding every possible // attribute to maximize inlining. This should only be used when really necessary: in particular, // it uses __attribute__((always_inline)) on GCC, which most of the time is useless and can severely harm compile times. // FIXME with the always_inline attribute, // gcc 3.4.x and 4.1 reports the following compilation error: // Eval.h:91: sorry, unimplemented: inlining failed in call to 'const Eigen::Eval Eigen::MatrixBase::eval() const' // : function body not available // See also bug 1367 #if EIGEN_GNUC_AT_LEAST(4,2) && !defined(SYCL_DEVICE_ONLY) #define EIGEN_ALWAYS_INLINE __attribute__((always_inline)) inline #else #define EIGEN_ALWAYS_INLINE EIGEN_STRONG_INLINE #endif #if EIGEN_COMP_GNUC #define EIGEN_DONT_INLINE __attribute__((noinline)) #elif EIGEN_COMP_MSVC #define EIGEN_DONT_INLINE __declspec(noinline) #else #define EIGEN_DONT_INLINE #endif #if EIGEN_COMP_GNUC #define EIGEN_PERMISSIVE_EXPR __extension__ #else #define EIGEN_PERMISSIVE_EXPR #endif // GPU stuff // Disable some features when compiling with GPU compilers (NVCC/clang-cuda/SYCL/HIPCC) #if defined(EIGEN_CUDACC) || defined(SYCL_DEVICE_ONLY) || defined(EIGEN_HIPCC) // Do not try asserts on device code #ifndef EIGEN_NO_DEBUG #define EIGEN_NO_DEBUG #endif #ifdef EIGEN_INTERNAL_DEBUGGING #undef EIGEN_INTERNAL_DEBUGGING #endif #ifdef EIGEN_EXCEPTIONS #undef EIGEN_EXCEPTIONS #endif #endif #if defined(SYCL_DEVICE_ONLY) #ifndef EIGEN_DONT_VECTORIZE #define EIGEN_DONT_VECTORIZE #endif #define EIGEN_DEVICE_FUNC __attribute__((flatten)) __attribute__((always_inline)) // All functions callable from CUDA/HIP code must be qualified with __device__ #elif defined(EIGEN_GPUCC) #define EIGEN_DEVICE_FUNC __host__ __device__ #else #define EIGEN_DEVICE_FUNC #endif // this macro allows to get rid of linking errors about multiply defined functions. // - static is not very good because it prevents definitions from different object files to be merged. // So static causes the resulting linked executable to be bloated with multiple copies of the same function. // - inline is not perfect either as it unwantedly hints the compiler toward inlining the function. #define EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_DEVICE_FUNC #define EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_DEVICE_FUNC inline #ifdef NDEBUG # ifndef EIGEN_NO_DEBUG # define EIGEN_NO_DEBUG # endif #endif // eigen_plain_assert is where we implement the workaround for the assert() bug in GCC <= 4.3, see bug 89 #ifdef EIGEN_NO_DEBUG #ifdef SYCL_DEVICE_ONLY // used to silence the warning on SYCL device #define eigen_plain_assert(x) EIGEN_UNUSED_VARIABLE(x) #else #define eigen_plain_assert(x) #endif #else #if EIGEN_SAFE_TO_USE_STANDARD_ASSERT_MACRO namespace Eigen { namespace internal { inline bool copy_bool(bool b) { return b; } } } #define eigen_plain_assert(x) assert(x) #else // work around bug 89 #include // for abort #include // for std::cerr namespace Eigen { namespace internal { // trivial function copying a bool. Must be EIGEN_DONT_INLINE, so we implement it after including Eigen headers. // see bug 89. namespace { EIGEN_DONT_INLINE bool copy_bool(bool b) { return b; } } inline void assert_fail(const char *condition, const char *function, const char *file, int line) { std::cerr << "assertion failed: " << condition << " in function " << function << " at " << file << ":" << line << std::endl; abort(); } } } #define eigen_plain_assert(x) \ do { \ if(!Eigen::internal::copy_bool(x)) \ Eigen::internal::assert_fail(EIGEN_MAKESTRING(x), __PRETTY_FUNCTION__, __FILE__, __LINE__); \ } while(false) #endif #endif // eigen_assert can be overridden #ifndef eigen_assert #define eigen_assert(x) eigen_plain_assert(x) #endif #ifdef EIGEN_INTERNAL_DEBUGGING #define eigen_internal_assert(x) eigen_assert(x) #else #define eigen_internal_assert(x) #endif #ifdef EIGEN_NO_DEBUG #define EIGEN_ONLY_USED_FOR_DEBUG(x) EIGEN_UNUSED_VARIABLE(x) #else #define EIGEN_ONLY_USED_FOR_DEBUG(x) #endif #ifndef EIGEN_NO_DEPRECATED_WARNING #if EIGEN_COMP_GNUC #define EIGEN_DEPRECATED __attribute__((deprecated)) #elif EIGEN_COMP_MSVC #define EIGEN_DEPRECATED __declspec(deprecated) #else #define EIGEN_DEPRECATED #endif #else #define EIGEN_DEPRECATED #endif #if EIGEN_COMP_GNUC #define EIGEN_UNUSED __attribute__((unused)) #else #define EIGEN_UNUSED #endif // Suppresses 'unused variable' warnings. namespace Eigen { namespace internal { template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ignore_unused_variable(const T&) {} } } #define EIGEN_UNUSED_VARIABLE(var) Eigen::internal::ignore_unused_variable(var); #if !defined(EIGEN_ASM_COMMENT) #if EIGEN_COMP_GNUC && (EIGEN_ARCH_i386_OR_x86_64 || EIGEN_ARCH_ARM_OR_ARM64) #define EIGEN_ASM_COMMENT(X) __asm__("#" X) #else #define EIGEN_ASM_COMMENT(X) #endif #endif // Acts as a barrier preventing operations involving `X` from crossing. This // occurs, for example, in the fast rounding trick where a magic constant is // added then subtracted, which is otherwise compiled away with -ffast-math. // // See bug 1674 #if !defined(EIGEN_OPTIMIZATION_BARRIER) #if EIGEN_COMP_GNUC // According to https://gcc.gnu.org/onlinedocs/gcc/Constraints.html: // X: Any operand whatsoever. // r: A register operand is allowed provided that it is in a general // register. // g: Any register, memory or immediate integer operand is allowed, except // for registers that are not general registers. // w: (AArch32/AArch64) Floating point register, Advanced SIMD vector // register or SVE vector register. // x: (SSE) Any SSE register. // (AArch64) Like w, but restricted to registers 0 to 15 inclusive. // v: (PowerPC) An Altivec vector register. // wa:(PowerPC) A VSX register. // // "X" (uppercase) should work for all cases, though this seems to fail for // some versions of GCC for arm/aarch64 with // "error: inconsistent operand constraints in an 'asm'" // Clang x86_64/arm/aarch64 seems to require "g" to support both scalars and // vectors, otherwise // "error: non-trivial scalar-to-vector conversion, possible invalid // constraint for vector type" // // GCC for ppc64le generates an internal compiler error with x/X/g. // GCC for AVX generates an internal compiler error with X. // // Tested on icc/gcc/clang for sse, avx, avx2, avx512dq // gcc for arm, aarch64, // gcc for ppc64le, // both vectors and scalars. // // Note that this is restricted to plain types - this will not work // directly for std::complex, Eigen::half, Eigen::bfloat16. For these, // you will need to apply to the underlying POD type. #if EIGEN_ARCH_PPC && EIGEN_COMP_GNUC_STRICT // This seems to be broken on clang. Packet4f is loaded into a single // register rather than a vector, zeroing out some entries. Integer // types also generate a compile error. // General, Altivec, VSX. #define EIGEN_OPTIMIZATION_BARRIER(X) __asm__ ("" : "+r,v,wa" (X)); #elif EIGEN_ARCH_ARM_OR_ARM64 // General, NEON. #define EIGEN_OPTIMIZATION_BARRIER(X) __asm__ ("" : "+g,w" (X)); #elif EIGEN_ARCH_i386_OR_x86_64 // General, SSE. #define EIGEN_OPTIMIZATION_BARRIER(X) __asm__ ("" : "+g,x" (X)); #else // Not implemented for other architectures. #define EIGEN_OPTIMIZATION_BARRIER(X) #endif #else // Not implemented for other compilers. #define EIGEN_OPTIMIZATION_BARRIER(X) #endif #endif #if EIGEN_COMP_MSVC // NOTE MSVC often gives C4127 warnings with compiletime if statements. See bug 1362. // This workaround is ugly, but it does the job. # define EIGEN_CONST_CONDITIONAL(cond) (void)0, cond #else # define EIGEN_CONST_CONDITIONAL(cond) cond #endif #ifdef EIGEN_DONT_USE_RESTRICT_KEYWORD #define EIGEN_RESTRICT #endif #ifndef EIGEN_RESTRICT #define EIGEN_RESTRICT __restrict #endif #ifndef EIGEN_DEFAULT_IO_FORMAT #ifdef EIGEN_MAKING_DOCS // format used in Eigen's documentation // needed to define it here as escaping characters in CMake add_definition's argument seems very problematic. #define EIGEN_DEFAULT_IO_FORMAT Eigen::IOFormat(3, 0, " ", "\n", "", "") #else #define EIGEN_DEFAULT_IO_FORMAT Eigen::IOFormat() #endif #endif // just an empty macro ! #define EIGEN_EMPTY // When compiling CUDA/HIP device code with NVCC or HIPCC // pull in math functions from the global namespace. // In host mode, and when device code is compiled with clang, // use the std versions. #if (defined(EIGEN_CUDA_ARCH) && defined(__NVCC__)) || defined(EIGEN_HIP_DEVICE_COMPILE) #define EIGEN_USING_STD(FUNC) using ::FUNC; #else #define EIGEN_USING_STD(FUNC) using std::FUNC; #endif #if EIGEN_COMP_MSVC_STRICT && (EIGEN_COMP_MSVC < 1900 || (EIGEN_COMP_MSVC == 1900 && EIGEN_COMP_NVCC)) // For older MSVC versions, as well as 1900 && CUDA 8, using the base operator is necessary, // otherwise we get duplicate definition errors // For later MSVC versions, we require explicit operator= definition, otherwise we get // use of implicitly deleted operator errors. // (cf Bugs 920, 1000, 1324, 2291) #define EIGEN_INHERIT_ASSIGNMENT_EQUAL_OPERATOR(Derived) \ using Base::operator =; #elif EIGEN_COMP_CLANG // workaround clang bug (see http://forum.kde.org/viewtopic.php?f=74&t=102653) #define EIGEN_INHERIT_ASSIGNMENT_EQUAL_OPERATOR(Derived) \ using Base::operator =; \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const Derived& other) { Base::operator=(other); return *this; } \ template \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const DenseBase& other) { Base::operator=(other.derived()); return *this; } #else #define EIGEN_INHERIT_ASSIGNMENT_EQUAL_OPERATOR(Derived) \ using Base::operator =; \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const Derived& other) \ { \ Base::operator=(other); \ return *this; \ } #endif /** * \internal * \brief Macro to explicitly define the default copy constructor. * This is necessary, because the implicit definition is deprecated if the copy-assignment is overridden. */ #if EIGEN_HAS_CXX11 #define EIGEN_DEFAULT_COPY_CONSTRUCTOR(CLASS) CLASS(const CLASS&) = default; #else #define EIGEN_DEFAULT_COPY_CONSTRUCTOR(CLASS) #endif /** \internal * \brief Macro to manually inherit assignment operators. * This is necessary, because the implicitly defined assignment operator gets deleted when a custom operator= is defined. * With C++11 or later this also default-implements the copy-constructor */ #define EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Derived) \ EIGEN_INHERIT_ASSIGNMENT_EQUAL_OPERATOR(Derived) \ EIGEN_DEFAULT_COPY_CONSTRUCTOR(Derived) /** \internal * \brief Macro to manually define default constructors and destructors. * This is necessary when the copy constructor is re-defined. * For empty helper classes this should usually be protected, to avoid accidentally creating empty objects. * * Hiding the default destructor lead to problems in C++03 mode together with boost::multiprecision */ #if EIGEN_HAS_CXX11 #define EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(Derived) \ Derived() = default; \ ~Derived() = default; #else #define EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(Derived) \ Derived() {}; \ /* ~Derived() {}; */ #endif /** * Just a side note. Commenting within defines works only by documenting * behind the object (via '!<'). Comments cannot be multi-line and thus * we have these extra long lines. What is confusing doxygen over here is * that we use '\' and basically have a bunch of typedefs with their * documentation in a single line. **/ #define EIGEN_GENERIC_PUBLIC_INTERFACE(Derived) \ typedef typename Eigen::internal::traits::Scalar Scalar; /*!< \brief Numeric type, e.g. float, double, int or std::complex. */ \ typedef typename Eigen::NumTraits::Real RealScalar; /*!< \brief The underlying numeric type for composed scalar types. \details In cases where Scalar is e.g. std::complex, T were corresponding to RealScalar. */ \ typedef typename Base::CoeffReturnType CoeffReturnType; /*!< \brief The return type for coefficient access. \details Depending on whether the object allows direct coefficient access (e.g. for a MatrixXd), this type is either 'const Scalar&' or simply 'Scalar' for objects that do not allow direct coefficient access. */ \ typedef typename Eigen::internal::ref_selector::type Nested; \ typedef typename Eigen::internal::traits::StorageKind StorageKind; \ typedef typename Eigen::internal::traits::StorageIndex StorageIndex; \ enum CompileTimeTraits \ { RowsAtCompileTime = Eigen::internal::traits::RowsAtCompileTime, \ ColsAtCompileTime = Eigen::internal::traits::ColsAtCompileTime, \ Flags = Eigen::internal::traits::Flags, \ SizeAtCompileTime = Base::SizeAtCompileTime, \ MaxSizeAtCompileTime = Base::MaxSizeAtCompileTime, \ IsVectorAtCompileTime = Base::IsVectorAtCompileTime }; \ using Base::derived; \ using Base::const_cast_derived; // FIXME Maybe the EIGEN_DENSE_PUBLIC_INTERFACE could be removed as importing PacketScalar is rarely needed #define EIGEN_DENSE_PUBLIC_INTERFACE(Derived) \ EIGEN_GENERIC_PUBLIC_INTERFACE(Derived) \ typedef typename Base::PacketScalar PacketScalar; #define EIGEN_PLAIN_ENUM_MIN(a,b) (((int)a <= (int)b) ? (int)a : (int)b) #define EIGEN_PLAIN_ENUM_MAX(a,b) (((int)a >= (int)b) ? (int)a : (int)b) // EIGEN_SIZE_MIN_PREFER_DYNAMIC gives the min between compile-time sizes. 0 has absolute priority, followed by 1, // followed by Dynamic, followed by other finite values. The reason for giving Dynamic the priority over // finite values is that min(3, Dynamic) should be Dynamic, since that could be anything between 0 and 3. #define EIGEN_SIZE_MIN_PREFER_DYNAMIC(a,b) (((int)a == 0 || (int)b == 0) ? 0 \ : ((int)a == 1 || (int)b == 1) ? 1 \ : ((int)a == Dynamic || (int)b == Dynamic) ? Dynamic \ : ((int)a <= (int)b) ? (int)a : (int)b) // EIGEN_SIZE_MIN_PREFER_FIXED is a variant of EIGEN_SIZE_MIN_PREFER_DYNAMIC comparing MaxSizes. The difference is that finite values // now have priority over Dynamic, so that min(3, Dynamic) gives 3. Indeed, whatever the actual value is // (between 0 and 3), it is not more than 3. #define EIGEN_SIZE_MIN_PREFER_FIXED(a,b) (((int)a == 0 || (int)b == 0) ? 0 \ : ((int)a == 1 || (int)b == 1) ? 1 \ : ((int)a == Dynamic && (int)b == Dynamic) ? Dynamic \ : ((int)a == Dynamic) ? (int)b \ : ((int)b == Dynamic) ? (int)a \ : ((int)a <= (int)b) ? (int)a : (int)b) // see EIGEN_SIZE_MIN_PREFER_DYNAMIC. No need for a separate variant for MaxSizes here. #define EIGEN_SIZE_MAX(a,b) (((int)a == Dynamic || (int)b == Dynamic) ? Dynamic \ : ((int)a >= (int)b) ? (int)a : (int)b) #define EIGEN_LOGICAL_XOR(a,b) (((a) || (b)) && !((a) && (b))) #define EIGEN_IMPLIES(a,b) (!(a) || (b)) #if EIGEN_HAS_BUILTIN(__builtin_expect) || EIGEN_COMP_GNUC #define EIGEN_PREDICT_FALSE(x) (__builtin_expect(x, false)) #define EIGEN_PREDICT_TRUE(x) (__builtin_expect(false || (x), true)) #else #define EIGEN_PREDICT_FALSE(x) (x) #define EIGEN_PREDICT_TRUE(x) (x) #endif // the expression type of a standard coefficient wise binary operation #define EIGEN_CWISE_BINARY_RETURN_TYPE(LHS,RHS,OPNAME) \ CwiseBinaryOp< \ EIGEN_CAT(EIGEN_CAT(internal::scalar_,OPNAME),_op)< \ typename internal::traits::Scalar, \ typename internal::traits::Scalar \ >, \ const LHS, \ const RHS \ > #define EIGEN_MAKE_CWISE_BINARY_OP(METHOD,OPNAME) \ template \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const EIGEN_CWISE_BINARY_RETURN_TYPE(Derived,OtherDerived,OPNAME) \ (METHOD)(const EIGEN_CURRENT_STORAGE_BASE_CLASS &other) const \ { \ return EIGEN_CWISE_BINARY_RETURN_TYPE(Derived,OtherDerived,OPNAME)(derived(), other.derived()); \ } #define EIGEN_SCALAR_BINARY_SUPPORTED(OPNAME,TYPEA,TYPEB) \ (Eigen::internal::has_ReturnType > >::value) #define EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(EXPR,SCALAR,OPNAME) \ CwiseBinaryOp::Scalar,SCALAR>, const EXPR, \ const typename internal::plain_constant_type::type> #define EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(SCALAR,EXPR,OPNAME) \ CwiseBinaryOp::Scalar>, \ const typename internal::plain_constant_type::type, const EXPR> // Workaround for MSVC 2010 (see ML thread "patch with compile for for MSVC 2010") #if EIGEN_COMP_MSVC_STRICT && (EIGEN_COMP_MSVC_STRICT<=1600) #define EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE(X) typename internal::enable_if::type #else #define EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE(X) X #endif #define EIGEN_MAKE_SCALAR_BINARY_OP_ONTHERIGHT(METHOD,OPNAME) \ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE(const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(Derived,typename internal::promote_scalar_arg::type,OPNAME))\ (METHOD)(const T& scalar) const { \ typedef typename internal::promote_scalar_arg::type PromotedT; \ return EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(Derived,PromotedT,OPNAME)(derived(), \ typename internal::plain_constant_type::type(derived().rows(), derived().cols(), internal::scalar_constant_op(scalar))); \ } #define EIGEN_MAKE_SCALAR_BINARY_OP_ONTHELEFT(METHOD,OPNAME) \ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE friend \ EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE(const EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(typename internal::promote_scalar_arg::type,Derived,OPNAME)) \ (METHOD)(const T& scalar, const StorageBaseType& matrix) { \ typedef typename internal::promote_scalar_arg::type PromotedT; \ return EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(PromotedT,Derived,OPNAME)( \ typename internal::plain_constant_type::type(matrix.derived().rows(), matrix.derived().cols(), internal::scalar_constant_op(scalar)), matrix.derived()); \ } #define EIGEN_MAKE_SCALAR_BINARY_OP(METHOD,OPNAME) \ EIGEN_MAKE_SCALAR_BINARY_OP_ONTHELEFT(METHOD,OPNAME) \ EIGEN_MAKE_SCALAR_BINARY_OP_ONTHERIGHT(METHOD,OPNAME) #if (defined(_CPPUNWIND) || defined(__EXCEPTIONS)) && !defined(EIGEN_CUDA_ARCH) && !defined(EIGEN_EXCEPTIONS) && !defined(EIGEN_USE_SYCL) && !defined(EIGEN_HIP_DEVICE_COMPILE) #define EIGEN_EXCEPTIONS #endif #ifdef EIGEN_EXCEPTIONS # define EIGEN_THROW_X(X) throw X # define EIGEN_THROW throw # define EIGEN_TRY try # define EIGEN_CATCH(X) catch (X) #else # if defined(EIGEN_CUDA_ARCH) # define EIGEN_THROW_X(X) asm("trap;") # define EIGEN_THROW asm("trap;") # elif defined(EIGEN_HIP_DEVICE_COMPILE) # define EIGEN_THROW_X(X) asm("s_trap 0") # define EIGEN_THROW asm("s_trap 0") # else # define EIGEN_THROW_X(X) std::abort() # define EIGEN_THROW std::abort() # endif # define EIGEN_TRY if (true) # define EIGEN_CATCH(X) else #endif #if EIGEN_HAS_CXX11_NOEXCEPT # define EIGEN_INCLUDE_TYPE_TRAITS # define EIGEN_NOEXCEPT noexcept # define EIGEN_NOEXCEPT_IF(x) noexcept(x) # define EIGEN_NO_THROW noexcept(true) # define EIGEN_EXCEPTION_SPEC(X) noexcept(false) #else # define EIGEN_NOEXCEPT # define EIGEN_NOEXCEPT_IF(x) # define EIGEN_NO_THROW throw() # if EIGEN_COMP_MSVC || EIGEN_COMP_CXXVER>=17 // MSVC does not support exception specifications (warning C4290), // and they are deprecated in c++11 anyway. This is even an error in c++17. # define EIGEN_EXCEPTION_SPEC(X) throw() # else # define EIGEN_EXCEPTION_SPEC(X) throw(X) # endif #endif #if EIGEN_HAS_VARIADIC_TEMPLATES // The all function is used to enable a variadic version of eigen_assert which can take a parameter pack as its input. namespace Eigen { namespace internal { inline bool all(){ return true; } template bool all(T t, Ts ... ts){ return t && all(ts...); } } } #endif #if EIGEN_HAS_CXX11_OVERRIDE_FINAL // provide override and final specifiers if they are available: # define EIGEN_OVERRIDE override # define EIGEN_FINAL final #else # define EIGEN_OVERRIDE # define EIGEN_FINAL #endif // Wrapping #pragma unroll in a macro since it is required for SYCL #if defined(SYCL_DEVICE_ONLY) #if defined(_MSC_VER) #define EIGEN_UNROLL_LOOP __pragma(unroll) #else #define EIGEN_UNROLL_LOOP _Pragma("unroll") #endif #else #define EIGEN_UNROLL_LOOP #endif #endif // EIGEN_MACROS_H RcppEigen/inst/include/Eigen/src/Core/util/IntegralConstant.h0000644000176200001440000002530514567757725023677 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2017 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_INTEGRAL_CONSTANT_H #define EIGEN_INTEGRAL_CONSTANT_H namespace Eigen { namespace internal { template class FixedInt; template class VariableAndFixedInt; /** \internal * \class FixedInt * * This class embeds a compile-time integer \c N. * * It is similar to c++11 std::integral_constant but with some additional features * such as: * - implicit conversion to int * - arithmetic and some bitwise operators: -, +, *, /, %, &, | * - c++98/14 compatibility with fix and fix() syntax to define integral constants. * * It is strongly discouraged to directly deal with this class FixedInt. Instances are expcected to * be created by the user using Eigen::fix or Eigen::fix(). In C++98-11, the former syntax does * not create a FixedInt instance but rather a point to function that needs to be \em cleaned-up * using the generic helper: * \code * internal::cleanup_index_type::type * internal::cleanup_index_type::type * \endcode * where T can a FixedInt, a pointer to function FixedInt (*)(), or numerous other integer-like representations. * \c DynamicKey is either Dynamic (default) or DynamicIndex and used to identify true compile-time values. * * For convenience, you can extract the compile-time value \c N in a generic way using the following helper: * \code * internal::get_fixed_value::value * \endcode * that will give you \c N if T equals FixedInt or FixedInt (*)(), and \c DefaultVal if T does not embed any compile-time value (e.g., T==int). * * \sa fix, class VariableAndFixedInt */ template class FixedInt { public: static const int value = N; EIGEN_CONSTEXPR operator int() const { return value; } FixedInt() {} FixedInt( VariableAndFixedInt other) { #ifndef EIGEN_INTERNAL_DEBUGGING EIGEN_UNUSED_VARIABLE(other); #endif eigen_internal_assert(int(other)==N); } FixedInt<-N> operator-() const { return FixedInt<-N>(); } template FixedInt operator+( FixedInt) const { return FixedInt(); } template FixedInt operator-( FixedInt) const { return FixedInt(); } template FixedInt operator*( FixedInt) const { return FixedInt(); } template FixedInt operator/( FixedInt) const { return FixedInt(); } template FixedInt operator%( FixedInt) const { return FixedInt(); } template FixedInt operator|( FixedInt) const { return FixedInt(); } template FixedInt operator&( FixedInt) const { return FixedInt(); } #if EIGEN_HAS_CXX14_VARIABLE_TEMPLATES // Needed in C++14 to allow fix(): FixedInt operator() () const { return *this; } VariableAndFixedInt operator() (int val) const { return VariableAndFixedInt(val); } #else FixedInt ( FixedInt (*)() ) {} #endif #if EIGEN_HAS_CXX11 FixedInt(std::integral_constant) {} #endif }; /** \internal * \class VariableAndFixedInt * * This class embeds both a compile-time integer \c N and a runtime integer. * Both values are supposed to be equal unless the compile-time value \c N has a special * value meaning that the runtime-value should be used. Depending on the context, this special * value can be either Eigen::Dynamic (for positive quantities) or Eigen::DynamicIndex (for * quantities that can be negative). * * It is the return-type of the function Eigen::fix(int), and most of the time this is the only * way it is used. It is strongly discouraged to directly deal with instances of VariableAndFixedInt. * Indeed, in order to write generic code, it is the responsibility of the callee to properly convert * it to either a true compile-time quantity (i.e. a FixedInt), or to a runtime quantity (e.g., an Index) * using the following generic helper: * \code * internal::cleanup_index_type::type * internal::cleanup_index_type::type * \endcode * where T can be a template instantiation of VariableAndFixedInt or numerous other integer-like representations. * \c DynamicKey is either Dynamic (default) or DynamicIndex and used to identify true compile-time values. * * For convenience, you can also extract the compile-time value \c N using the following helper: * \code * internal::get_fixed_value::value * \endcode * that will give you \c N if T equals VariableAndFixedInt, and \c DefaultVal if T does not embed any compile-time value (e.g., T==int). * * \sa fix(int), class FixedInt */ template class VariableAndFixedInt { public: static const int value = N; operator int() const { return m_value; } VariableAndFixedInt(int val) { m_value = val; } protected: int m_value; }; template struct get_fixed_value { static const int value = Default; }; template struct get_fixed_value,Default> { static const int value = N; }; #if !EIGEN_HAS_CXX14 template struct get_fixed_value (*)(),Default> { static const int value = N; }; #endif template struct get_fixed_value,Default> { static const int value = N ; }; template struct get_fixed_value,Default> { static const int value = N; }; template EIGEN_DEVICE_FUNC Index get_runtime_value(const T &x) { return x; } #if !EIGEN_HAS_CXX14 template EIGEN_DEVICE_FUNC Index get_runtime_value(FixedInt (*)()) { return N; } #endif // Cleanup integer/FixedInt/VariableAndFixedInt/etc types: // By default, no cleanup: template struct cleanup_index_type { typedef T type; }; // Convert any integral type (e.g., short, int, unsigned int, etc.) to Eigen::Index template struct cleanup_index_type::value>::type> { typedef Index type; }; #if !EIGEN_HAS_CXX14 // In c++98/c++11, fix is a pointer to function that we better cleanup to a true FixedInt: template struct cleanup_index_type (*)(), DynamicKey> { typedef FixedInt type; }; #endif // If VariableAndFixedInt does not match DynamicKey, then we turn it to a pure compile-time value: template struct cleanup_index_type, DynamicKey> { typedef FixedInt type; }; // If VariableAndFixedInt matches DynamicKey, then we turn it to a pure runtime-value (aka Index): template struct cleanup_index_type, DynamicKey> { typedef Index type; }; #if EIGEN_HAS_CXX11 template struct cleanup_index_type, DynamicKey> { typedef FixedInt type; }; #endif } // end namespace internal #ifndef EIGEN_PARSED_BY_DOXYGEN #if EIGEN_HAS_CXX14_VARIABLE_TEMPLATES template static const internal::FixedInt fix{}; #else template inline internal::FixedInt fix() { return internal::FixedInt(); } // The generic typename T is mandatory. Otherwise, a code like fix could refer to either the function above or this next overload. // This way a code like fix can only refer to the previous function. template inline internal::VariableAndFixedInt fix(T val) { return internal::VariableAndFixedInt(internal::convert_index(val)); } #endif #else // EIGEN_PARSED_BY_DOXYGEN /** \var fix() * \ingroup Core_Module * * This \em identifier permits to construct an object embedding a compile-time integer \c N. * * \tparam N the compile-time integer value * * It is typically used in conjunction with the Eigen::seq and Eigen::seqN functions to pass compile-time values to them: * \code * seqN(10,fix<4>,fix<-3>) // <=> [10 7 4 1] * \endcode * * See also the function fix(int) to pass both a compile-time and runtime value. * * In c++14, it is implemented as: * \code * template static const internal::FixedInt fix{}; * \endcode * where internal::FixedInt is an internal template class similar to * \c std::integral_constant * Here, \c fix is thus an object of type \c internal::FixedInt. * * In c++98/11, it is implemented as a function: * \code * template inline internal::FixedInt fix(); * \endcode * Here internal::FixedInt is thus a pointer to function. * * If for some reason you want a true object in c++98 then you can write: \code fix() \endcode which is also valid in c++14. * * \sa fix(int), seq, seqN */ template static const auto fix(); /** \fn fix(int) * \ingroup Core_Module * * This function returns an object embedding both a compile-time integer \c N, and a fallback runtime value \a val. * * \tparam N the compile-time integer value * \param val the fallback runtime integer value * * This function is a more general version of the \ref fix identifier/function that can be used in template code * where the compile-time value could turn out to actually mean "undefined at compile-time". For positive integers * such as a size or a dimension, this case is identified by Eigen::Dynamic, whereas runtime signed integers * (e.g., an increment/stride) are identified as Eigen::DynamicIndex. In such a case, the runtime value \a val * will be used as a fallback. * * A typical use case would be: * \code * template void foo(const MatrixBase &mat) { * const int N = Derived::RowsAtCompileTime==Dynamic ? Dynamic : Derived::RowsAtCompileTime/2; * const int n = mat.rows()/2; * ... mat( seqN(0,fix(n) ) ...; * } * \endcode * In this example, the function Eigen::seqN knows that the second argument is expected to be a size. * If the passed compile-time value N equals Eigen::Dynamic, then the proxy object returned by fix will be dissmissed, and converted to an Eigen::Index of value \c n. * Otherwise, the runtime-value \c n will be dissmissed, and the returned ArithmeticSequence will be of the exact same type as seqN(0,fix) . * * \sa fix, seqN, class ArithmeticSequence */ template static const auto fix(int val); #endif // EIGEN_PARSED_BY_DOXYGEN } // end namespace Eigen #endif // EIGEN_INTEGRAL_CONSTANT_H RcppEigen/inst/include/Eigen/src/Core/util/ReenableStupidWarnings.h0000644000176200001440000000200614567757725025030 0ustar liggesusers#ifdef EIGEN_WARNINGS_DISABLED_2 // "DisableStupidWarnings.h" was included twice recursively: Do not reenable warnings yet! # undef EIGEN_WARNINGS_DISABLED_2 #elif defined(EIGEN_WARNINGS_DISABLED) #undef EIGEN_WARNINGS_DISABLED #ifndef EIGEN_PERMANENTLY_DISABLE_STUPID_WARNINGS #ifdef _MSC_VER #pragma warning( pop ) #elif defined __INTEL_COMPILER #pragma warning pop #elif defined __clang__ // #pragma clang diagnostic pop #elif defined __GNUC__ && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) // #pragma GCC diagnostic pop #endif #if defined __NVCC__ // Don't reenable the diagnostic messages, as it turns out these messages need // to be disabled at the point of the template instantiation (i.e the user code) // otherwise they'll be triggered by nvcc. // #pragma diag_default code_is_unreachable // #pragma diag_default initialization_not_reachable // #pragma diag_default 2651 // #pragma diag_default 2653 #endif #endif #endif // EIGEN_WARNINGS_DISABLED RcppEigen/inst/include/Eigen/src/Core/util/SymbolicIndex.h0000644000176200001440000002734314567757725023175 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2017 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_SYMBOLIC_INDEX_H #define EIGEN_SYMBOLIC_INDEX_H namespace Eigen { /** \namespace Eigen::symbolic * \ingroup Core_Module * * This namespace defines a set of classes and functions to build and evaluate symbolic expressions of scalar type Index. * Here is a simple example: * * \code * // First step, defines symbols: * struct x_tag {}; static const symbolic::SymbolExpr x; * struct y_tag {}; static const symbolic::SymbolExpr y; * struct z_tag {}; static const symbolic::SymbolExpr z; * * // Defines an expression: * auto expr = (x+3)/y+z; * * // And evaluate it: (c++14) * std::cout << expr.eval(x=6,y=3,z=-13) << "\n"; * * // In c++98/11, only one symbol per expression is supported for now: * auto expr98 = (3-x)/2; * std::cout << expr98.eval(x=6) << "\n"; * \endcode * * It is currently only used internally to define and manipulate the Eigen::last and Eigen::lastp1 symbols in Eigen::seq and Eigen::seqN. * */ namespace symbolic { template class Symbol; template class NegateExpr; template class AddExpr; template class ProductExpr; template class QuotientExpr; // A simple wrapper around an integral value to provide the eval method. // We could also use a free-function symbolic_eval... template class ValueExpr { public: ValueExpr(IndexType val) : m_value(val) {} template IndexType eval_impl(const T&) const { return m_value; } protected: IndexType m_value; }; // Specialization for compile-time value, // It is similar to ValueExpr(N) but this version helps the compiler to generate better code. template class ValueExpr > { public: ValueExpr() {} template EIGEN_CONSTEXPR Index eval_impl(const T&) const { return N; } }; /** \class BaseExpr * \ingroup Core_Module * Common base class of any symbolic expressions */ template class BaseExpr { public: const Derived& derived() const { return *static_cast(this); } /** Evaluate the expression given the \a values of the symbols. * * \param values defines the values of the symbols, it can either be a SymbolValue or a std::tuple of SymbolValue * as constructed by SymbolExpr::operator= operator. * */ template Index eval(const T& values) const { return derived().eval_impl(values); } #if EIGEN_HAS_CXX14 template Index eval(Types&&... values) const { return derived().eval_impl(std::make_tuple(values...)); } #endif NegateExpr operator-() const { return NegateExpr(derived()); } AddExpr > operator+(Index b) const { return AddExpr >(derived(), b); } AddExpr > operator-(Index a) const { return AddExpr >(derived(), -a); } ProductExpr > operator*(Index a) const { return ProductExpr >(derived(),a); } QuotientExpr > operator/(Index a) const { return QuotientExpr >(derived(),a); } friend AddExpr > operator+(Index a, const BaseExpr& b) { return AddExpr >(b.derived(), a); } friend AddExpr,ValueExpr<> > operator-(Index a, const BaseExpr& b) { return AddExpr,ValueExpr<> >(-b.derived(), a); } friend ProductExpr,Derived> operator*(Index a, const BaseExpr& b) { return ProductExpr,Derived>(a,b.derived()); } friend QuotientExpr,Derived> operator/(Index a, const BaseExpr& b) { return QuotientExpr,Derived>(a,b.derived()); } template AddExpr > > operator+(internal::FixedInt) const { return AddExpr > >(derived(), ValueExpr >()); } template AddExpr > > operator-(internal::FixedInt) const { return AddExpr > >(derived(), ValueExpr >()); } template ProductExpr > > operator*(internal::FixedInt) const { return ProductExpr > >(derived(),ValueExpr >()); } template QuotientExpr > > operator/(internal::FixedInt) const { return QuotientExpr > >(derived(),ValueExpr >()); } template friend AddExpr > > operator+(internal::FixedInt, const BaseExpr& b) { return AddExpr > >(b.derived(), ValueExpr >()); } template friend AddExpr,ValueExpr > > operator-(internal::FixedInt, const BaseExpr& b) { return AddExpr,ValueExpr > >(-b.derived(), ValueExpr >()); } template friend ProductExpr >,Derived> operator*(internal::FixedInt, const BaseExpr& b) { return ProductExpr >,Derived>(ValueExpr >(),b.derived()); } template friend QuotientExpr >,Derived> operator/(internal::FixedInt, const BaseExpr& b) { return QuotientExpr > ,Derived>(ValueExpr >(),b.derived()); } #if (!EIGEN_HAS_CXX14) template AddExpr > > operator+(internal::FixedInt (*)()) const { return AddExpr > >(derived(), ValueExpr >()); } template AddExpr > > operator-(internal::FixedInt (*)()) const { return AddExpr > >(derived(), ValueExpr >()); } template ProductExpr > > operator*(internal::FixedInt (*)()) const { return ProductExpr > >(derived(),ValueExpr >()); } template QuotientExpr > > operator/(internal::FixedInt (*)()) const { return QuotientExpr > >(derived(),ValueExpr >()); } template friend AddExpr > > operator+(internal::FixedInt (*)(), const BaseExpr& b) { return AddExpr > >(b.derived(), ValueExpr >()); } template friend AddExpr,ValueExpr > > operator-(internal::FixedInt (*)(), const BaseExpr& b) { return AddExpr,ValueExpr > >(-b.derived(), ValueExpr >()); } template friend ProductExpr >,Derived> operator*(internal::FixedInt (*)(), const BaseExpr& b) { return ProductExpr >,Derived>(ValueExpr >(),b.derived()); } template friend QuotientExpr >,Derived> operator/(internal::FixedInt (*)(), const BaseExpr& b) { return QuotientExpr > ,Derived>(ValueExpr >(),b.derived()); } #endif template AddExpr operator+(const BaseExpr &b) const { return AddExpr(derived(), b.derived()); } template AddExpr > operator-(const BaseExpr &b) const { return AddExpr >(derived(), -b.derived()); } template ProductExpr operator*(const BaseExpr &b) const { return ProductExpr(derived(), b.derived()); } template QuotientExpr operator/(const BaseExpr &b) const { return QuotientExpr(derived(), b.derived()); } }; template struct is_symbolic { // BaseExpr has no conversion ctor, so we only have to check whether T can be statically cast to its base class BaseExpr. enum { value = internal::is_convertible >::value }; }; /** Represents the actual value of a symbol identified by its tag * * It is the return type of SymbolValue::operator=, and most of the time this is only way it is used. */ template class SymbolValue { public: /** Default constructor from the value \a val */ SymbolValue(Index val) : m_value(val) {} /** \returns the stored value of the symbol */ Index value() const { return m_value; } protected: Index m_value; }; /** Expression of a symbol uniquely identified by the template parameter type \c tag */ template class SymbolExpr : public BaseExpr > { public: /** Alias to the template parameter \c tag */ typedef tag Tag; SymbolExpr() {} /** Associate the value \a val to the given symbol \c *this, uniquely identified by its \c Tag. * * The returned object should be passed to ExprBase::eval() to evaluate a given expression with this specified runtime-time value. */ SymbolValue operator=(Index val) const { return SymbolValue(val); } Index eval_impl(const SymbolValue &values) const { return values.value(); } #if EIGEN_HAS_CXX14 // C++14 versions suitable for multiple symbols template Index eval_impl(const std::tuple& values) const { return std::get >(values).value(); } #endif }; template class NegateExpr : public BaseExpr > { public: NegateExpr(const Arg0& arg0) : m_arg0(arg0) {} template Index eval_impl(const T& values) const { return -m_arg0.eval_impl(values); } protected: Arg0 m_arg0; }; template class AddExpr : public BaseExpr > { public: AddExpr(const Arg0& arg0, const Arg1& arg1) : m_arg0(arg0), m_arg1(arg1) {} template Index eval_impl(const T& values) const { return m_arg0.eval_impl(values) + m_arg1.eval_impl(values); } protected: Arg0 m_arg0; Arg1 m_arg1; }; template class ProductExpr : public BaseExpr > { public: ProductExpr(const Arg0& arg0, const Arg1& arg1) : m_arg0(arg0), m_arg1(arg1) {} template Index eval_impl(const T& values) const { return m_arg0.eval_impl(values) * m_arg1.eval_impl(values); } protected: Arg0 m_arg0; Arg1 m_arg1; }; template class QuotientExpr : public BaseExpr > { public: QuotientExpr(const Arg0& arg0, const Arg1& arg1) : m_arg0(arg0), m_arg1(arg1) {} template Index eval_impl(const T& values) const { return m_arg0.eval_impl(values) / m_arg1.eval_impl(values); } protected: Arg0 m_arg0; Arg1 m_arg1; }; } // end namespace symbolic } // end namespace Eigen #endif // EIGEN_SYMBOLIC_INDEX_H RcppEigen/inst/include/Eigen/src/Core/util/Constants.h0000644000176200001440000005265314567757725022402 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2015 Gael Guennebaud // Copyright (C) 2007-2009 Benoit Jacob // Copyright (C) 2020, Arm Limited and Contributors // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_CONSTANTS_H #define EIGEN_CONSTANTS_H namespace Eigen { /** This value means that a positive quantity (e.g., a size) is not known at compile-time, and that instead the value is * stored in some runtime variable. * * Changing the value of Dynamic breaks the ABI, as Dynamic is often used as a template parameter for Matrix. */ const int Dynamic = -1; /** This value means that a signed quantity (e.g., a signed index) is not known at compile-time, and that instead its value * has to be specified at runtime. */ const int DynamicIndex = 0xffffff; /** This value means that the increment to go from one value to another in a sequence is not constant for each step. */ const int UndefinedIncr = 0xfffffe; /** This value means +Infinity; it is currently used only as the p parameter to MatrixBase::lpNorm(). * The value Infinity there means the L-infinity norm. */ const int Infinity = -1; /** This value means that the cost to evaluate an expression coefficient is either very expensive or * cannot be known at compile time. * * This value has to be positive to (1) simplify cost computation, and (2) allow to distinguish between a very expensive and very very expensive expressions. * It thus must also be large enough to make sure unrolling won't happen and that sub expressions will be evaluated, but not too large to avoid overflow. */ const int HugeCost = 10000; /** \defgroup flags Flags * \ingroup Core_Module * * These are the possible bits which can be OR'ed to constitute the flags of a matrix or * expression. * * It is important to note that these flags are a purely compile-time notion. They are a compile-time property of * an expression type, implemented as enum's. They are not stored in memory at runtime, and they do not incur any * runtime overhead. * * \sa MatrixBase::Flags */ /** \ingroup flags * * for a matrix, this means that the storage order is row-major. * If this bit is not set, the storage order is column-major. * For an expression, this determines the storage order of * the matrix created by evaluation of that expression. * \sa \blank \ref TopicStorageOrders */ const unsigned int RowMajorBit = 0x1; /** \ingroup flags * means the expression should be evaluated by the calling expression */ const unsigned int EvalBeforeNestingBit = 0x2; /** \ingroup flags * \deprecated * means the expression should be evaluated before any assignment */ EIGEN_DEPRECATED const unsigned int EvalBeforeAssigningBit = 0x4; // FIXME deprecated /** \ingroup flags * * Short version: means the expression might be vectorized * * Long version: means that the coefficients can be handled by packets * and start at a memory location whose alignment meets the requirements * of the present CPU architecture for optimized packet access. In the fixed-size * case, there is the additional condition that it be possible to access all the * coefficients by packets (this implies the requirement that the size be a multiple of 16 bytes, * and that any nontrivial strides don't break the alignment). In the dynamic-size case, * there is no such condition on the total size and strides, so it might not be possible to access * all coeffs by packets. * * \note This bit can be set regardless of whether vectorization is actually enabled. * To check for actual vectorizability, see \a ActualPacketAccessBit. */ const unsigned int PacketAccessBit = 0x8; #ifdef EIGEN_VECTORIZE /** \ingroup flags * * If vectorization is enabled (EIGEN_VECTORIZE is defined) this constant * is set to the value \a PacketAccessBit. * * If vectorization is not enabled (EIGEN_VECTORIZE is not defined) this constant * is set to the value 0. */ const unsigned int ActualPacketAccessBit = PacketAccessBit; #else const unsigned int ActualPacketAccessBit = 0x0; #endif /** \ingroup flags * * Short version: means the expression can be seen as 1D vector. * * Long version: means that one can access the coefficients * of this expression by coeff(int), and coeffRef(int) in the case of a lvalue expression. These * index-based access methods are guaranteed * to not have to do any runtime computation of a (row, col)-pair from the index, so that it * is guaranteed that whenever it is available, index-based access is at least as fast as * (row,col)-based access. Expressions for which that isn't possible don't have the LinearAccessBit. * * If both PacketAccessBit and LinearAccessBit are set, then the * packets of this expression can be accessed by packet(int), and writePacket(int) in the case of a * lvalue expression. * * Typically, all vector expressions have the LinearAccessBit, but there is one exception: * Product expressions don't have it, because it would be troublesome for vectorization, even when the * Product is a vector expression. Thus, vector Product expressions allow index-based coefficient access but * not index-based packet access, so they don't have the LinearAccessBit. */ const unsigned int LinearAccessBit = 0x10; /** \ingroup flags * * Means the expression has a coeffRef() method, i.e. is writable as its individual coefficients are directly addressable. * This rules out read-only expressions. * * Note that DirectAccessBit and LvalueBit are mutually orthogonal, as there are examples of expression having one but note * the other: * \li writable expressions that don't have a very simple memory layout as a strided array, have LvalueBit but not DirectAccessBit * \li Map-to-const expressions, for example Map, have DirectAccessBit but not LvalueBit * * Expressions having LvalueBit also have their coeff() method returning a const reference instead of returning a new value. */ const unsigned int LvalueBit = 0x20; /** \ingroup flags * * Means that the underlying array of coefficients can be directly accessed as a plain strided array. The memory layout * of the array of coefficients must be exactly the natural one suggested by rows(), cols(), * outerStride(), innerStride(), and the RowMajorBit. This rules out expressions such as Diagonal, whose coefficients, * though referencable, do not have such a regular memory layout. * * See the comment on LvalueBit for an explanation of how LvalueBit and DirectAccessBit are mutually orthogonal. */ const unsigned int DirectAccessBit = 0x40; /** \deprecated \ingroup flags * * means the first coefficient packet is guaranteed to be aligned. * An expression cannot have the AlignedBit without the PacketAccessBit flag. * In other words, this means we are allow to perform an aligned packet access to the first element regardless * of the expression kind: * \code * expression.packet(0); * \endcode */ EIGEN_DEPRECATED const unsigned int AlignedBit = 0x80; const unsigned int NestByRefBit = 0x100; /** \ingroup flags * * for an expression, this means that the storage order * can be either row-major or column-major. * The precise choice will be decided at evaluation time or when * combined with other expressions. * \sa \blank \ref RowMajorBit, \ref TopicStorageOrders */ const unsigned int NoPreferredStorageOrderBit = 0x200; /** \ingroup flags * * Means that the underlying coefficients can be accessed through pointers to the sparse (un)compressed storage format, * that is, the expression provides: * \code inline const Scalar* valuePtr() const; inline const Index* innerIndexPtr() const; inline const Index* outerIndexPtr() const; inline const Index* innerNonZeroPtr() const; \endcode */ const unsigned int CompressedAccessBit = 0x400; // list of flags that are inherited by default const unsigned int HereditaryBits = RowMajorBit | EvalBeforeNestingBit; /** \defgroup enums Enumerations * \ingroup Core_Module * * Various enumerations used in %Eigen. Many of these are used as template parameters. */ /** \ingroup enums * Enum containing possible values for the \c Mode or \c UpLo parameter of * MatrixBase::selfadjointView() and MatrixBase::triangularView(), and selfadjoint solvers. */ enum UpLoType { /** View matrix as a lower triangular matrix. */ Lower=0x1, /** View matrix as an upper triangular matrix. */ Upper=0x2, /** %Matrix has ones on the diagonal; to be used in combination with #Lower or #Upper. */ UnitDiag=0x4, /** %Matrix has zeros on the diagonal; to be used in combination with #Lower or #Upper. */ ZeroDiag=0x8, /** View matrix as a lower triangular matrix with ones on the diagonal. */ UnitLower=UnitDiag|Lower, /** View matrix as an upper triangular matrix with ones on the diagonal. */ UnitUpper=UnitDiag|Upper, /** View matrix as a lower triangular matrix with zeros on the diagonal. */ StrictlyLower=ZeroDiag|Lower, /** View matrix as an upper triangular matrix with zeros on the diagonal. */ StrictlyUpper=ZeroDiag|Upper, /** Used in BandMatrix and SelfAdjointView to indicate that the matrix is self-adjoint. */ SelfAdjoint=0x10, /** Used to support symmetric, non-selfadjoint, complex matrices. */ Symmetric=0x20 }; /** \ingroup enums * Enum for indicating whether a buffer is aligned or not. */ enum AlignmentType { Unaligned=0, /**< Data pointer has no specific alignment. */ Aligned8=8, /**< Data pointer is aligned on a 8 bytes boundary. */ Aligned16=16, /**< Data pointer is aligned on a 16 bytes boundary. */ Aligned32=32, /**< Data pointer is aligned on a 32 bytes boundary. */ Aligned64=64, /**< Data pointer is aligned on a 64 bytes boundary. */ Aligned128=128, /**< Data pointer is aligned on a 128 bytes boundary. */ AlignedMask=255, Aligned=16, /**< \deprecated Synonym for Aligned16. */ #if EIGEN_MAX_ALIGN_BYTES==128 AlignedMax = Aligned128 #elif EIGEN_MAX_ALIGN_BYTES==64 AlignedMax = Aligned64 #elif EIGEN_MAX_ALIGN_BYTES==32 AlignedMax = Aligned32 #elif EIGEN_MAX_ALIGN_BYTES==16 AlignedMax = Aligned16 #elif EIGEN_MAX_ALIGN_BYTES==8 AlignedMax = Aligned8 #elif EIGEN_MAX_ALIGN_BYTES==0 AlignedMax = Unaligned #else #error Invalid value for EIGEN_MAX_ALIGN_BYTES #endif }; /** \ingroup enums * Enum containing possible values for the \p Direction parameter of * Reverse, PartialReduxExpr and VectorwiseOp. */ enum DirectionType { /** For Reverse, all columns are reversed; * for PartialReduxExpr and VectorwiseOp, act on columns. */ Vertical, /** For Reverse, all rows are reversed; * for PartialReduxExpr and VectorwiseOp, act on rows. */ Horizontal, /** For Reverse, both rows and columns are reversed; * not used for PartialReduxExpr and VectorwiseOp. */ BothDirections }; /** \internal \ingroup enums * Enum to specify how to traverse the entries of a matrix. */ enum TraversalType { /** \internal Default traversal, no vectorization, no index-based access */ DefaultTraversal, /** \internal No vectorization, use index-based access to have only one for loop instead of 2 nested loops */ LinearTraversal, /** \internal Equivalent to a slice vectorization for fixed-size matrices having good alignment * and good size */ InnerVectorizedTraversal, /** \internal Vectorization path using a single loop plus scalar loops for the * unaligned boundaries */ LinearVectorizedTraversal, /** \internal Generic vectorization path using one vectorized loop per row/column with some * scalar loops to handle the unaligned boundaries */ SliceVectorizedTraversal, /** \internal Special case to properly handle incompatible scalar types or other defecting cases*/ InvalidTraversal, /** \internal Evaluate all entries at once */ AllAtOnceTraversal }; /** \internal \ingroup enums * Enum to specify whether to unroll loops when traversing over the entries of a matrix. */ enum UnrollingType { /** \internal Do not unroll loops. */ NoUnrolling, /** \internal Unroll only the inner loop, but not the outer loop. */ InnerUnrolling, /** \internal Unroll both the inner and the outer loop. If there is only one loop, * because linear traversal is used, then unroll that loop. */ CompleteUnrolling }; /** \internal \ingroup enums * Enum to specify whether to use the default (built-in) implementation or the specialization. */ enum SpecializedType { Specialized, BuiltIn }; /** \ingroup enums * Enum containing possible values for the \p _Options template parameter of * Matrix, Array and BandMatrix. */ enum StorageOptions { /** Storage order is column major (see \ref TopicStorageOrders). */ ColMajor = 0, /** Storage order is row major (see \ref TopicStorageOrders). */ RowMajor = 0x1, // it is only a coincidence that this is equal to RowMajorBit -- don't rely on that /** Align the matrix itself if it is vectorizable fixed-size */ AutoAlign = 0, /** Don't require alignment for the matrix itself (the array of coefficients, if dynamically allocated, may still be requested to be aligned) */ // FIXME --- clarify the situation DontAlign = 0x2 }; /** \ingroup enums * Enum for specifying whether to apply or solve on the left or right. */ enum SideType { /** Apply transformation on the left. */ OnTheLeft = 1, /** Apply transformation on the right. */ OnTheRight = 2 }; /** \ingroup enums * Enum for specifying NaN-propagation behavior, e.g. for coeff-wise min/max. */ enum NaNPropagationOptions { /** Implementation defined behavior if NaNs are present. */ PropagateFast = 0, /** Always propagate NaNs. */ PropagateNaN, /** Always propagate not-NaNs. */ PropagateNumbers }; /* the following used to be written as: * * struct NoChange_t {}; * namespace { * EIGEN_UNUSED NoChange_t NoChange; * } * * on the ground that it feels dangerous to disambiguate overloaded functions on enum/integer types. * However, this leads to "variable declared but never referenced" warnings on Intel Composer XE, * and we do not know how to get rid of them (bug 450). */ enum NoChange_t { NoChange }; enum Sequential_t { Sequential }; enum Default_t { Default }; /** \internal \ingroup enums * Used in AmbiVector. */ enum AmbiVectorMode { IsDense = 0, IsSparse }; /** \ingroup enums * Used as template parameter in DenseCoeffBase and MapBase to indicate * which accessors should be provided. */ enum AccessorLevels { /** Read-only access via a member function. */ ReadOnlyAccessors, /** Read/write access via member functions. */ WriteAccessors, /** Direct read-only access to the coefficients. */ DirectAccessors, /** Direct read/write access to the coefficients. */ DirectWriteAccessors }; /** \ingroup enums * Enum with options to give to various decompositions. */ enum DecompositionOptions { /** \internal Not used (meant for LDLT?). */ Pivoting = 0x01, /** \internal Not used (meant for LDLT?). */ NoPivoting = 0x02, /** Used in JacobiSVD to indicate that the square matrix U is to be computed. */ ComputeFullU = 0x04, /** Used in JacobiSVD to indicate that the thin matrix U is to be computed. */ ComputeThinU = 0x08, /** Used in JacobiSVD to indicate that the square matrix V is to be computed. */ ComputeFullV = 0x10, /** Used in JacobiSVD to indicate that the thin matrix V is to be computed. */ ComputeThinV = 0x20, /** Used in SelfAdjointEigenSolver and GeneralizedSelfAdjointEigenSolver to specify * that only the eigenvalues are to be computed and not the eigenvectors. */ EigenvaluesOnly = 0x40, /** Used in SelfAdjointEigenSolver and GeneralizedSelfAdjointEigenSolver to specify * that both the eigenvalues and the eigenvectors are to be computed. */ ComputeEigenvectors = 0x80, /** \internal */ EigVecMask = EigenvaluesOnly | ComputeEigenvectors, /** Used in GeneralizedSelfAdjointEigenSolver to indicate that it should * solve the generalized eigenproblem \f$ Ax = \lambda B x \f$. */ Ax_lBx = 0x100, /** Used in GeneralizedSelfAdjointEigenSolver to indicate that it should * solve the generalized eigenproblem \f$ ABx = \lambda x \f$. */ ABx_lx = 0x200, /** Used in GeneralizedSelfAdjointEigenSolver to indicate that it should * solve the generalized eigenproblem \f$ BAx = \lambda x \f$. */ BAx_lx = 0x400, /** \internal */ GenEigMask = Ax_lBx | ABx_lx | BAx_lx }; /** \ingroup enums * Possible values for the \p QRPreconditioner template parameter of JacobiSVD. */ enum QRPreconditioners { /** Do not specify what is to be done if the SVD of a non-square matrix is asked for. */ NoQRPreconditioner, /** Use a QR decomposition without pivoting as the first step. */ HouseholderQRPreconditioner, /** Use a QR decomposition with column pivoting as the first step. */ ColPivHouseholderQRPreconditioner, /** Use a QR decomposition with full pivoting as the first step. */ FullPivHouseholderQRPreconditioner }; #ifdef Success #error The preprocessor symbol 'Success' is defined, possibly by the X11 header file X.h #endif /** \ingroup enums * Enum for reporting the status of a computation. */ enum ComputationInfo { /** Computation was successful. */ Success = 0, /** The provided data did not satisfy the prerequisites. */ NumericalIssue = 1, /** Iterative procedure did not converge. */ NoConvergence = 2, /** The inputs are invalid, or the algorithm has been improperly called. * When assertions are enabled, such errors trigger an assert. */ InvalidInput = 3 }; /** \ingroup enums * Enum used to specify how a particular transformation is stored in a matrix. * \sa Transform, Hyperplane::transform(). */ enum TransformTraits { /** Transformation is an isometry. */ Isometry = 0x1, /** Transformation is an affine transformation stored as a (Dim+1)^2 matrix whose last row is * assumed to be [0 ... 0 1]. */ Affine = 0x2, /** Transformation is an affine transformation stored as a (Dim) x (Dim+1) matrix. */ AffineCompact = 0x10 | Affine, /** Transformation is a general projective transformation stored as a (Dim+1)^2 matrix. */ Projective = 0x20 }; /** \internal \ingroup enums * Enum used to choose between implementation depending on the computer architecture. */ namespace Architecture { enum Type { Generic = 0x0, SSE = 0x1, AltiVec = 0x2, VSX = 0x3, NEON = 0x4, MSA = 0x5, SVE = 0x6, #if defined EIGEN_VECTORIZE_SSE Target = SSE #elif defined EIGEN_VECTORIZE_ALTIVEC Target = AltiVec #elif defined EIGEN_VECTORIZE_VSX Target = VSX #elif defined EIGEN_VECTORIZE_NEON Target = NEON #elif defined EIGEN_VECTORIZE_SVE Target = SVE #elif defined EIGEN_VECTORIZE_MSA Target = MSA #else Target = Generic #endif }; } /** \internal \ingroup enums * Enum used as template parameter in Product and product evaluators. */ enum ProductImplType { DefaultProduct=0, LazyProduct, AliasFreeProduct, CoeffBasedProductMode, LazyCoeffBasedProductMode, OuterProduct, InnerProduct, GemvProduct, GemmProduct }; /** \internal \ingroup enums * Enum used in experimental parallel implementation. */ enum Action {GetAction, SetAction}; /** The type used to identify a dense storage. */ struct Dense {}; /** The type used to identify a general sparse storage. */ struct Sparse {}; /** The type used to identify a general solver (factored) storage. */ struct SolverStorage {}; /** The type used to identify a permutation storage. */ struct PermutationStorage {}; /** The type used to identify a permutation storage. */ struct TranspositionsStorage {}; /** The type used to identify a matrix expression */ struct MatrixXpr {}; /** The type used to identify an array expression */ struct ArrayXpr {}; // An evaluator must define its shape. By default, it can be one of the following: struct DenseShape { static std::string debugName() { return "DenseShape"; } }; struct SolverShape { static std::string debugName() { return "SolverShape"; } }; struct HomogeneousShape { static std::string debugName() { return "HomogeneousShape"; } }; struct DiagonalShape { static std::string debugName() { return "DiagonalShape"; } }; struct BandShape { static std::string debugName() { return "BandShape"; } }; struct TriangularShape { static std::string debugName() { return "TriangularShape"; } }; struct SelfAdjointShape { static std::string debugName() { return "SelfAdjointShape"; } }; struct PermutationShape { static std::string debugName() { return "PermutationShape"; } }; struct TranspositionsShape { static std::string debugName() { return "TranspositionsShape"; } }; struct SparseShape { static std::string debugName() { return "SparseShape"; } }; namespace internal { // random access iterators based on coeff*() accessors. struct IndexBased {}; // evaluator based on iterators to access coefficients. struct IteratorBased {}; /** \internal * Constants for comparison functors */ enum ComparisonName { cmp_EQ = 0, cmp_LT = 1, cmp_LE = 2, cmp_UNORD = 3, cmp_NEQ = 4, cmp_GT = 5, cmp_GE = 6 }; } // end namespace internal } // end namespace Eigen #endif // EIGEN_CONSTANTS_H RcppEigen/inst/include/Eigen/src/Core/util/ForwardDeclarations.h0000644000176200001440000003630314567757725024355 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2007-2010 Benoit Jacob // Copyright (C) 2008-2009 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_FORWARDDECLARATIONS_H #define EIGEN_FORWARDDECLARATIONS_H namespace Eigen { namespace internal { template struct traits; // here we say once and for all that traits == traits // When constness must affect traits, it has to be constness on template parameters on which T itself depends. // For example, traits > != traits >, but // traits > == traits > template struct traits : traits {}; template struct has_direct_access { enum { ret = (traits::Flags & DirectAccessBit) ? 1 : 0 }; }; template struct accessors_level { enum { has_direct_access = (traits::Flags & DirectAccessBit) ? 1 : 0, has_write_access = (traits::Flags & LvalueBit) ? 1 : 0, value = has_direct_access ? (has_write_access ? DirectWriteAccessors : DirectAccessors) : (has_write_access ? WriteAccessors : ReadOnlyAccessors) }; }; template struct evaluator_traits; template< typename T> struct evaluator; } // end namespace internal template struct NumTraits; template struct EigenBase; template class DenseBase; template class PlainObjectBase; template class DenseCoeffsBase; template class Matrix; template class MatrixBase; template class ArrayBase; template class Flagged; template class StorageBase > class NoAlias; template class NestByValue; template class ForceAlignedAccess; template class SwapWrapper; template class Block; template class IndexedView; template class Reshaped; template class VectorBlock; template class Transpose; template class Conjugate; template class CwiseNullaryOp; template class CwiseUnaryOp; template class CwiseUnaryView; template class CwiseBinaryOp; template class CwiseTernaryOp; template class Solve; template class Inverse; template class Product; template class DiagonalBase; template class DiagonalWrapper; template class DiagonalMatrix; template class DiagonalProduct; template class Diagonal; template class PermutationMatrix; template class Transpositions; template class PermutationBase; template class TranspositionsBase; template class PermutationWrapper; template class TranspositionsWrapper; template::has_write_access ? WriteAccessors : ReadOnlyAccessors > class MapBase; template class Stride; template class InnerStride; template class OuterStride; template > class Map; template class RefBase; template,OuterStride<> >::type > class Ref; template class TriangularBase; template class TriangularView; template class SelfAdjointView; template class SparseView; template class WithFormat; template struct CommaInitializer; template class ReturnByValue; template class ArrayWrapper; template class MatrixWrapper; template class SolverBase; template class InnerIterator; namespace internal { template class generic_randaccess_stl_iterator; template class pointer_based_stl_iterator; template class subvector_stl_iterator; template class subvector_stl_reverse_iterator; template struct kernel_retval_base; template struct kernel_retval; template struct image_retval_base; template struct image_retval; } // end namespace internal namespace internal { template class BandMatrix; } namespace internal { template struct product_type; template struct EnableIf; /** \internal * \class product_evaluator * Products need their own evaluator with more template arguments allowing for * easier partial template specializations. */ template< typename T, int ProductTag = internal::product_type::ret, typename LhsShape = typename evaluator_traits::Shape, typename RhsShape = typename evaluator_traits::Shape, typename LhsScalar = typename traits::Scalar, typename RhsScalar = typename traits::Scalar > struct product_evaluator; } template::value> struct ProductReturnType; // this is a workaround for sun CC template struct LazyProductReturnType; namespace internal { // Provides scalar/packet-wise product and product with accumulation // with optional conjugation of the arguments. template struct conj_helper; template struct scalar_sum_op; template struct scalar_difference_op; template struct scalar_conj_product_op; template struct scalar_min_op; template struct scalar_max_op; template struct scalar_opposite_op; template struct scalar_conjugate_op; template struct scalar_real_op; template struct scalar_imag_op; template struct scalar_abs_op; template struct scalar_abs2_op; template struct scalar_absolute_difference_op; template struct scalar_sqrt_op; template struct scalar_rsqrt_op; template struct scalar_exp_op; template struct scalar_log_op; template struct scalar_cos_op; template struct scalar_sin_op; template struct scalar_acos_op; template struct scalar_asin_op; template struct scalar_tan_op; template struct scalar_inverse_op; template struct scalar_square_op; template struct scalar_cube_op; template struct scalar_cast_op; template struct scalar_random_op; template struct scalar_constant_op; template struct scalar_identity_op; template struct scalar_sign_op; template struct scalar_pow_op; template struct scalar_hypot_op; template struct scalar_product_op; template struct scalar_quotient_op; // SpecialFunctions module template struct scalar_lgamma_op; template struct scalar_digamma_op; template struct scalar_erf_op; template struct scalar_erfc_op; template struct scalar_ndtri_op; template struct scalar_igamma_op; template struct scalar_igammac_op; template struct scalar_zeta_op; template struct scalar_betainc_op; // Bessel functions in SpecialFunctions module template struct scalar_bessel_i0_op; template struct scalar_bessel_i0e_op; template struct scalar_bessel_i1_op; template struct scalar_bessel_i1e_op; template struct scalar_bessel_j0_op; template struct scalar_bessel_y0_op; template struct scalar_bessel_j1_op; template struct scalar_bessel_y1_op; template struct scalar_bessel_k0_op; template struct scalar_bessel_k0e_op; template struct scalar_bessel_k1_op; template struct scalar_bessel_k1e_op; } // end namespace internal struct IOFormat; // Array module template class Array; template class Select; template class PartialReduxExpr; template class VectorwiseOp; template class Replicate; template class Reverse; template class FullPivLU; template class PartialPivLU; namespace internal { template struct inverse_impl; } template class HouseholderQR; template class ColPivHouseholderQR; template class FullPivHouseholderQR; template class CompleteOrthogonalDecomposition; template class SVDBase; template class JacobiSVD; template class BDCSVD; template class LLT; template class LDLT; template class HouseholderSequence; template class JacobiRotation; // Geometry module: template class RotationBase; template class Cross; template class QuaternionBase; template class Rotation2D; template class AngleAxis; template class Translation; template class AlignedBox; template class Quaternion; template class Transform; template class ParametrizedLine; template class Hyperplane; template class UniformScaling; template class Homogeneous; // Sparse module: template class SparseMatrixBase; // MatrixFunctions module template struct MatrixExponentialReturnValue; template class MatrixFunctionReturnValue; template class MatrixSquareRootReturnValue; template class MatrixLogarithmReturnValue; template class MatrixPowerReturnValue; template class MatrixComplexPowerReturnValue; namespace internal { template struct stem_function { typedef std::complex::Real> ComplexScalar; typedef ComplexScalar type(ComplexScalar, int); }; } } // end namespace Eigen #endif // EIGEN_FORWARDDECLARATIONS_H RcppEigen/inst/include/Eigen/src/Core/util/StaticAssert.h0000644000176200001440000002466414567757725023040 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008 Gael Guennebaud // Copyright (C) 2008 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_STATIC_ASSERT_H #define EIGEN_STATIC_ASSERT_H /* Some notes on Eigen's static assertion mechanism: * * - in EIGEN_STATIC_ASSERT(CONDITION,MSG) the parameter CONDITION must be a compile time boolean * expression, and MSG an enum listed in struct internal::static_assertion * * - define EIGEN_NO_STATIC_ASSERT to disable them (and save compilation time) * in that case, the static assertion is converted to the following runtime assert: * eigen_assert(CONDITION && "MSG") * * - currently EIGEN_STATIC_ASSERT can only be used in function scope * */ #ifndef EIGEN_STATIC_ASSERT #ifndef EIGEN_NO_STATIC_ASSERT #if EIGEN_MAX_CPP_VER>=11 && (__has_feature(cxx_static_assert) || (EIGEN_COMP_CXXVER >= 11) || (EIGEN_COMP_MSVC >= 1600)) // if native static_assert is enabled, let's use it #define EIGEN_STATIC_ASSERT(X,MSG) static_assert(X,#MSG); #else // not CXX0X namespace Eigen { namespace internal { template struct static_assertion {}; template<> struct static_assertion { enum { YOU_TRIED_CALLING_A_VECTOR_METHOD_ON_A_MATRIX=1, YOU_MIXED_VECTORS_OF_DIFFERENT_SIZES=1, YOU_MIXED_MATRICES_OF_DIFFERENT_SIZES=1, THIS_METHOD_IS_ONLY_FOR_VECTORS_OF_A_SPECIFIC_SIZE=1, THIS_METHOD_IS_ONLY_FOR_MATRICES_OF_A_SPECIFIC_SIZE=1, THIS_METHOD_IS_ONLY_FOR_OBJECTS_OF_A_SPECIFIC_SIZE=1, OUT_OF_RANGE_ACCESS=1, YOU_MADE_A_PROGRAMMING_MISTAKE=1, EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT=1, EIGEN_INTERNAL_COMPILATION_ERROR_OR_YOU_MADE_A_PROGRAMMING_MISTAKE=1, YOU_CALLED_A_FIXED_SIZE_METHOD_ON_A_DYNAMIC_SIZE_MATRIX_OR_VECTOR=1, YOU_CALLED_A_DYNAMIC_SIZE_METHOD_ON_A_FIXED_SIZE_MATRIX_OR_VECTOR=1, UNALIGNED_LOAD_AND_STORE_OPERATIONS_UNIMPLEMENTED_ON_ALTIVEC=1, THIS_FUNCTION_IS_NOT_FOR_INTEGER_NUMERIC_TYPES=1, FLOATING_POINT_ARGUMENT_PASSED__INTEGER_WAS_EXPECTED=1, NUMERIC_TYPE_MUST_BE_REAL=1, COEFFICIENT_WRITE_ACCESS_TO_SELFADJOINT_NOT_SUPPORTED=1, WRITING_TO_TRIANGULAR_PART_WITH_UNIT_DIAGONAL_IS_NOT_SUPPORTED=1, THIS_METHOD_IS_ONLY_FOR_FIXED_SIZE=1, INVALID_MATRIX_PRODUCT=1, INVALID_VECTOR_VECTOR_PRODUCT__IF_YOU_WANTED_A_DOT_OR_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTIONS=1, INVALID_MATRIX_PRODUCT__IF_YOU_WANTED_A_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTION=1, YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY=1, THIS_METHOD_IS_ONLY_FOR_COLUMN_MAJOR_MATRICES=1, THIS_METHOD_IS_ONLY_FOR_ROW_MAJOR_MATRICES=1, INVALID_MATRIX_TEMPLATE_PARAMETERS=1, INVALID_MATRIXBASE_TEMPLATE_PARAMETERS=1, BOTH_MATRICES_MUST_HAVE_THE_SAME_STORAGE_ORDER=1, THIS_METHOD_IS_ONLY_FOR_DIAGONAL_MATRIX=1, THE_MATRIX_OR_EXPRESSION_THAT_YOU_PASSED_DOES_NOT_HAVE_THE_EXPECTED_TYPE=1, THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_WITH_DIRECT_MEMORY_ACCESS_SUCH_AS_MAP_OR_PLAIN_MATRICES=1, YOU_ALREADY_SPECIFIED_THIS_STRIDE=1, INVALID_STORAGE_ORDER_FOR_THIS_VECTOR_EXPRESSION=1, THE_BRACKET_OPERATOR_IS_ONLY_FOR_VECTORS__USE_THE_PARENTHESIS_OPERATOR_INSTEAD=1, PACKET_ACCESS_REQUIRES_TO_HAVE_INNER_STRIDE_FIXED_TO_1=1, THIS_METHOD_IS_ONLY_FOR_SPECIFIC_TRANSFORMATIONS=1, YOU_CANNOT_MIX_ARRAYS_AND_MATRICES=1, YOU_PERFORMED_AN_INVALID_TRANSFORMATION_CONVERSION=1, THIS_EXPRESSION_IS_NOT_A_LVALUE__IT_IS_READ_ONLY=1, YOU_ARE_TRYING_TO_USE_AN_INDEX_BASED_ACCESSOR_ON_AN_EXPRESSION_THAT_DOES_NOT_SUPPORT_THAT=1, THIS_METHOD_IS_ONLY_FOR_1x1_EXPRESSIONS=1, THIS_METHOD_IS_ONLY_FOR_INNER_OR_LAZY_PRODUCTS=1, THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_OF_BOOL=1, THIS_METHOD_IS_ONLY_FOR_ARRAYS_NOT_MATRICES=1, YOU_PASSED_A_ROW_VECTOR_BUT_A_COLUMN_VECTOR_WAS_EXPECTED=1, YOU_PASSED_A_COLUMN_VECTOR_BUT_A_ROW_VECTOR_WAS_EXPECTED=1, THE_INDEX_TYPE_MUST_BE_A_SIGNED_TYPE=1, THE_STORAGE_ORDER_OF_BOTH_SIDES_MUST_MATCH=1, OBJECT_ALLOCATED_ON_STACK_IS_TOO_BIG=1, IMPLICIT_CONVERSION_TO_SCALAR_IS_FOR_INNER_PRODUCT_ONLY=1, STORAGE_LAYOUT_DOES_NOT_MATCH=1, EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT__INVALID_COST_VALUE=1, THIS_COEFFICIENT_ACCESSOR_TAKING_ONE_ACCESS_IS_ONLY_FOR_EXPRESSIONS_ALLOWING_LINEAR_ACCESS=1, MATRIX_FREE_CONJUGATE_GRADIENT_IS_COMPATIBLE_WITH_UPPER_UNION_LOWER_MODE_ONLY=1, THIS_TYPE_IS_NOT_SUPPORTED=1, STORAGE_KIND_MUST_MATCH=1, STORAGE_INDEX_MUST_MATCH=1, CHOLMOD_SUPPORTS_DOUBLE_PRECISION_ONLY=1, SELFADJOINTVIEW_ACCEPTS_UPPER_AND_LOWER_MODE_ONLY=1, INVALID_TEMPLATE_PARAMETER=1, GPU_TENSOR_CONTRACTION_DOES_NOT_SUPPORT_OUTPUT_KERNELS=1, THE_ARRAY_SIZE_SHOULD_EQUAL_WITH_PACKET_SIZE=1 }; }; } // end namespace internal } // end namespace Eigen // Specialized implementation for MSVC to avoid "conditional // expression is constant" warnings. This implementation doesn't // appear to work under GCC, hence the multiple implementations. #if EIGEN_COMP_MSVC #define EIGEN_STATIC_ASSERT(CONDITION,MSG) \ {Eigen::internal::static_assertion::MSG;} #else // In some cases clang interprets bool(CONDITION) as function declaration #define EIGEN_STATIC_ASSERT(CONDITION,MSG) \ if (Eigen::internal::static_assertion(CONDITION)>::MSG) {} #endif #endif // not CXX0X #else // EIGEN_NO_STATIC_ASSERT #define EIGEN_STATIC_ASSERT(CONDITION,MSG) eigen_assert((CONDITION) && #MSG); #endif // EIGEN_NO_STATIC_ASSERT #endif // EIGEN_STATIC_ASSERT // static assertion failing if the type \a TYPE is not a vector type #define EIGEN_STATIC_ASSERT_VECTOR_ONLY(TYPE) \ EIGEN_STATIC_ASSERT(TYPE::IsVectorAtCompileTime, \ YOU_TRIED_CALLING_A_VECTOR_METHOD_ON_A_MATRIX) // static assertion failing if the type \a TYPE is not fixed-size #define EIGEN_STATIC_ASSERT_FIXED_SIZE(TYPE) \ EIGEN_STATIC_ASSERT(TYPE::SizeAtCompileTime!=Eigen::Dynamic, \ YOU_CALLED_A_FIXED_SIZE_METHOD_ON_A_DYNAMIC_SIZE_MATRIX_OR_VECTOR) // static assertion failing if the type \a TYPE is not dynamic-size #define EIGEN_STATIC_ASSERT_DYNAMIC_SIZE(TYPE) \ EIGEN_STATIC_ASSERT(TYPE::SizeAtCompileTime==Eigen::Dynamic, \ YOU_CALLED_A_DYNAMIC_SIZE_METHOD_ON_A_FIXED_SIZE_MATRIX_OR_VECTOR) // static assertion failing if the type \a TYPE is not a vector type of the given size #define EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(TYPE, SIZE) \ EIGEN_STATIC_ASSERT(TYPE::IsVectorAtCompileTime && TYPE::SizeAtCompileTime==SIZE, \ THIS_METHOD_IS_ONLY_FOR_VECTORS_OF_A_SPECIFIC_SIZE) // static assertion failing if the type \a TYPE is not a vector type of the given size #define EIGEN_STATIC_ASSERT_MATRIX_SPECIFIC_SIZE(TYPE, ROWS, COLS) \ EIGEN_STATIC_ASSERT(TYPE::RowsAtCompileTime==ROWS && TYPE::ColsAtCompileTime==COLS, \ THIS_METHOD_IS_ONLY_FOR_MATRICES_OF_A_SPECIFIC_SIZE) // static assertion failing if the two vector expression types are not compatible (same fixed-size or dynamic size) #define EIGEN_STATIC_ASSERT_SAME_VECTOR_SIZE(TYPE0,TYPE1) \ EIGEN_STATIC_ASSERT( \ (int(TYPE0::SizeAtCompileTime)==Eigen::Dynamic \ || int(TYPE1::SizeAtCompileTime)==Eigen::Dynamic \ || int(TYPE0::SizeAtCompileTime)==int(TYPE1::SizeAtCompileTime)),\ YOU_MIXED_VECTORS_OF_DIFFERENT_SIZES) #define EIGEN_PREDICATE_SAME_MATRIX_SIZE(TYPE0,TYPE1) \ ( \ (int(Eigen::internal::size_of_xpr_at_compile_time::ret)==0 && int(Eigen::internal::size_of_xpr_at_compile_time::ret)==0) \ || (\ (int(TYPE0::RowsAtCompileTime)==Eigen::Dynamic \ || int(TYPE1::RowsAtCompileTime)==Eigen::Dynamic \ || int(TYPE0::RowsAtCompileTime)==int(TYPE1::RowsAtCompileTime)) \ && (int(TYPE0::ColsAtCompileTime)==Eigen::Dynamic \ || int(TYPE1::ColsAtCompileTime)==Eigen::Dynamic \ || int(TYPE0::ColsAtCompileTime)==int(TYPE1::ColsAtCompileTime))\ ) \ ) #define EIGEN_STATIC_ASSERT_NON_INTEGER(TYPE) \ EIGEN_STATIC_ASSERT(!Eigen::NumTraits::IsInteger, THIS_FUNCTION_IS_NOT_FOR_INTEGER_NUMERIC_TYPES) // static assertion failing if it is guaranteed at compile-time that the two matrix expression types have different sizes #define EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(TYPE0,TYPE1) \ EIGEN_STATIC_ASSERT( \ EIGEN_PREDICATE_SAME_MATRIX_SIZE(TYPE0,TYPE1),\ YOU_MIXED_MATRICES_OF_DIFFERENT_SIZES) #define EIGEN_STATIC_ASSERT_SIZE_1x1(TYPE) \ EIGEN_STATIC_ASSERT((TYPE::RowsAtCompileTime == 1 || TYPE::RowsAtCompileTime == Eigen::Dynamic) && \ (TYPE::ColsAtCompileTime == 1 || TYPE::ColsAtCompileTime == Eigen::Dynamic), \ THIS_METHOD_IS_ONLY_FOR_1x1_EXPRESSIONS) #define EIGEN_STATIC_ASSERT_LVALUE(Derived) \ EIGEN_STATIC_ASSERT(Eigen::internal::is_lvalue::value, \ THIS_EXPRESSION_IS_NOT_A_LVALUE__IT_IS_READ_ONLY) #define EIGEN_STATIC_ASSERT_ARRAYXPR(Derived) \ EIGEN_STATIC_ASSERT((Eigen::internal::is_same::XprKind, ArrayXpr>::value), \ THIS_METHOD_IS_ONLY_FOR_ARRAYS_NOT_MATRICES) #define EIGEN_STATIC_ASSERT_SAME_XPR_KIND(Derived1, Derived2) \ EIGEN_STATIC_ASSERT((Eigen::internal::is_same::XprKind, \ typename Eigen::internal::traits::XprKind \ >::value), \ YOU_CANNOT_MIX_ARRAYS_AND_MATRICES) // Check that a cost value is positive, and that is stay within a reasonable range // TODO this check could be enabled for internal debugging only #define EIGEN_INTERNAL_CHECK_COST_VALUE(C) \ EIGEN_STATIC_ASSERT((C)>=0 && (C)<=HugeCost*HugeCost, EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT__INVALID_COST_VALUE); #endif // EIGEN_STATIC_ASSERT_H RcppEigen/inst/include/Eigen/src/Core/util/ReshapedHelper.h0000644000176200001440000000263014567757725023307 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2017 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_RESHAPED_HELPER_H #define EIGEN_RESHAPED_HELPER_H namespace Eigen { enum AutoSize_t { AutoSize }; const int AutoOrder = 2; namespace internal { template struct get_compiletime_reshape_size { enum { value = get_fixed_value::value }; }; template Index get_runtime_reshape_size(SizeType size, Index /*other*/, Index /*total*/) { return internal::get_runtime_value(size); } template struct get_compiletime_reshape_size { enum { other_size = get_fixed_value::value, value = (TotalSize==Dynamic || other_size==Dynamic) ? Dynamic : TotalSize / other_size }; }; inline Index get_runtime_reshape_size(AutoSize_t /*size*/, Index other, Index total) { return total/other; } template struct get_compiletime_reshape_order { enum { value = Order == AutoOrder ? Flags & RowMajorBit : Order }; }; } } // end namespace Eigen #endif // EIGEN_RESHAPED_HELPER_H RcppEigen/inst/include/Eigen/src/Core/util/DisableStupidWarnings.h0000755000176200001440000001157114567757725024670 0ustar liggesusers#ifndef EIGEN_WARNINGS_DISABLED #define EIGEN_WARNINGS_DISABLED #ifdef _MSC_VER // 4100 - unreferenced formal parameter (occurred e.g. in aligned_allocator::destroy(pointer p)) // 4101 - unreferenced local variable // 4181 - qualifier applied to reference type ignored // 4211 - nonstandard extension used : redefined extern to static // 4244 - 'argument' : conversion from 'type1' to 'type2', possible loss of data // 4273 - QtAlignedMalloc, inconsistent DLL linkage // 4324 - structure was padded due to declspec(align()) // 4503 - decorated name length exceeded, name was truncated // 4512 - assignment operator could not be generated // 4522 - 'class' : multiple assignment operators specified // 4700 - uninitialized local variable 'xyz' used // 4714 - function marked as __forceinline not inlined // 4717 - 'function' : recursive on all control paths, function will cause runtime stack overflow // 4800 - 'type' : forcing value to bool 'true' or 'false' (performance warning) #ifndef EIGEN_PERMANENTLY_DISABLE_STUPID_WARNINGS #pragma warning( push ) #endif #pragma warning( disable : 4100 4101 4181 4211 4244 4273 4324 4503 4512 4522 4700 4714 4717 4800) #elif defined __INTEL_COMPILER // 2196 - routine is both "inline" and "noinline" ("noinline" assumed) // ICC 12 generates this warning even without any inline keyword, when defining class methods 'inline' i.e. inside of class body // typedef that may be a reference type. // 279 - controlling expression is constant // ICC 12 generates this warning on assert(constant_expression_depending_on_template_params) and frankly this is a legitimate use case. // 1684 - conversion from pointer to same-sized integral type (potential portability problem) // 2259 - non-pointer conversion from "Eigen::Index={ptrdiff_t={long}}" to "int" may lose significant bits #ifndef EIGEN_PERMANENTLY_DISABLE_STUPID_WARNINGS #pragma warning push #endif #pragma warning disable 2196 279 1684 2259 #elif defined __clang__ // -Wconstant-logical-operand - warning: use of logical && with constant operand; switch to bitwise & or remove constant // this is really a stupid warning as it warns on compile-time expressions involving enums // #ifndef EIGEN_PERMANENTLY_DISABLE_STUPID_WARNINGS // #pragma clang diagnostic push // #endif // #pragma clang diagnostic ignored "-Wconstant-logical-operand" // #if __clang_major__ >= 3 && __clang_minor__ >= 5 // #pragma clang diagnostic ignored "-Wabsolute-value" // #endif // #if __clang_major__ >= 10 // #pragma clang diagnostic ignored "-Wimplicit-int-float-conversion" // #endif // #if ( defined(__ALTIVEC__) || defined(__VSX__) ) && __cplusplus < 201103L // // warning: generic selections are a C11-specific feature // // ignoring warnings thrown at vec_ctf in Altivec/PacketMath.h // #pragma clang diagnostic ignored "-Wc11-extensions" // #endif #elif defined __GNUC__ // #if (!defined(EIGEN_PERMANENTLY_DISABLE_STUPID_WARNINGS)) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) // #pragma GCC diagnostic push // #endif // // g++ warns about local variables shadowing member functions, which is too strict // #pragma GCC diagnostic ignored "-Wshadow" // #if __GNUC__ == 4 && __GNUC_MINOR__ < 8 // // Until g++-4.7 there are warnings when comparing unsigned int vs 0, even in templated functions: // #pragma GCC diagnostic ignored "-Wtype-limits" // #endif // #if __GNUC__>=6 // #pragma GCC diagnostic ignored "-Wignored-attributes" // #endif // #if __GNUC__==7 // // See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89325 // #pragma GCC diagnostic ignored "-Wattributes" // #endif #endif #if defined __NVCC__ #pragma diag_suppress boolean_controlling_expr_is_constant // Disable the "statement is unreachable" message #pragma diag_suppress code_is_unreachable // Disable the "dynamic initialization in unreachable code" message #pragma diag_suppress initialization_not_reachable // Disable the "invalid error number" message that we get with older versions of nvcc #pragma diag_suppress 1222 // Disable the "calling a __host__ function from a __host__ __device__ function is not allowed" messages (yes, there are many of them and they seem to change with every version of the compiler) #pragma diag_suppress 2527 #pragma diag_suppress 2529 #pragma diag_suppress 2651 #pragma diag_suppress 2653 #pragma diag_suppress 2668 #pragma diag_suppress 2669 #pragma diag_suppress 2670 #pragma diag_suppress 2671 #pragma diag_suppress 2735 #pragma diag_suppress 2737 #pragma diag_suppress 2739 #endif #else // warnings already disabled: # ifndef EIGEN_WARNINGS_DISABLED_2 # define EIGEN_WARNINGS_DISABLED_2 # elif defined(EIGEN_INTERNAL_DEBUGGING) # error "Do not include \"DisableStupidWarnings.h\" recursively more than twice!" # endif #endif // not EIGEN_WARNINGS_DISABLED RcppEigen/inst/include/Eigen/src/Core/util/ConfigureVectorization.h0000644000176200001440000004664414567757725025133 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2018 Gael Guennebaud // Copyright (C) 2020, Arm Limited and Contributors // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_CONFIGURE_VECTORIZATION_H #define EIGEN_CONFIGURE_VECTORIZATION_H //------------------------------------------------------------------------------------------ // Static and dynamic alignment control // // The main purpose of this section is to define EIGEN_MAX_ALIGN_BYTES and EIGEN_MAX_STATIC_ALIGN_BYTES // as the maximal boundary in bytes on which dynamically and statically allocated data may be alignment respectively. // The values of EIGEN_MAX_ALIGN_BYTES and EIGEN_MAX_STATIC_ALIGN_BYTES can be specified by the user. If not, // a default value is automatically computed based on architecture, compiler, and OS. // // This section also defines macros EIGEN_ALIGN_TO_BOUNDARY(N) and the shortcuts EIGEN_ALIGN{8,16,32,_MAX} // to be used to declare statically aligned buffers. //------------------------------------------------------------------------------------------ /* EIGEN_ALIGN_TO_BOUNDARY(n) forces data to be n-byte aligned. This is used to satisfy SIMD requirements. * However, we do that EVEN if vectorization (EIGEN_VECTORIZE) is disabled, * so that vectorization doesn't affect binary compatibility. * * If we made alignment depend on whether or not EIGEN_VECTORIZE is defined, it would be impossible to link * vectorized and non-vectorized code. * * FIXME: this code can be cleaned up once we switch to proper C++11 only. */ #if (defined EIGEN_CUDACC) #define EIGEN_ALIGN_TO_BOUNDARY(n) __align__(n) #define EIGEN_ALIGNOF(x) __alignof(x) #elif EIGEN_HAS_ALIGNAS #define EIGEN_ALIGN_TO_BOUNDARY(n) alignas(n) #define EIGEN_ALIGNOF(x) alignof(x) #elif EIGEN_COMP_GNUC || EIGEN_COMP_PGI || EIGEN_COMP_IBM || EIGEN_COMP_ARM #define EIGEN_ALIGN_TO_BOUNDARY(n) __attribute__((aligned(n))) #define EIGEN_ALIGNOF(x) __alignof(x) #elif EIGEN_COMP_MSVC #define EIGEN_ALIGN_TO_BOUNDARY(n) __declspec(align(n)) #define EIGEN_ALIGNOF(x) __alignof(x) #elif EIGEN_COMP_SUNCC // FIXME not sure about this one: #define EIGEN_ALIGN_TO_BOUNDARY(n) __attribute__((aligned(n))) #define EIGEN_ALIGNOF(x) __alignof(x) #else #error Please tell me what is the equivalent of alignas(n) and alignof(x) for your compiler #endif // If the user explicitly disable vectorization, then we also disable alignment #if defined(EIGEN_DONT_VECTORIZE) #if defined(EIGEN_GPUCC) // GPU code is always vectorized and requires memory alignment for // statically allocated buffers. #define EIGEN_IDEAL_MAX_ALIGN_BYTES 16 #else #define EIGEN_IDEAL_MAX_ALIGN_BYTES 0 #endif #elif defined(__AVX512F__) // 64 bytes static alignment is preferred only if really required #define EIGEN_IDEAL_MAX_ALIGN_BYTES 64 #elif defined(__AVX__) // 32 bytes static alignment is preferred only if really required #define EIGEN_IDEAL_MAX_ALIGN_BYTES 32 #else #define EIGEN_IDEAL_MAX_ALIGN_BYTES 16 #endif // EIGEN_MIN_ALIGN_BYTES defines the minimal value for which the notion of explicit alignment makes sense #define EIGEN_MIN_ALIGN_BYTES 16 // Defined the boundary (in bytes) on which the data needs to be aligned. Note // that unless EIGEN_ALIGN is defined and not equal to 0, the data may not be // aligned at all regardless of the value of this #define. #if (defined(EIGEN_DONT_ALIGN_STATICALLY) || defined(EIGEN_DONT_ALIGN)) && defined(EIGEN_MAX_STATIC_ALIGN_BYTES) && EIGEN_MAX_STATIC_ALIGN_BYTES>0 #error EIGEN_MAX_STATIC_ALIGN_BYTES and EIGEN_DONT_ALIGN[_STATICALLY] are both defined with EIGEN_MAX_STATIC_ALIGN_BYTES!=0. Use EIGEN_MAX_STATIC_ALIGN_BYTES=0 as a synonym of EIGEN_DONT_ALIGN_STATICALLY. #endif // EIGEN_DONT_ALIGN_STATICALLY and EIGEN_DONT_ALIGN are deprecated // They imply EIGEN_MAX_STATIC_ALIGN_BYTES=0 #if defined(EIGEN_DONT_ALIGN_STATICALLY) || defined(EIGEN_DONT_ALIGN) #ifdef EIGEN_MAX_STATIC_ALIGN_BYTES #undef EIGEN_MAX_STATIC_ALIGN_BYTES #endif #define EIGEN_MAX_STATIC_ALIGN_BYTES 0 #endif #ifndef EIGEN_MAX_STATIC_ALIGN_BYTES // Try to automatically guess what is the best default value for EIGEN_MAX_STATIC_ALIGN_BYTES // 16 byte alignment is only useful for vectorization. Since it affects the ABI, we need to enable // 16 byte alignment on all platforms where vectorization might be enabled. In theory we could always // enable alignment, but it can be a cause of problems on some platforms, so we just disable it in // certain common platform (compiler+architecture combinations) to avoid these problems. // Only static alignment is really problematic (relies on nonstandard compiler extensions), // try to keep heap alignment even when we have to disable static alignment. #if EIGEN_COMP_GNUC && !(EIGEN_ARCH_i386_OR_x86_64 || EIGEN_ARCH_ARM_OR_ARM64 || EIGEN_ARCH_PPC || EIGEN_ARCH_IA64 || EIGEN_ARCH_MIPS) #define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 1 #elif EIGEN_ARCH_ARM_OR_ARM64 && EIGEN_COMP_GNUC_STRICT && EIGEN_GNUC_AT_MOST(4, 6) // Old versions of GCC on ARM, at least 4.4, were once seen to have buggy static alignment support. // Not sure which version fixed it, hopefully it doesn't affect 4.7, which is still somewhat in use. // 4.8 and newer seem definitely unaffected. #define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 1 #else #define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 0 #endif // static alignment is completely disabled with GCC 3, Sun Studio, and QCC/QNX #if !EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT \ && !EIGEN_GCC3_OR_OLDER \ && !EIGEN_COMP_SUNCC \ && !EIGEN_OS_QNX #define EIGEN_ARCH_WANTS_STACK_ALIGNMENT 1 #else #define EIGEN_ARCH_WANTS_STACK_ALIGNMENT 0 #endif #if EIGEN_ARCH_WANTS_STACK_ALIGNMENT #define EIGEN_MAX_STATIC_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES #else #define EIGEN_MAX_STATIC_ALIGN_BYTES 0 #endif #endif // If EIGEN_MAX_ALIGN_BYTES is defined, then it is considered as an upper bound for EIGEN_MAX_STATIC_ALIGN_BYTES #if defined(EIGEN_MAX_ALIGN_BYTES) && EIGEN_MAX_ALIGN_BYTES0 is the true test whether we want to align arrays on the stack or not. // It takes into account both the user choice to explicitly enable/disable alignment (by setting EIGEN_MAX_STATIC_ALIGN_BYTES) // and the architecture config (EIGEN_ARCH_WANTS_STACK_ALIGNMENT). // Henceforth, only EIGEN_MAX_STATIC_ALIGN_BYTES should be used. // Shortcuts to EIGEN_ALIGN_TO_BOUNDARY #define EIGEN_ALIGN8 EIGEN_ALIGN_TO_BOUNDARY(8) #define EIGEN_ALIGN16 EIGEN_ALIGN_TO_BOUNDARY(16) #define EIGEN_ALIGN32 EIGEN_ALIGN_TO_BOUNDARY(32) #define EIGEN_ALIGN64 EIGEN_ALIGN_TO_BOUNDARY(64) #if EIGEN_MAX_STATIC_ALIGN_BYTES>0 #define EIGEN_ALIGN_MAX EIGEN_ALIGN_TO_BOUNDARY(EIGEN_MAX_STATIC_ALIGN_BYTES) #else #define EIGEN_ALIGN_MAX #endif // Dynamic alignment control #if defined(EIGEN_DONT_ALIGN) && defined(EIGEN_MAX_ALIGN_BYTES) && EIGEN_MAX_ALIGN_BYTES>0 #error EIGEN_MAX_ALIGN_BYTES and EIGEN_DONT_ALIGN are both defined with EIGEN_MAX_ALIGN_BYTES!=0. Use EIGEN_MAX_ALIGN_BYTES=0 as a synonym of EIGEN_DONT_ALIGN. #endif #ifdef EIGEN_DONT_ALIGN #ifdef EIGEN_MAX_ALIGN_BYTES #undef EIGEN_MAX_ALIGN_BYTES #endif #define EIGEN_MAX_ALIGN_BYTES 0 #elif !defined(EIGEN_MAX_ALIGN_BYTES) #define EIGEN_MAX_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES #endif #if EIGEN_IDEAL_MAX_ALIGN_BYTES > EIGEN_MAX_ALIGN_BYTES #define EIGEN_DEFAULT_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES #else #define EIGEN_DEFAULT_ALIGN_BYTES EIGEN_MAX_ALIGN_BYTES #endif #ifndef EIGEN_UNALIGNED_VECTORIZE #define EIGEN_UNALIGNED_VECTORIZE 1 #endif //---------------------------------------------------------------------- // if alignment is disabled, then disable vectorization. Note: EIGEN_MAX_ALIGN_BYTES is the proper check, it takes into // account both the user's will (EIGEN_MAX_ALIGN_BYTES,EIGEN_DONT_ALIGN) and our own platform checks #if EIGEN_MAX_ALIGN_BYTES==0 #ifndef EIGEN_DONT_VECTORIZE #define EIGEN_DONT_VECTORIZE #endif #endif // The following (except #include and _M_IX86_FP ??) can likely be // removed as gcc 4.1 and msvc 2008 are not supported anyways. #if EIGEN_COMP_MSVC #include // for _aligned_malloc -- need it regardless of whether vectorization is enabled #if (EIGEN_COMP_MSVC >= 1500) // 2008 or later // a user reported that in 64-bit mode, MSVC doesn't care to define _M_IX86_FP. #if (defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) || EIGEN_ARCH_x86_64 #define EIGEN_SSE2_ON_MSVC_2008_OR_LATER #endif #endif #else #if (defined __SSE2__) && ( (!EIGEN_COMP_GNUC) || EIGEN_COMP_ICC || EIGEN_GNUC_AT_LEAST(4,2) ) #define EIGEN_SSE2_ON_NON_MSVC_BUT_NOT_OLD_GCC #endif #endif #if !(defined(EIGEN_DONT_VECTORIZE) || defined(EIGEN_GPUCC)) #if defined (EIGEN_SSE2_ON_NON_MSVC_BUT_NOT_OLD_GCC) || defined(EIGEN_SSE2_ON_MSVC_2008_OR_LATER) // Defines symbols for compile-time detection of which instructions are // used. // EIGEN_VECTORIZE_YY is defined if and only if the instruction set YY is used #define EIGEN_VECTORIZE #define EIGEN_VECTORIZE_SSE #define EIGEN_VECTORIZE_SSE2 // Detect sse3/ssse3/sse4: // gcc and icc defines __SSE3__, ... // there is no way to know about this on msvc. You can define EIGEN_VECTORIZE_SSE* if you // want to force the use of those instructions with msvc. #ifdef __SSE3__ #define EIGEN_VECTORIZE_SSE3 #endif #ifdef __SSSE3__ #define EIGEN_VECTORIZE_SSSE3 #endif #ifdef __SSE4_1__ #define EIGEN_VECTORIZE_SSE4_1 #endif #ifdef __SSE4_2__ #define EIGEN_VECTORIZE_SSE4_2 #endif #ifdef __AVX__ #ifndef EIGEN_USE_SYCL #define EIGEN_VECTORIZE_AVX #endif #define EIGEN_VECTORIZE_SSE3 #define EIGEN_VECTORIZE_SSSE3 #define EIGEN_VECTORIZE_SSE4_1 #define EIGEN_VECTORIZE_SSE4_2 #endif #ifdef __AVX2__ #ifndef EIGEN_USE_SYCL #define EIGEN_VECTORIZE_AVX2 #define EIGEN_VECTORIZE_AVX #endif #define EIGEN_VECTORIZE_SSE3 #define EIGEN_VECTORIZE_SSSE3 #define EIGEN_VECTORIZE_SSE4_1 #define EIGEN_VECTORIZE_SSE4_2 #endif #if defined(__FMA__) || (EIGEN_COMP_MSVC && defined(__AVX2__)) // MSVC does not expose a switch dedicated for FMA // For MSVC, AVX2 => FMA #define EIGEN_VECTORIZE_FMA #endif #if defined(__AVX512F__) #ifndef EIGEN_VECTORIZE_FMA #if EIGEN_COMP_GNUC #error Please add -mfma to your compiler flags: compiling with -mavx512f alone without SSE/AVX FMA is not supported (bug 1638). #else #error Please enable FMA in your compiler flags (e.g. -mfma): compiling with AVX512 alone without SSE/AVX FMA is not supported (bug 1638). #endif #endif #ifndef EIGEN_USE_SYCL #define EIGEN_VECTORIZE_AVX512 #define EIGEN_VECTORIZE_AVX2 #define EIGEN_VECTORIZE_AVX #endif #define EIGEN_VECTORIZE_FMA #define EIGEN_VECTORIZE_SSE3 #define EIGEN_VECTORIZE_SSSE3 #define EIGEN_VECTORIZE_SSE4_1 #define EIGEN_VECTORIZE_SSE4_2 #ifndef EIGEN_USE_SYCL #ifdef __AVX512DQ__ #define EIGEN_VECTORIZE_AVX512DQ #endif #ifdef __AVX512ER__ #define EIGEN_VECTORIZE_AVX512ER #endif #ifdef __AVX512BF16__ #define EIGEN_VECTORIZE_AVX512BF16 #endif #endif #endif // Disable AVX support on broken xcode versions #if defined(__apple_build_version__) && (__apple_build_version__ == 11000033 ) && ( __MAC_OS_X_VERSION_MIN_REQUIRED == 101500 ) // A nasty bug in the clang compiler shipped with xcode in a common compilation situation // when XCode 11.0 and Mac deployment target macOS 10.15 is https://trac.macports.org/ticket/58776#no1 #ifdef EIGEN_VECTORIZE_AVX #undef EIGEN_VECTORIZE_AVX #warning "Disabling AVX support: clang compiler shipped with XCode 11.[012] generates broken assembly with -macosx-version-min=10.15 and AVX enabled. " #ifdef EIGEN_VECTORIZE_AVX2 #undef EIGEN_VECTORIZE_AVX2 #endif #ifdef EIGEN_VECTORIZE_FMA #undef EIGEN_VECTORIZE_FMA #endif #ifdef EIGEN_VECTORIZE_AVX512 #undef EIGEN_VECTORIZE_AVX512 #endif #ifdef EIGEN_VECTORIZE_AVX512DQ #undef EIGEN_VECTORIZE_AVX512DQ #endif #ifdef EIGEN_VECTORIZE_AVX512ER #undef EIGEN_VECTORIZE_AVX512ER #endif #endif // NOTE: Confirmed test failures in XCode 11.0, and XCode 11.2 with -macosx-version-min=10.15 and AVX // NOTE using -macosx-version-min=10.15 with Xcode 11.0 results in runtime segmentation faults in many tests, 11.2 produce core dumps in 3 tests // NOTE using -macosx-version-min=10.14 produces functioning and passing tests in all cases // NOTE __clang_version__ "11.0.0 (clang-1100.0.33.8)" XCode 11.0 <- Produces many segfault and core dumping tests // with -macosx-version-min=10.15 and AVX // NOTE __clang_version__ "11.0.0 (clang-1100.0.33.12)" XCode 11.2 <- Produces 3 core dumping tests with // -macosx-version-min=10.15 and AVX #endif // include files // This extern "C" works around a MINGW-w64 compilation issue // https://sourceforge.net/tracker/index.php?func=detail&aid=3018394&group_id=202880&atid=983354 // In essence, intrin.h is included by windows.h and also declares intrinsics (just as emmintrin.h etc. below do). // However, intrin.h uses an extern "C" declaration, and g++ thus complains of duplicate declarations // with conflicting linkage. The linkage for intrinsics doesn't matter, but at that stage the compiler doesn't know; // so, to avoid compile errors when windows.h is included after Eigen/Core, ensure intrinsics are extern "C" here too. // notice that since these are C headers, the extern "C" is theoretically needed anyways. extern "C" { // In theory we should only include immintrin.h and not the other *mmintrin.h header files directly. // Doing so triggers some issues with ICC. However old gcc versions seems to not have this file, thus: #if EIGEN_COMP_ICC >= 1110 #include #else #include #include #include #ifdef EIGEN_VECTORIZE_SSE3 #include #endif #ifdef EIGEN_VECTORIZE_SSSE3 #include #endif #ifdef EIGEN_VECTORIZE_SSE4_1 #include #endif #ifdef EIGEN_VECTORIZE_SSE4_2 #include #endif #if defined(EIGEN_VECTORIZE_AVX) || defined(EIGEN_VECTORIZE_AVX512) #include #endif #endif } // end extern "C" #elif defined __VSX__ #define EIGEN_VECTORIZE #define EIGEN_VECTORIZE_VSX #include // We need to #undef all these ugly tokens defined in // => use __vector instead of vector #undef bool #undef vector #undef pixel #elif defined __ALTIVEC__ #define EIGEN_VECTORIZE #define EIGEN_VECTORIZE_ALTIVEC #include // We need to #undef all these ugly tokens defined in // => use __vector instead of vector #undef bool #undef vector #undef pixel #elif ((defined __ARM_NEON) || (defined __ARM_NEON__)) && !(defined EIGEN_ARM64_USE_SVE) #define EIGEN_VECTORIZE #define EIGEN_VECTORIZE_NEON #include // We currently require SVE to be enabled explicitly via EIGEN_ARM64_USE_SVE and // will not select the backend automatically #elif (defined __ARM_FEATURE_SVE) && (defined EIGEN_ARM64_USE_SVE) #define EIGEN_VECTORIZE #define EIGEN_VECTORIZE_SVE #include // Since we depend on knowing SVE vector lengths at compile-time, we need // to ensure a fixed lengths is set #if defined __ARM_FEATURE_SVE_BITS #define EIGEN_ARM64_SVE_VL __ARM_FEATURE_SVE_BITS #else #error "Eigen requires a fixed SVE lector length but EIGEN_ARM64_SVE_VL is not set." #endif #elif (defined __s390x__ && defined __VEC__) #define EIGEN_VECTORIZE #define EIGEN_VECTORIZE_ZVECTOR #include #elif defined __mips_msa // Limit MSA optimizations to little-endian CPUs for now. // TODO: Perhaps, eventually support MSA optimizations on big-endian CPUs? #if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) #if defined(__LP64__) #define EIGEN_MIPS_64 #else #define EIGEN_MIPS_32 #endif #define EIGEN_VECTORIZE #define EIGEN_VECTORIZE_MSA #include #endif #endif #endif // Following the Arm ACLE arm_neon.h should also include arm_fp16.h but not all // compilers seem to follow this. We therefore include it explicitly. // See also: https://bugs.llvm.org/show_bug.cgi?id=47955 #if defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) #include #endif #if defined(__F16C__) && (!defined(EIGEN_GPUCC) && (!defined(EIGEN_COMP_CLANG) || EIGEN_COMP_CLANG>=380)) // We can use the optimized fp16 to float and float to fp16 conversion routines #define EIGEN_HAS_FP16_C #if defined(EIGEN_COMP_CLANG) // Workaround for clang: The FP16C intrinsics for clang are included by // immintrin.h, as opposed to emmintrin.h as suggested by Intel: // https://software.intel.com/sites/landingpage/IntrinsicsGuide/#othertechs=FP16C&expand=1711 #include #endif #endif #if defined EIGEN_CUDACC #define EIGEN_VECTORIZE_GPU #include #if EIGEN_CUDA_SDK_VER >= 70500 #define EIGEN_HAS_CUDA_FP16 #endif #endif #if defined(EIGEN_HAS_CUDA_FP16) #include #include #endif #if defined(EIGEN_HIPCC) #define EIGEN_VECTORIZE_GPU #include #define EIGEN_HAS_HIP_FP16 #include #endif /** \brief Namespace containing all symbols from the %Eigen library. */ namespace Eigen { inline static const char *SimdInstructionSetsInUse(void) { #if defined(EIGEN_VECTORIZE_AVX512) return "AVX512, FMA, AVX2, AVX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2"; #elif defined(EIGEN_VECTORIZE_AVX) return "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2"; #elif defined(EIGEN_VECTORIZE_SSE4_2) return "SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2"; #elif defined(EIGEN_VECTORIZE_SSE4_1) return "SSE, SSE2, SSE3, SSSE3, SSE4.1"; #elif defined(EIGEN_VECTORIZE_SSSE3) return "SSE, SSE2, SSE3, SSSE3"; #elif defined(EIGEN_VECTORIZE_SSE3) return "SSE, SSE2, SSE3"; #elif defined(EIGEN_VECTORIZE_SSE2) return "SSE, SSE2"; #elif defined(EIGEN_VECTORIZE_ALTIVEC) return "AltiVec"; #elif defined(EIGEN_VECTORIZE_VSX) return "VSX"; #elif defined(EIGEN_VECTORIZE_NEON) return "ARM NEON"; #elif defined(EIGEN_VECTORIZE_SVE) return "ARM SVE"; #elif defined(EIGEN_VECTORIZE_ZVECTOR) return "S390X ZVECTOR"; #elif defined(EIGEN_VECTORIZE_MSA) return "MIPS MSA"; #else return "None"; #endif } } // end namespace Eigen #endif // EIGEN_CONFIGURE_VECTORIZATION_H RcppEigen/inst/include/Eigen/src/Core/util/XprHelper.h0000644000176200001440000010566214567757725022336 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008 Gael Guennebaud // Copyright (C) 2006-2008 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_XPRHELPER_H #define EIGEN_XPRHELPER_H // just a workaround because GCC seems to not really like empty structs // FIXME: gcc 4.3 generates bad code when strict-aliasing is enabled // so currently we simply disable this optimization for gcc 4.3 #if EIGEN_COMP_GNUC && !EIGEN_GNUC_AT(4,3) #define EIGEN_EMPTY_STRUCT_CTOR(X) \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE X() {} \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE X(const X& ) {} #else #define EIGEN_EMPTY_STRUCT_CTOR(X) #endif namespace Eigen { namespace internal { template EIGEN_DEVICE_FUNC inline IndexDest convert_index(const IndexSrc& idx) { // for sizeof(IndexDest)>=sizeof(IndexSrc) compilers should be able to optimize this away: eigen_internal_assert(idx <= NumTraits::highest() && "Index value to big for target type"); return IndexDest(idx); } // true if T can be considered as an integral index (i.e., and integral type or enum) template struct is_valid_index_type { enum { value = #if EIGEN_HAS_TYPE_TRAITS internal::is_integral::value || std::is_enum::value #elif EIGEN_COMP_MSVC internal::is_integral::value || __is_enum(T) #else // without C++11, we use is_convertible to Index instead of is_integral in order to treat enums as Index. internal::is_convertible::value && !internal::is_same::value && !is_same::value #endif }; }; // true if both types are not valid index types template struct valid_indexed_view_overload { enum { value = !(internal::is_valid_index_type::value && internal::is_valid_index_type::value) }; }; // promote_scalar_arg is an helper used in operation between an expression and a scalar, like: // expression * scalar // Its role is to determine how the type T of the scalar operand should be promoted given the scalar type ExprScalar of the given expression. // The IsSupported template parameter must be provided by the caller as: internal::has_ReturnType >::value using the proper order for ExprScalar and T. // Then the logic is as follows: // - if the operation is natively supported as defined by IsSupported, then the scalar type is not promoted, and T is returned. // - otherwise, NumTraits::Literal is returned if T is implicitly convertible to NumTraits::Literal AND that this does not imply a float to integer conversion. // - otherwise, ExprScalar is returned if T is implicitly convertible to ExprScalar AND that this does not imply a float to integer conversion. // - In all other cases, the promoted type is not defined, and the respective operation is thus invalid and not available (SFINAE). template struct promote_scalar_arg; template struct promote_scalar_arg { typedef T type; }; // Recursively check safe conversion to PromotedType, and then ExprScalar if they are different. template::value, bool IsSafe = NumTraits::IsInteger || !NumTraits::IsInteger> struct promote_scalar_arg_unsupported; // Start recursion with NumTraits::Literal template struct promote_scalar_arg : promote_scalar_arg_unsupported::Literal> {}; // We found a match! template struct promote_scalar_arg_unsupported { typedef PromotedType type; }; // No match, but no real-to-integer issues, and ExprScalar and current PromotedType are different, // so let's try to promote to ExprScalar template struct promote_scalar_arg_unsupported : promote_scalar_arg_unsupported {}; // Unsafe real-to-integer, let's stop. template struct promote_scalar_arg_unsupported {}; // T is not even convertible to ExprScalar, let's stop. template struct promote_scalar_arg_unsupported {}; //classes inheriting no_assignment_operator don't generate a default operator=. class no_assignment_operator { private: no_assignment_operator& operator=(const no_assignment_operator&); protected: EIGEN_DEFAULT_COPY_CONSTRUCTOR(no_assignment_operator) EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(no_assignment_operator) }; /** \internal return the index type with the largest number of bits */ template struct promote_index_type { typedef typename conditional<(sizeof(I1)::type type; }; /** \internal If the template parameter Value is Dynamic, this class is just a wrapper around a T variable that * can be accessed using value() and setValue(). * Otherwise, this class is an empty structure and value() just returns the template parameter Value. */ template class variable_if_dynamic { public: EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(variable_if_dynamic) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit variable_if_dynamic(T v) { EIGEN_ONLY_USED_FOR_DEBUG(v); eigen_assert(v == T(Value)); } EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR T value() { return T(Value); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR operator T() const { return T(Value); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void setValue(T v) const { EIGEN_ONLY_USED_FOR_DEBUG(v); eigen_assert(v == T(Value)); } }; template class variable_if_dynamic { T m_value; public: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit variable_if_dynamic(T value = 0) EIGEN_NO_THROW : m_value(value) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T value() const { return m_value; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE operator T() const { return m_value; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void setValue(T value) { m_value = value; } }; /** \internal like variable_if_dynamic but for DynamicIndex */ template class variable_if_dynamicindex { public: EIGEN_EMPTY_STRUCT_CTOR(variable_if_dynamicindex) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit variable_if_dynamicindex(T v) { EIGEN_ONLY_USED_FOR_DEBUG(v); eigen_assert(v == T(Value)); } EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR T value() { return T(Value); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void setValue(T) {} }; template class variable_if_dynamicindex { T m_value; EIGEN_DEVICE_FUNC variable_if_dynamicindex() { eigen_assert(false); } public: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit variable_if_dynamicindex(T value) : m_value(value) {} EIGEN_DEVICE_FUNC T EIGEN_STRONG_INLINE value() const { return m_value; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void setValue(T value) { m_value = value; } }; template struct functor_traits { enum { Cost = 10, PacketAccess = false, IsRepeatable = false }; }; template struct packet_traits; template struct unpacket_traits; template::size)==0 || is_same::half>::value> struct find_best_packet_helper; template< int Size, typename PacketType> struct find_best_packet_helper { typedef PacketType type; }; template struct find_best_packet_helper { typedef typename find_best_packet_helper::half>::type type; }; template struct find_best_packet { typedef typename find_best_packet_helper::type>::type type; }; #if EIGEN_MAX_STATIC_ALIGN_BYTES>0 template struct compute_default_alignment_helper { enum { value = 0 }; }; template struct compute_default_alignment_helper // Match { enum { value = AlignmentBytes }; }; template struct compute_default_alignment_helper // Try-half { // current packet too large, try with an half-packet enum { value = compute_default_alignment_helper::value }; }; #else // If static alignment is disabled, no need to bother. // This also avoids a division by zero in "bool Match = bool((ArrayBytes%AlignmentBytes)==0)" template struct compute_default_alignment_helper { enum { value = 0 }; }; #endif template struct compute_default_alignment { enum { value = compute_default_alignment_helper::value }; }; template struct compute_default_alignment { enum { value = EIGEN_MAX_ALIGN_BYTES }; }; template class make_proper_matrix_type { enum { IsColVector = _Cols==1 && _Rows!=1, IsRowVector = _Rows==1 && _Cols!=1, Options = IsColVector ? (_Options | ColMajor) & ~RowMajor : IsRowVector ? (_Options | RowMajor) & ~ColMajor : _Options }; public: typedef Matrix<_Scalar, _Rows, _Cols, Options, _MaxRows, _MaxCols> type; }; template class compute_matrix_flags { enum { row_major_bit = Options&RowMajor ? RowMajorBit : 0 }; public: // FIXME currently we still have to handle DirectAccessBit at the expression level to handle DenseCoeffsBase<> // and then propagate this information to the evaluator's flags. // However, I (Gael) think that DirectAccessBit should only matter at the evaluation stage. enum { ret = DirectAccessBit | LvalueBit | NestByRefBit | row_major_bit }; }; template struct size_at_compile_time { enum { ret = (_Rows==Dynamic || _Cols==Dynamic) ? Dynamic : _Rows * _Cols }; }; template struct size_of_xpr_at_compile_time { enum { ret = size_at_compile_time::RowsAtCompileTime,traits::ColsAtCompileTime>::ret }; }; /* plain_matrix_type : the difference from eval is that plain_matrix_type is always a plain matrix type, * whereas eval is a const reference in the case of a matrix */ template::StorageKind> struct plain_matrix_type; template struct plain_matrix_type_dense; template struct plain_matrix_type { typedef typename plain_matrix_type_dense::XprKind, traits::Flags>::type type; }; template struct plain_matrix_type { typedef typename T::PlainObject type; }; template struct plain_matrix_type_dense { typedef Matrix::Scalar, traits::RowsAtCompileTime, traits::ColsAtCompileTime, AutoAlign | (Flags&RowMajorBit ? RowMajor : ColMajor), traits::MaxRowsAtCompileTime, traits::MaxColsAtCompileTime > type; }; template struct plain_matrix_type_dense { typedef Array::Scalar, traits::RowsAtCompileTime, traits::ColsAtCompileTime, AutoAlign | (Flags&RowMajorBit ? RowMajor : ColMajor), traits::MaxRowsAtCompileTime, traits::MaxColsAtCompileTime > type; }; /* eval : the return type of eval(). For matrices, this is just a const reference * in order to avoid a useless copy */ template::StorageKind> struct eval; template struct eval { typedef typename plain_matrix_type::type type; // typedef typename T::PlainObject type; // typedef T::Matrix::Scalar, // traits::RowsAtCompileTime, // traits::ColsAtCompileTime, // AutoAlign | (traits::Flags&RowMajorBit ? RowMajor : ColMajor), // traits::MaxRowsAtCompileTime, // traits::MaxColsAtCompileTime // > type; }; template struct eval { typedef typename plain_matrix_type::type type; }; // for matrices, no need to evaluate, just use a const reference to avoid a useless copy template struct eval, Dense> { typedef const Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols>& type; }; template struct eval, Dense> { typedef const Array<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols>& type; }; /* similar to plain_matrix_type, but using the evaluator's Flags */ template::StorageKind> struct plain_object_eval; template struct plain_object_eval { typedef typename plain_matrix_type_dense::XprKind, evaluator::Flags>::type type; }; /* plain_matrix_type_column_major : same as plain_matrix_type but guaranteed to be column-major */ template struct plain_matrix_type_column_major { enum { Rows = traits::RowsAtCompileTime, Cols = traits::ColsAtCompileTime, MaxRows = traits::MaxRowsAtCompileTime, MaxCols = traits::MaxColsAtCompileTime }; typedef Matrix::Scalar, Rows, Cols, (MaxRows==1&&MaxCols!=1) ? RowMajor : ColMajor, MaxRows, MaxCols > type; }; /* plain_matrix_type_row_major : same as plain_matrix_type but guaranteed to be row-major */ template struct plain_matrix_type_row_major { enum { Rows = traits::RowsAtCompileTime, Cols = traits::ColsAtCompileTime, MaxRows = traits::MaxRowsAtCompileTime, MaxCols = traits::MaxColsAtCompileTime }; typedef Matrix::Scalar, Rows, Cols, (MaxCols==1&&MaxRows!=1) ? ColMajor : RowMajor, MaxRows, MaxCols > type; }; /** \internal The reference selector for template expressions. The idea is that we don't * need to use references for expressions since they are light weight proxy * objects which should generate no copying overhead. */ template struct ref_selector { typedef typename conditional< bool(traits::Flags & NestByRefBit), T const&, const T >::type type; typedef typename conditional< bool(traits::Flags & NestByRefBit), T &, T >::type non_const_type; }; /** \internal Adds the const qualifier on the value-type of T2 if and only if T1 is a const type */ template struct transfer_constness { typedef typename conditional< bool(internal::is_const::value), typename internal::add_const_on_value_type::type, T2 >::type type; }; // However, we still need a mechanism to detect whether an expression which is evaluated multiple time // has to be evaluated into a temporary. // That's the purpose of this new nested_eval helper: /** \internal Determines how a given expression should be nested when evaluated multiple times. * For example, when you do a * (b+c), Eigen will determine how the expression b+c should be * evaluated into the bigger product expression. The choice is between nesting the expression b+c as-is, or * evaluating that expression b+c into a temporary variable d, and nest d so that the resulting expression is * a*d. Evaluating can be beneficial for example if every coefficient access in the resulting expression causes * many coefficient accesses in the nested expressions -- as is the case with matrix product for example. * * \tparam T the type of the expression being nested. * \tparam n the number of coefficient accesses in the nested expression for each coefficient access in the bigger expression. * \tparam PlainObject the type of the temporary if needed. */ template::type> struct nested_eval { enum { ScalarReadCost = NumTraits::Scalar>::ReadCost, CoeffReadCost = evaluator::CoeffReadCost, // NOTE What if an evaluator evaluate itself into a temporary? // Then CoeffReadCost will be small (e.g., 1) but we still have to evaluate, especially if n>1. // This situation is already taken care by the EvalBeforeNestingBit flag, which is turned ON // for all evaluator creating a temporary. This flag is then propagated by the parent evaluators. // Another solution could be to count the number of temps? NAsInteger = n == Dynamic ? HugeCost : n, CostEval = (NAsInteger+1) * ScalarReadCost + CoeffReadCost, CostNoEval = NAsInteger * CoeffReadCost, Evaluate = (int(evaluator::Flags) & EvalBeforeNestingBit) || (int(CostEval) < int(CostNoEval)) }; typedef typename conditional::type>::type type; }; template EIGEN_DEVICE_FUNC inline T* const_cast_ptr(const T* ptr) { return const_cast(ptr); } template::XprKind> struct dense_xpr_base { /* dense_xpr_base should only ever be used on dense expressions, thus falling either into the MatrixXpr or into the ArrayXpr cases */ }; template struct dense_xpr_base { typedef MatrixBase type; }; template struct dense_xpr_base { typedef ArrayBase type; }; template::XprKind, typename StorageKind = typename traits::StorageKind> struct generic_xpr_base; template struct generic_xpr_base { typedef typename dense_xpr_base::type type; }; template struct cast_return_type { typedef typename XprType::Scalar CurrentScalarType; typedef typename remove_all::type _CastType; typedef typename _CastType::Scalar NewScalarType; typedef typename conditional::value, const XprType&,CastType>::type type; }; template struct promote_storage_type; template struct promote_storage_type { typedef A ret; }; template struct promote_storage_type { typedef A ret; }; template struct promote_storage_type { typedef A ret; }; /** \internal Specify the "storage kind" of applying a coefficient-wise * binary operations between two expressions of kinds A and B respectively. * The template parameter Functor permits to specialize the resulting storage kind wrt to * the functor. * The default rules are as follows: * \code * A op A -> A * A op dense -> dense * dense op B -> dense * sparse op dense -> sparse * dense op sparse -> sparse * \endcode */ template struct cwise_promote_storage_type; template struct cwise_promote_storage_type { typedef A ret; }; template struct cwise_promote_storage_type { typedef Dense ret; }; template struct cwise_promote_storage_type { typedef Dense ret; }; template struct cwise_promote_storage_type { typedef Dense ret; }; template struct cwise_promote_storage_type { typedef Sparse ret; }; template struct cwise_promote_storage_type { typedef Sparse ret; }; template struct cwise_promote_storage_order { enum { value = LhsOrder }; }; template struct cwise_promote_storage_order { enum { value = RhsOrder }; }; template struct cwise_promote_storage_order { enum { value = LhsOrder }; }; template struct cwise_promote_storage_order { enum { value = Order }; }; /** \internal Specify the "storage kind" of multiplying an expression of kind A with kind B. * The template parameter ProductTag permits to specialize the resulting storage kind wrt to * some compile-time properties of the product: GemmProduct, GemvProduct, OuterProduct, InnerProduct. * The default rules are as follows: * \code * K * K -> K * dense * K -> dense * K * dense -> dense * diag * K -> K * K * diag -> K * Perm * K -> K * K * Perm -> K * \endcode */ template struct product_promote_storage_type; template struct product_promote_storage_type { typedef A ret;}; template struct product_promote_storage_type { typedef Dense ret;}; template struct product_promote_storage_type { typedef Dense ret; }; template struct product_promote_storage_type { typedef Dense ret; }; template struct product_promote_storage_type { typedef A ret; }; template struct product_promote_storage_type { typedef B ret; }; template struct product_promote_storage_type { typedef Dense ret; }; template struct product_promote_storage_type { typedef Dense ret; }; template struct product_promote_storage_type { typedef A ret; }; template struct product_promote_storage_type { typedef B ret; }; template struct product_promote_storage_type { typedef Dense ret; }; template struct product_promote_storage_type { typedef Dense ret; }; /** \internal gives the plain matrix or array type to store a row/column/diagonal of a matrix type. * \tparam Scalar optional parameter allowing to pass a different scalar type than the one of the MatrixType. */ template struct plain_row_type { typedef Matrix MatrixRowType; typedef Array ArrayRowType; typedef typename conditional< is_same< typename traits::XprKind, MatrixXpr >::value, MatrixRowType, ArrayRowType >::type type; }; template struct plain_col_type { typedef Matrix MatrixColType; typedef Array ArrayColType; typedef typename conditional< is_same< typename traits::XprKind, MatrixXpr >::value, MatrixColType, ArrayColType >::type type; }; template struct plain_diag_type { enum { diag_size = EIGEN_SIZE_MIN_PREFER_DYNAMIC(ExpressionType::RowsAtCompileTime, ExpressionType::ColsAtCompileTime), max_diag_size = EIGEN_SIZE_MIN_PREFER_FIXED(ExpressionType::MaxRowsAtCompileTime, ExpressionType::MaxColsAtCompileTime) }; typedef Matrix MatrixDiagType; typedef Array ArrayDiagType; typedef typename conditional< is_same< typename traits::XprKind, MatrixXpr >::value, MatrixDiagType, ArrayDiagType >::type type; }; template struct plain_constant_type { enum { Options = (traits::Flags&RowMajorBit)?RowMajor:0 }; typedef Array::RowsAtCompileTime, traits::ColsAtCompileTime, Options, traits::MaxRowsAtCompileTime,traits::MaxColsAtCompileTime> array_type; typedef Matrix::RowsAtCompileTime, traits::ColsAtCompileTime, Options, traits::MaxRowsAtCompileTime,traits::MaxColsAtCompileTime> matrix_type; typedef CwiseNullaryOp, const typename conditional::XprKind, MatrixXpr >::value, matrix_type, array_type>::type > type; }; template struct is_lvalue { enum { value = (!bool(is_const::value)) && bool(traits::Flags & LvalueBit) }; }; template struct is_diagonal { enum { ret = false }; }; template struct is_diagonal > { enum { ret = true }; }; template struct is_diagonal > { enum { ret = true }; }; template struct is_diagonal > { enum { ret = true }; }; template struct is_identity { enum { value = false }; }; template struct is_identity, T> > { enum { value = true }; }; template struct glue_shapes; template<> struct glue_shapes { typedef TriangularShape type; }; template struct possibly_same_dense { enum { value = has_direct_access::ret && has_direct_access::ret && is_same::value }; }; template EIGEN_DEVICE_FUNC bool is_same_dense(const T1 &mat1, const T2 &mat2, typename enable_if::value>::type * = 0) { return (mat1.data()==mat2.data()) && (mat1.innerStride()==mat2.innerStride()) && (mat1.outerStride()==mat2.outerStride()); } template EIGEN_DEVICE_FUNC bool is_same_dense(const T1 &, const T2 &, typename enable_if::value>::type * = 0) { return false; } // Internal helper defining the cost of a scalar division for the type T. // The default heuristic can be specialized for each scalar type and architecture. template struct scalar_div_cost { enum { value = 8*NumTraits::MulCost }; }; template struct scalar_div_cost, Vectorized> { enum { value = 2*scalar_div_cost::value + 6*NumTraits::MulCost + 3*NumTraits::AddCost }; }; template struct scalar_div_cost::type> { enum { value = 24 }; }; template struct scalar_div_cost::type> { enum { value = 21 }; }; #ifdef EIGEN_DEBUG_ASSIGN std::string demangle_traversal(int t) { if(t==DefaultTraversal) return "DefaultTraversal"; if(t==LinearTraversal) return "LinearTraversal"; if(t==InnerVectorizedTraversal) return "InnerVectorizedTraversal"; if(t==LinearVectorizedTraversal) return "LinearVectorizedTraversal"; if(t==SliceVectorizedTraversal) return "SliceVectorizedTraversal"; return "?"; } std::string demangle_unrolling(int t) { if(t==NoUnrolling) return "NoUnrolling"; if(t==InnerUnrolling) return "InnerUnrolling"; if(t==CompleteUnrolling) return "CompleteUnrolling"; return "?"; } std::string demangle_flags(int f) { std::string res; if(f&RowMajorBit) res += " | RowMajor"; if(f&PacketAccessBit) res += " | Packet"; if(f&LinearAccessBit) res += " | Linear"; if(f&LvalueBit) res += " | Lvalue"; if(f&DirectAccessBit) res += " | Direct"; if(f&NestByRefBit) res += " | NestByRef"; if(f&NoPreferredStorageOrderBit) res += " | NoPreferredStorageOrderBit"; return res; } #endif } // end namespace internal /** \class ScalarBinaryOpTraits * \ingroup Core_Module * * \brief Determines whether the given binary operation of two numeric types is allowed and what the scalar return type is. * * This class permits to control the scalar return type of any binary operation performed on two different scalar types through (partial) template specializations. * * For instance, let \c U1, \c U2 and \c U3 be three user defined scalar types for which most operations between instances of \c U1 and \c U2 returns an \c U3. * You can let %Eigen knows that by defining: \code template struct ScalarBinaryOpTraits { typedef U3 ReturnType; }; template struct ScalarBinaryOpTraits { typedef U3 ReturnType; }; \endcode * You can then explicitly disable some particular operations to get more explicit error messages: \code template<> struct ScalarBinaryOpTraits > {}; \endcode * Or customize the return type for individual operation: \code template<> struct ScalarBinaryOpTraits > { typedef U1 ReturnType; }; \endcode * * By default, the following generic combinations are supported:
ScalarAScalarBBinaryOpReturnTypeNote
\c T \c T \c * \c T
\c NumTraits::Real \c T \c * \c T Only if \c NumTraits::IsComplex
\c T \c NumTraits::Real \c * \c T Only if \c NumTraits::IsComplex
* * \sa CwiseBinaryOp */ template > struct ScalarBinaryOpTraits #ifndef EIGEN_PARSED_BY_DOXYGEN // for backward compatibility, use the hints given by the (deprecated) internal::scalar_product_traits class. : internal::scalar_product_traits #endif // EIGEN_PARSED_BY_DOXYGEN {}; template struct ScalarBinaryOpTraits { typedef T ReturnType; }; template struct ScalarBinaryOpTraits::IsComplex,T>::type>::Real, BinaryOp> { typedef T ReturnType; }; template struct ScalarBinaryOpTraits::IsComplex,T>::type>::Real, T, BinaryOp> { typedef T ReturnType; }; // For Matrix * Permutation template struct ScalarBinaryOpTraits { typedef T ReturnType; }; // For Permutation * Matrix template struct ScalarBinaryOpTraits { typedef T ReturnType; }; // for Permutation*Permutation template struct ScalarBinaryOpTraits { typedef void ReturnType; }; // We require Lhs and Rhs to have "compatible" scalar types. // It is tempting to always allow mixing different types but remember that this is often impossible in the vectorized paths. // So allowing mixing different types gives very unexpected errors when enabling vectorization, when the user tries to // add together a float matrix and a double matrix. #define EIGEN_CHECK_BINARY_COMPATIBILIY(BINOP,LHS,RHS) \ EIGEN_STATIC_ASSERT((Eigen::internal::has_ReturnType >::value), \ YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) } // end namespace Eigen #endif // EIGEN_XPRHELPER_H RcppEigen/inst/include/Eigen/src/Core/util/MKL_support.h0000755000176200001440000001025414567757725022637 0ustar liggesusers/* Copyright (c) 2011, Intel Corporation. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Intel Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************** * Content : Eigen bindings to Intel(R) MKL * Include file with common MKL declarations ******************************************************************************** */ #ifndef EIGEN_MKL_SUPPORT_H #define EIGEN_MKL_SUPPORT_H #ifdef EIGEN_USE_MKL_ALL #ifndef EIGEN_USE_BLAS #define EIGEN_USE_BLAS #endif #ifndef EIGEN_USE_LAPACKE #define EIGEN_USE_LAPACKE #endif #ifndef EIGEN_USE_MKL_VML #define EIGEN_USE_MKL_VML #endif #endif #ifdef EIGEN_USE_LAPACKE_STRICT #define EIGEN_USE_LAPACKE #endif #if defined(EIGEN_USE_MKL_VML) && !defined(EIGEN_USE_MKL) #define EIGEN_USE_MKL #endif #if defined EIGEN_USE_MKL # if (!defined MKL_DIRECT_CALL) && (!defined EIGEN_MKL_NO_DIRECT_CALL) # define MKL_DIRECT_CALL # define MKL_DIRECT_CALL_JUST_SET # endif # include /*Check IMKL version for compatibility: < 10.3 is not usable with Eigen*/ # ifndef INTEL_MKL_VERSION # undef EIGEN_USE_MKL /* INTEL_MKL_VERSION is not even defined on older versions */ # elif INTEL_MKL_VERSION < 100305 /* the intel-mkl-103-release-notes say this was when the lapacke.h interface was added*/ # undef EIGEN_USE_MKL # endif # ifndef EIGEN_USE_MKL /*If the MKL version is too old, undef everything*/ # undef EIGEN_USE_MKL_ALL # undef EIGEN_USE_LAPACKE # undef EIGEN_USE_MKL_VML # undef EIGEN_USE_LAPACKE_STRICT # undef EIGEN_USE_LAPACKE # ifdef MKL_DIRECT_CALL_JUST_SET # undef MKL_DIRECT_CALL # endif # endif #endif #if defined EIGEN_USE_MKL #define EIGEN_MKL_VML_THRESHOLD 128 /* MKL_DOMAIN_BLAS, etc are defined only in 10.3 update 7 */ /* MKL_BLAS, etc are not defined in 11.2 */ #ifdef MKL_DOMAIN_ALL #define EIGEN_MKL_DOMAIN_ALL MKL_DOMAIN_ALL #else #define EIGEN_MKL_DOMAIN_ALL MKL_ALL #endif #ifdef MKL_DOMAIN_BLAS #define EIGEN_MKL_DOMAIN_BLAS MKL_DOMAIN_BLAS #else #define EIGEN_MKL_DOMAIN_BLAS MKL_BLAS #endif #ifdef MKL_DOMAIN_FFT #define EIGEN_MKL_DOMAIN_FFT MKL_DOMAIN_FFT #else #define EIGEN_MKL_DOMAIN_FFT MKL_FFT #endif #ifdef MKL_DOMAIN_VML #define EIGEN_MKL_DOMAIN_VML MKL_DOMAIN_VML #else #define EIGEN_MKL_DOMAIN_VML MKL_VML #endif #ifdef MKL_DOMAIN_PARDISO #define EIGEN_MKL_DOMAIN_PARDISO MKL_DOMAIN_PARDISO #else #define EIGEN_MKL_DOMAIN_PARDISO MKL_PARDISO #endif #endif #if defined(EIGEN_USE_BLAS) && !defined(EIGEN_USE_MKL) #include "../../misc/blas.h" #endif namespace Eigen { typedef std::complex dcomplex; typedef std::complex scomplex; #if defined(EIGEN_USE_MKL) typedef MKL_INT BlasIndex; #else typedef int BlasIndex; #endif } // end namespace Eigen #endif // EIGEN_MKL_SUPPORT_H RcppEigen/inst/include/Eigen/src/Core/util/IndexedViewHelper.h0000644000176200001440000001505014567757725023767 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2017 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_INDEXED_VIEW_HELPER_H #define EIGEN_INDEXED_VIEW_HELPER_H namespace Eigen { namespace internal { struct symbolic_last_tag {}; } /** \var last * \ingroup Core_Module * * Can be used as a parameter to Eigen::seq and Eigen::seqN functions to symbolically reference the last element/row/columns * of the underlying vector or matrix once passed to DenseBase::operator()(const RowIndices&, const ColIndices&). * * This symbolic placeholder supports standard arithmetic operations. * * A typical usage example would be: * \code * using namespace Eigen; * using Eigen::last; * VectorXd v(n); * v(seq(2,last-2)).setOnes(); * \endcode * * \sa end */ static const symbolic::SymbolExpr last; // PLEASE use Eigen::last instead of Eigen::placeholders::last /** \var lastp1 * \ingroup Core_Module * * Can be used as a parameter to Eigen::seq and Eigen::seqN functions to symbolically * reference the last+1 element/row/columns of the underlying vector or matrix once * passed to DenseBase::operator()(const RowIndices&, const ColIndices&). * * This symbolic placeholder supports standard arithmetic operations. * It is essentially an alias to last+fix<1>. * * \sa last */ #ifdef EIGEN_PARSED_BY_DOXYGEN static const auto lastp1 = last+fix<1>; #else // Using a FixedExpr<1> expression is important here to make sure the compiler // can fully optimize the computation starting indices with zero overhead. static const symbolic::AddExpr,symbolic::ValueExpr > > lastp1(last+fix<1>()); #endif namespace internal { // Replace symbolic last/end "keywords" by their true runtime value inline Index eval_expr_given_size(Index x, Index /* size */) { return x; } template FixedInt eval_expr_given_size(FixedInt x, Index /*size*/) { return x; } template Index eval_expr_given_size(const symbolic::BaseExpr &x, Index size) { return x.derived().eval(last=size-1); } // Extract increment/step at compile time template struct get_compile_time_incr { enum { value = UndefinedIncr }; }; // Analogue of std::get<0>(x), but tailored for our needs. template EIGEN_CONSTEXPR Index first(const T& x) EIGEN_NOEXCEPT { return x.first(); } // IndexedViewCompatibleType/makeIndexedViewCompatible turn an arbitrary object of type T into something usable by MatrixSlice // The generic implementation is a no-op template struct IndexedViewCompatibleType { typedef T type; }; template const T& makeIndexedViewCompatible(const T& x, Index /*size*/, Q) { return x; } //-------------------------------------------------------------------------------- // Handling of a single Index //-------------------------------------------------------------------------------- struct SingleRange { enum { SizeAtCompileTime = 1 }; SingleRange(Index val) : m_value(val) {} Index operator[](Index) const { return m_value; } static EIGEN_CONSTEXPR Index size() EIGEN_NOEXCEPT { return 1; } Index first() const EIGEN_NOEXCEPT { return m_value; } Index m_value; }; template<> struct get_compile_time_incr { enum { value = 1 }; // 1 or 0 ?? }; // Turn a single index into something that looks like an array (i.e., that exposes a .size(), and operator[](int) methods) template struct IndexedViewCompatibleType::value>::type> { // Here we could simply use Array, but maybe it's less work for the compiler to use // a simpler wrapper as SingleRange //typedef Eigen::Array type; typedef SingleRange type; }; template struct IndexedViewCompatibleType::value>::type> { typedef SingleRange type; }; template typename enable_if::value,SingleRange>::type makeIndexedViewCompatible(const T& id, Index size, SpecializedType) { return eval_expr_given_size(id,size); } //-------------------------------------------------------------------------------- // Handling of all //-------------------------------------------------------------------------------- struct all_t { all_t() {} }; // Convert a symbolic 'all' into a usable range type template struct AllRange { enum { SizeAtCompileTime = XprSize }; AllRange(Index size = XprSize) : m_size(size) {} EIGEN_CONSTEXPR Index operator[](Index i) const EIGEN_NOEXCEPT { return i; } EIGEN_CONSTEXPR Index size() const EIGEN_NOEXCEPT { return m_size.value(); } EIGEN_CONSTEXPR Index first() const EIGEN_NOEXCEPT { return 0; } variable_if_dynamic m_size; }; template struct IndexedViewCompatibleType { typedef AllRange type; }; template inline AllRange::value> makeIndexedViewCompatible(all_t , XprSizeType size, SpecializedType) { return AllRange::value>(size); } template struct get_compile_time_incr > { enum { value = 1 }; }; } // end namespace internal /** \var all * \ingroup Core_Module * Can be used as a parameter to DenseBase::operator()(const RowIndices&, const ColIndices&) to index all rows or columns */ static const Eigen::internal::all_t all; // PLEASE use Eigen::all instead of Eigen::placeholders::all namespace placeholders { typedef symbolic::SymbolExpr last_t; typedef symbolic::AddExpr,symbolic::ValueExpr > > end_t; typedef Eigen::internal::all_t all_t; EIGEN_DEPRECATED static const all_t all = Eigen::all; // PLEASE use Eigen::all instead of Eigen::placeholders::all EIGEN_DEPRECATED static const last_t last = Eigen::last; // PLEASE use Eigen::last instead of Eigen::placeholders::last EIGEN_DEPRECATED static const end_t end = Eigen::lastp1; // PLEASE use Eigen::lastp1 instead of Eigen::placeholders::end } } // end namespace Eigen #endif // EIGEN_INDEXED_VIEW_HELPER_H RcppEigen/inst/include/Eigen/src/Core/util/Meta.h0000755000176200001440000007123014567757725021307 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2015 Gael Guennebaud // Copyright (C) 2006-2008 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_META_H #define EIGEN_META_H #if defined(EIGEN_GPU_COMPILE_PHASE) #include #if defined(EIGEN_CUDA_ARCH) #include #endif #if defined(EIGEN_HIP_DEVICE_COMPILE) #include "Eigen/src/Core/arch/HIP/hcc/math_constants.h" #endif #endif // Recent versions of ICC require for pointer types below. #define EIGEN_ICC_NEEDS_CSTDINT (EIGEN_COMP_ICC>=1600 && EIGEN_COMP_CXXVER >= 11) // Define portable (u)int{32,64} types #if EIGEN_HAS_CXX11 || EIGEN_ICC_NEEDS_CSTDINT #include namespace Eigen { namespace numext { typedef std::uint8_t uint8_t; typedef std::int8_t int8_t; typedef std::uint16_t uint16_t; typedef std::int16_t int16_t; typedef std::uint32_t uint32_t; typedef std::int32_t int32_t; typedef std::uint64_t uint64_t; typedef std::int64_t int64_t; } } #else // Without c++11, all compilers able to compile Eigen also // provide the C99 stdint.h header file. #include namespace Eigen { namespace numext { typedef ::uint8_t uint8_t; typedef ::int8_t int8_t; typedef ::uint16_t uint16_t; typedef ::int16_t int16_t; typedef ::uint32_t uint32_t; typedef ::int32_t int32_t; typedef ::uint64_t uint64_t; typedef ::int64_t int64_t; } } #endif namespace Eigen { typedef EIGEN_DEFAULT_DENSE_INDEX_TYPE DenseIndex; /** * \brief The Index type as used for the API. * \details To change this, \c \#define the preprocessor symbol \c EIGEN_DEFAULT_DENSE_INDEX_TYPE. * \sa \blank \ref TopicPreprocessorDirectives, StorageIndex. */ typedef EIGEN_DEFAULT_DENSE_INDEX_TYPE Index; namespace internal { /** \internal * \file Meta.h * This file contains generic metaprogramming classes which are not specifically related to Eigen. * \note In case you wonder, yes we're aware that Boost already provides all these features, * we however don't want to add a dependency to Boost. */ // Only recent versions of ICC complain about using ptrdiff_t to hold pointers, // and older versions do not provide *intptr_t types. #if EIGEN_ICC_NEEDS_CSTDINT typedef std::intptr_t IntPtr; typedef std::uintptr_t UIntPtr; #else typedef std::ptrdiff_t IntPtr; typedef std::size_t UIntPtr; #endif #undef EIGEN_ICC_NEEDS_CSTDINT struct true_type { enum { value = 1 }; }; struct false_type { enum { value = 0 }; }; template struct bool_constant; template<> struct bool_constant : true_type {}; template<> struct bool_constant : false_type {}; template struct conditional { typedef Then type; }; template struct conditional { typedef Else type; }; template struct remove_reference { typedef T type; }; template struct remove_reference { typedef T type; }; template struct remove_pointer { typedef T type; }; template struct remove_pointer { typedef T type; }; template struct remove_pointer { typedef T type; }; template struct remove_const { typedef T type; }; template struct remove_const { typedef T type; }; template struct remove_const { typedef T type[]; }; template struct remove_const { typedef T type[Size]; }; template struct remove_all { typedef T type; }; template struct remove_all { typedef typename remove_all::type type; }; template struct remove_all { typedef typename remove_all::type type; }; template struct remove_all { typedef typename remove_all::type type; }; template struct remove_all { typedef typename remove_all::type type; }; template struct remove_all { typedef typename remove_all::type type; }; template struct is_arithmetic { enum { value = false }; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic{ enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; template struct is_same { enum { value = 0 }; }; template struct is_same { enum { value = 1 }; }; template< class T > struct is_void : is_same::type> {}; #if EIGEN_HAS_CXX11 template<> struct is_arithmetic { enum { value = true }; }; template<> struct is_arithmetic { enum { value = true }; }; using std::is_integral; #else template struct is_integral { enum { value = false }; }; template<> struct is_integral { enum { value = true }; }; template<> struct is_integral { enum { value = true }; }; template<> struct is_integral { enum { value = true }; }; template<> struct is_integral { enum { value = true }; }; template<> struct is_integral { enum { value = true }; }; template<> struct is_integral { enum { value = true }; }; template<> struct is_integral { enum { value = true }; }; template<> struct is_integral { enum { value = true }; }; template<> struct is_integral { enum { value = true }; }; template<> struct is_integral { enum { value = true }; }; #if EIGEN_COMP_MSVC template<> struct is_integral { enum { value = true }; }; template<> struct is_integral { enum { value = true }; }; #endif #endif #if EIGEN_HAS_CXX11 using std::make_unsigned; #else // TODO: Possibly improve this implementation of make_unsigned. // It is currently used only by // template struct random_default_impl. template struct make_unsigned; template<> struct make_unsigned { typedef unsigned char type; }; template<> struct make_unsigned { typedef unsigned char type; }; template<> struct make_unsigned { typedef unsigned char type; }; template<> struct make_unsigned { typedef unsigned short type; }; template<> struct make_unsigned { typedef unsigned short type; }; template<> struct make_unsigned { typedef unsigned int type; }; template<> struct make_unsigned { typedef unsigned int type; }; template<> struct make_unsigned { typedef unsigned long type; }; template<> struct make_unsigned { typedef unsigned long type; }; #if EIGEN_COMP_MSVC template<> struct make_unsigned { typedef unsigned __int64 type; }; template<> struct make_unsigned { typedef unsigned __int64 type; }; #endif // Some platforms define int64_t as `long long` even for C++03, where // `long long` is not guaranteed by the standard. In this case we are missing // the definition for make_unsigned. If we just define it, we run into issues // where `long long` doesn't exist in some compilers for C++03. We therefore add // the specialization for these platforms only. #if EIGEN_OS_MAC || EIGEN_COMP_MINGW template<> struct make_unsigned { typedef unsigned long long type; }; template<> struct make_unsigned { typedef unsigned long long type; }; #endif #endif template struct add_const { typedef const T type; }; template struct add_const { typedef T& type; }; template struct is_const { enum { value = 0 }; }; template struct is_const { enum { value = 1 }; }; template struct add_const_on_value_type { typedef const T type; }; template struct add_const_on_value_type { typedef T const& type; }; template struct add_const_on_value_type { typedef T const* type; }; template struct add_const_on_value_type { typedef T const* const type; }; template struct add_const_on_value_type { typedef T const* const type; }; #if EIGEN_HAS_CXX11 using std::is_convertible; #else template struct is_convertible_impl { private: struct any_conversion { template any_conversion(const volatile T&); template any_conversion(T&); }; struct yes {int a[1];}; struct no {int a[2];}; template static yes test(T, int); template static no test(any_conversion, ...); public: static typename internal::remove_reference::type* ms_from; #ifdef __INTEL_COMPILER #pragma warning push #pragma warning ( disable : 2259 ) #endif enum { value = sizeof(test(*ms_from, 0))==sizeof(yes) }; #ifdef __INTEL_COMPILER #pragma warning pop #endif }; template struct is_convertible { enum { value = is_convertible_impl::value }; }; template struct is_convertible { enum { value = false }; }; template struct is_convertible { enum { value = true }; }; #endif /** \internal Allows to enable/disable an overload * according to a compile time condition. */ template struct enable_if; template struct enable_if { typedef T type; }; #if defined(EIGEN_GPU_COMPILE_PHASE) && !EIGEN_HAS_CXX11 #if !defined(__FLT_EPSILON__) #define __FLT_EPSILON__ FLT_EPSILON #define __DBL_EPSILON__ DBL_EPSILON #endif namespace device { template struct numeric_limits { EIGEN_DEVICE_FUNC static EIGEN_CONSTEXPR T epsilon() { return 0; } static T (max)() { assert(false && "Highest not supported for this type"); } static T (min)() { assert(false && "Lowest not supported for this type"); } static T infinity() { assert(false && "Infinity not supported for this type"); } static T quiet_NaN() { assert(false && "quiet_NaN not supported for this type"); } }; template<> struct numeric_limits { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static float epsilon() { return __FLT_EPSILON__; } EIGEN_DEVICE_FUNC static float (max)() { #if defined(EIGEN_CUDA_ARCH) return CUDART_MAX_NORMAL_F; #else return HIPRT_MAX_NORMAL_F; #endif } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static float (min)() { return FLT_MIN; } EIGEN_DEVICE_FUNC static float infinity() { #if defined(EIGEN_CUDA_ARCH) return CUDART_INF_F; #else return HIPRT_INF_F; #endif } EIGEN_DEVICE_FUNC static float quiet_NaN() { #if defined(EIGEN_CUDA_ARCH) return CUDART_NAN_F; #else return HIPRT_NAN_F; #endif } }; template<> struct numeric_limits { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static double epsilon() { return __DBL_EPSILON__; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static double (max)() { return DBL_MAX; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static double (min)() { return DBL_MIN; } EIGEN_DEVICE_FUNC static double infinity() { #if defined(EIGEN_CUDA_ARCH) return CUDART_INF; #else return HIPRT_INF; #endif } EIGEN_DEVICE_FUNC static double quiet_NaN() { #if defined(EIGEN_CUDA_ARCH) return CUDART_NAN; #else return HIPRT_NAN; #endif } }; template<> struct numeric_limits { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int epsilon() { return 0; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int (max)() { return INT_MAX; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int (min)() { return INT_MIN; } }; template<> struct numeric_limits { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static unsigned int epsilon() { return 0; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static unsigned int (max)() { return UINT_MAX; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static unsigned int (min)() { return 0; } }; template<> struct numeric_limits { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static long epsilon() { return 0; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static long (max)() { return LONG_MAX; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static long (min)() { return LONG_MIN; } }; template<> struct numeric_limits { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static unsigned long epsilon() { return 0; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static unsigned long (max)() { return ULONG_MAX; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static unsigned long (min)() { return 0; } }; template<> struct numeric_limits { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static long long epsilon() { return 0; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static long long (max)() { return LLONG_MAX; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static long long (min)() { return LLONG_MIN; } }; template<> struct numeric_limits { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static unsigned long long epsilon() { return 0; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static unsigned long long (max)() { return ULLONG_MAX; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static unsigned long long (min)() { return 0; } }; template<> struct numeric_limits { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static bool epsilon() { return false; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static bool (max)() { return true; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static bool (min)() { return false; } }; } #endif // defined(EIGEN_GPU_COMPILE_PHASE) && !EIGEN_HAS_CXX11 /** \internal * A base class do disable default copy ctor and copy assignment operator. */ class noncopyable { EIGEN_DEVICE_FUNC noncopyable(const noncopyable&); EIGEN_DEVICE_FUNC const noncopyable& operator=(const noncopyable&); protected: EIGEN_DEVICE_FUNC noncopyable() {} EIGEN_DEVICE_FUNC ~noncopyable() {} }; /** \internal * Provides access to the number of elements in the object of as a compile-time constant expression. * It "returns" Eigen::Dynamic if the size cannot be resolved at compile-time (default). * * Similar to std::tuple_size, but more general. * * It currently supports: * - any types T defining T::SizeAtCompileTime * - plain C arrays as T[N] * - std::array (c++11) * - some internal types such as SingleRange and AllRange * * The second template parameter eases SFINAE-based specializations. */ template struct array_size { enum { value = Dynamic }; }; template struct array_size::type> { enum { value = T::SizeAtCompileTime }; }; template struct array_size { enum { value = N }; }; template struct array_size { enum { value = N }; }; #if EIGEN_HAS_CXX11 template struct array_size > { enum { value = N }; }; template struct array_size > { enum { value = N }; }; #endif /** \internal * Analogue of the std::size free function. * It returns the size of the container or view \a x of type \c T * * It currently supports: * - any types T defining a member T::size() const * - plain C arrays as T[N] * */ template EIGEN_CONSTEXPR Index size(const T& x) { return x.size(); } template EIGEN_CONSTEXPR Index size(const T (&) [N]) { return N; } /** \internal * Convenient struct to get the result type of a nullary, unary, binary, or * ternary functor. * * Pre C++11: * Supports both a Func::result_type member and templated * Func::result::type member. * * If none of these members is provided, then the type of the first * argument is returned. * * Post C++11: * This uses std::result_of. However, note the `type` member removes * const and converts references/pointers to their corresponding value type. */ #if EIGEN_HAS_STD_INVOKE_RESULT template struct result_of; template struct result_of { typedef typename std::invoke_result::type type1; typedef typename remove_all::type type; }; #elif EIGEN_HAS_STD_RESULT_OF template struct result_of { typedef typename std::result_of::type type1; typedef typename remove_all::type type; }; #else template struct result_of { }; struct has_none {int a[1];}; struct has_std_result_type {int a[2];}; struct has_tr1_result {int a[3];}; template struct nullary_result_of_select {}; template struct nullary_result_of_select {typedef typename Func::result_type type;}; template struct nullary_result_of_select {typedef typename Func::template result::type type;}; template struct result_of { template static has_std_result_type testFunctor(T const *, typename T::result_type const * = 0); template static has_tr1_result testFunctor(T const *, typename T::template result::type const * = 0); static has_none testFunctor(...); // note that the following indirection is needed for gcc-3.3 enum {FunctorType = sizeof(testFunctor(static_cast(0)))}; typedef typename nullary_result_of_select::type type; }; template struct unary_result_of_select {typedef typename internal::remove_all::type type;}; template struct unary_result_of_select {typedef typename Func::result_type type;}; template struct unary_result_of_select {typedef typename Func::template result::type type;}; template struct result_of { template static has_std_result_type testFunctor(T const *, typename T::result_type const * = 0); template static has_tr1_result testFunctor(T const *, typename T::template result::type const * = 0); static has_none testFunctor(...); // note that the following indirection is needed for gcc-3.3 enum {FunctorType = sizeof(testFunctor(static_cast(0)))}; typedef typename unary_result_of_select::type type; }; template struct binary_result_of_select {typedef typename internal::remove_all::type type;}; template struct binary_result_of_select {typedef typename Func::result_type type;}; template struct binary_result_of_select {typedef typename Func::template result::type type;}; template struct result_of { template static has_std_result_type testFunctor(T const *, typename T::result_type const * = 0); template static has_tr1_result testFunctor(T const *, typename T::template result::type const * = 0); static has_none testFunctor(...); // note that the following indirection is needed for gcc-3.3 enum {FunctorType = sizeof(testFunctor(static_cast(0)))}; typedef typename binary_result_of_select::type type; }; template struct ternary_result_of_select {typedef typename internal::remove_all::type type;}; template struct ternary_result_of_select {typedef typename Func::result_type type;}; template struct ternary_result_of_select {typedef typename Func::template result::type type;}; template struct result_of { template static has_std_result_type testFunctor(T const *, typename T::result_type const * = 0); template static has_tr1_result testFunctor(T const *, typename T::template result::type const * = 0); static has_none testFunctor(...); // note that the following indirection is needed for gcc-3.3 enum {FunctorType = sizeof(testFunctor(static_cast(0)))}; typedef typename ternary_result_of_select::type type; }; #endif #if EIGEN_HAS_STD_INVOKE_RESULT template struct invoke_result { typedef typename std::invoke_result::type type1; typedef typename remove_all::type type; }; #elif EIGEN_HAS_CXX11 template struct invoke_result { typedef typename result_of::type type1; typedef typename remove_all::type type; }; #else template struct invoke_result { typedef typename result_of::type type1; typedef typename remove_all::type type; }; template struct invoke_result { typedef typename result_of::type type1; typedef typename remove_all::type type; }; template struct invoke_result { typedef typename result_of::type type1; typedef typename remove_all::type type; }; template struct invoke_result { typedef typename result_of::type type1; typedef typename remove_all::type type; }; #endif struct meta_yes { char a[1]; }; struct meta_no { char a[2]; }; // Check whether T::ReturnType does exist template struct has_ReturnType { template static meta_yes testFunctor(C const *, typename C::ReturnType const * = 0); template static meta_no testFunctor(...); enum { value = sizeof(testFunctor(static_cast(0))) == sizeof(meta_yes) }; }; template const T* return_ptr(); template struct has_nullary_operator { template static meta_yes testFunctor(C const *,typename enable_if<(sizeof(return_ptr()->operator()())>0)>::type * = 0); static meta_no testFunctor(...); enum { value = sizeof(testFunctor(static_cast(0))) == sizeof(meta_yes) }; }; template struct has_unary_operator { template static meta_yes testFunctor(C const *,typename enable_if<(sizeof(return_ptr()->operator()(IndexType(0)))>0)>::type * = 0); static meta_no testFunctor(...); enum { value = sizeof(testFunctor(static_cast(0))) == sizeof(meta_yes) }; }; template struct has_binary_operator { template static meta_yes testFunctor(C const *,typename enable_if<(sizeof(return_ptr()->operator()(IndexType(0),IndexType(0)))>0)>::type * = 0); static meta_no testFunctor(...); enum { value = sizeof(testFunctor(static_cast(0))) == sizeof(meta_yes) }; }; /** \internal In short, it computes int(sqrt(\a Y)) with \a Y an integer. * Usage example: \code meta_sqrt<1023>::ret \endcode */ template Y))) > // use ?: instead of || just to shut up a stupid gcc 4.3 warning class meta_sqrt { enum { MidX = (InfX+SupX)/2, TakeInf = MidX*MidX > Y ? 1 : 0, NewInf = int(TakeInf) ? InfX : int(MidX), NewSup = int(TakeInf) ? int(MidX) : SupX }; public: enum { ret = meta_sqrt::ret }; }; template class meta_sqrt { public: enum { ret = (SupX*SupX <= Y) ? SupX : InfX }; }; /** \internal Computes the least common multiple of two positive integer A and B * at compile-time. */ template=B)> struct meta_least_common_multiple { enum { ret = meta_least_common_multiple::ret }; }; template struct meta_least_common_multiple { enum { ret = meta_least_common_multiple::ret }; }; template struct meta_least_common_multiple { enum { ret = A*K }; }; /** \internal determines whether the product of two numeric types is allowed and what the return type is */ template struct scalar_product_traits { enum { Defined = 0 }; }; // FIXME quick workaround around current limitation of result_of // template // struct result_of(ArgType0,ArgType1)> { // typedef typename scalar_product_traits::type, typename remove_all::type>::ReturnType type; // }; /** \internal Obtains a POD type suitable to use as storage for an object of a size * of at most Len bytes, aligned as specified by \c Align. */ template struct aligned_storage { struct type { EIGEN_ALIGN_TO_BOUNDARY(Align) unsigned char data[Len]; }; }; } // end namespace internal namespace numext { #if defined(EIGEN_GPU_COMPILE_PHASE) template EIGEN_DEVICE_FUNC void swap(T &a, T &b) { T tmp = b; b = a; a = tmp; } #else template EIGEN_STRONG_INLINE void swap(T &a, T &b) { std::swap(a,b); } #endif #if defined(EIGEN_GPU_COMPILE_PHASE) && !EIGEN_HAS_CXX11 using internal::device::numeric_limits; #else using std::numeric_limits; #endif // Integer division with rounding up. // T is assumed to be an integer type with a>=0, and b>0 template EIGEN_DEVICE_FUNC T div_ceil(const T &a, const T &b) { return (a+b-1) / b; } // The aim of the following functions is to bypass -Wfloat-equal warnings // when we really want a strict equality comparison on floating points. template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool equal_strict(const X& x,const Y& y) { return x == y; } #if !defined(EIGEN_GPU_COMPILE_PHASE) || (!defined(EIGEN_CUDA_ARCH) && defined(EIGEN_CONSTEXPR_ARE_DEVICE_FUNC)) template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool equal_strict(const float& x,const float& y) { return std::equal_to()(x,y); } template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool equal_strict(const double& x,const double& y) { return std::equal_to()(x,y); } #endif template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool not_equal_strict(const X& x,const Y& y) { return x != y; } #if !defined(EIGEN_GPU_COMPILE_PHASE) || (!defined(EIGEN_CUDA_ARCH) && defined(EIGEN_CONSTEXPR_ARE_DEVICE_FUNC)) template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool not_equal_strict(const float& x,const float& y) { return std::not_equal_to()(x,y); } template<> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool not_equal_strict(const double& x,const double& y) { return std::not_equal_to()(x,y); } #endif } // end namespace numext } // end namespace Eigen #endif // EIGEN_META_H RcppEigen/inst/include/Eigen/src/Core/util/NonMPL2.h0000644000176200001440000000012514107270226021546 0ustar liggesusers#ifdef EIGEN_MPL2_ONLY #error Including non-MPL2 code in EIGEN_MPL2_ONLY mode #endif RcppEigen/inst/include/Eigen/src/Core/util/Memory.h0000644000176200001440000013310514567757725021666 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2015 Gael Guennebaud // Copyright (C) 2008-2009 Benoit Jacob // Copyright (C) 2009 Kenneth Riddile // Copyright (C) 2010 Hauke Heibel // Copyright (C) 2010 Thomas Capricelli // Copyright (C) 2013 Pavel Holoborodko // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. /***************************************************************************** *** Platform checks for aligned malloc functions *** *****************************************************************************/ #ifndef EIGEN_MEMORY_H #define EIGEN_MEMORY_H #ifndef EIGEN_MALLOC_ALREADY_ALIGNED // Try to determine automatically if malloc is already aligned. // On 64-bit systems, glibc's malloc returns 16-byte-aligned pointers, see: // http://www.gnu.org/s/libc/manual/html_node/Aligned-Memory-Blocks.html // This is true at least since glibc 2.8. // This leaves the question how to detect 64-bit. According to this document, // http://gcc.fyxm.net/summit/2003/Porting%20to%2064%20bit.pdf // page 114, "[The] LP64 model [...] is used by all 64-bit UNIX ports" so it's indeed // quite safe, at least within the context of glibc, to equate 64-bit with LP64. #if defined(__GLIBC__) && ((__GLIBC__>=2 && __GLIBC_MINOR__ >= 8) || __GLIBC__>2) \ && defined(__LP64__) && ! defined( __SANITIZE_ADDRESS__ ) && (EIGEN_DEFAULT_ALIGN_BYTES == 16) #define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 1 #else #define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 0 #endif // FreeBSD 6 seems to have 16-byte aligned malloc // See http://svn.freebsd.org/viewvc/base/stable/6/lib/libc/stdlib/malloc.c?view=markup // FreeBSD 7 seems to have 16-byte aligned malloc except on ARM and MIPS architectures // See http://svn.freebsd.org/viewvc/base/stable/7/lib/libc/stdlib/malloc.c?view=markup #if defined(__FreeBSD__) && !(EIGEN_ARCH_ARM || EIGEN_ARCH_MIPS) && (EIGEN_DEFAULT_ALIGN_BYTES == 16) #define EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED 1 #else #define EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED 0 #endif #if (EIGEN_OS_MAC && (EIGEN_DEFAULT_ALIGN_BYTES == 16)) \ || (EIGEN_OS_WIN64 && (EIGEN_DEFAULT_ALIGN_BYTES == 16)) \ || EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED \ || EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED #define EIGEN_MALLOC_ALREADY_ALIGNED 1 #else #define EIGEN_MALLOC_ALREADY_ALIGNED 0 #endif #endif namespace Eigen { namespace internal { EIGEN_DEVICE_FUNC inline void throw_std_bad_alloc() { #ifdef EIGEN_EXCEPTIONS throw std::bad_alloc(); #else std::size_t huge = static_cast(-1); #if defined(EIGEN_HIPCC) // // calls to "::operator new" are to be treated as opaque function calls (i.e no inlining), // and as a consequence the code in the #else block triggers the hipcc warning : // "no overloaded function has restriction specifiers that are compatible with the ambient context" // // "throw_std_bad_alloc" has the EIGEN_DEVICE_FUNC attribute, so it seems that hipcc expects // the same on "operator new" // Reverting code back to the old version in this #if block for the hipcc compiler // new int[huge]; #else void* unused = ::operator new(huge); EIGEN_UNUSED_VARIABLE(unused); #endif #endif } /***************************************************************************** *** Implementation of handmade aligned functions *** *****************************************************************************/ /* ----- Hand made implementations of aligned malloc/free and realloc ----- */ /** \internal Like malloc, but the returned pointer is guaranteed to be 16-byte aligned. * Fast, but wastes 16 additional bytes of memory. Does not throw any exception. */ EIGEN_DEVICE_FUNC inline void* handmade_aligned_malloc(std::size_t size, std::size_t alignment = EIGEN_DEFAULT_ALIGN_BYTES) { eigen_assert(alignment >= sizeof(void*) && (alignment & (alignment-1)) == 0 && "Alignment must be at least sizeof(void*) and a power of 2"); EIGEN_USING_STD(malloc) void *original = malloc(size+alignment); if (original == 0) return 0; void *aligned = reinterpret_cast((reinterpret_cast(original) & ~(std::size_t(alignment-1))) + alignment); *(reinterpret_cast(aligned) - 1) = original; return aligned; } /** \internal Frees memory allocated with handmade_aligned_malloc */ EIGEN_DEVICE_FUNC inline void handmade_aligned_free(void *ptr) { if (ptr) { EIGEN_USING_STD(free) free(*(reinterpret_cast(ptr) - 1)); } } /** \internal * \brief Reallocates aligned memory. * Since we know that our handmade version is based on std::malloc * we can use std::realloc to implement efficient reallocation. */ inline void* handmade_aligned_realloc(void* ptr, std::size_t size, std::size_t = 0) { if (ptr == 0) return handmade_aligned_malloc(size); void *original = *(reinterpret_cast(ptr) - 1); std::ptrdiff_t previous_offset = static_cast(ptr)-static_cast(original); original = std::realloc(original,size+EIGEN_DEFAULT_ALIGN_BYTES); if (original == 0) return 0; void *aligned = reinterpret_cast((reinterpret_cast(original) & ~(std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1))) + EIGEN_DEFAULT_ALIGN_BYTES); void *previous_aligned = static_cast(original)+previous_offset; if(aligned!=previous_aligned) std::memmove(aligned, previous_aligned, size); *(reinterpret_cast(aligned) - 1) = original; return aligned; } /***************************************************************************** *** Implementation of portable aligned versions of malloc/free/realloc *** *****************************************************************************/ #ifdef EIGEN_NO_MALLOC EIGEN_DEVICE_FUNC inline void check_that_malloc_is_allowed() { eigen_assert(false && "heap allocation is forbidden (EIGEN_NO_MALLOC is defined)"); } #elif defined EIGEN_RUNTIME_NO_MALLOC EIGEN_DEVICE_FUNC inline bool is_malloc_allowed_impl(bool update, bool new_value = false) { static bool value = true; if (update == 1) value = new_value; return value; } EIGEN_DEVICE_FUNC inline bool is_malloc_allowed() { return is_malloc_allowed_impl(false); } EIGEN_DEVICE_FUNC inline bool set_is_malloc_allowed(bool new_value) { return is_malloc_allowed_impl(true, new_value); } EIGEN_DEVICE_FUNC inline void check_that_malloc_is_allowed() { eigen_assert(is_malloc_allowed() && "heap allocation is forbidden (EIGEN_RUNTIME_NO_MALLOC is defined and g_is_malloc_allowed is false)"); } #else EIGEN_DEVICE_FUNC inline void check_that_malloc_is_allowed() {} #endif /** \internal Allocates \a size bytes. The returned pointer is guaranteed to have 16 or 32 bytes alignment depending on the requirements. * On allocation error, the returned pointer is null, and std::bad_alloc is thrown. */ EIGEN_DEVICE_FUNC inline void* aligned_malloc(std::size_t size) { check_that_malloc_is_allowed(); void *result; #if (EIGEN_DEFAULT_ALIGN_BYTES==0) || EIGEN_MALLOC_ALREADY_ALIGNED EIGEN_USING_STD(malloc) result = malloc(size); #if EIGEN_DEFAULT_ALIGN_BYTES==16 eigen_assert((size<16 || (std::size_t(result)%16)==0) && "System's malloc returned an unaligned pointer. Compile with EIGEN_MALLOC_ALREADY_ALIGNED=0 to fallback to handmade aligned memory allocator."); #endif #else result = handmade_aligned_malloc(size); #endif if(!result && size) throw_std_bad_alloc(); return result; } /** \internal Frees memory allocated with aligned_malloc. */ EIGEN_DEVICE_FUNC inline void aligned_free(void *ptr) { #if (EIGEN_DEFAULT_ALIGN_BYTES==0) || EIGEN_MALLOC_ALREADY_ALIGNED EIGEN_USING_STD(free) free(ptr); #else handmade_aligned_free(ptr); #endif } /** * \internal * \brief Reallocates an aligned block of memory. * \throws std::bad_alloc on allocation failure */ inline void* aligned_realloc(void *ptr, std::size_t new_size, std::size_t old_size) { EIGEN_UNUSED_VARIABLE(old_size) void *result; #if (EIGEN_DEFAULT_ALIGN_BYTES==0) || EIGEN_MALLOC_ALREADY_ALIGNED result = std::realloc(ptr,new_size); #else result = handmade_aligned_realloc(ptr,new_size,old_size); #endif if (!result && new_size) throw_std_bad_alloc(); return result; } /***************************************************************************** *** Implementation of conditionally aligned functions *** *****************************************************************************/ /** \internal Allocates \a size bytes. If Align is true, then the returned ptr is 16-byte-aligned. * On allocation error, the returned pointer is null, and a std::bad_alloc is thrown. */ template EIGEN_DEVICE_FUNC inline void* conditional_aligned_malloc(std::size_t size) { return aligned_malloc(size); } template<> EIGEN_DEVICE_FUNC inline void* conditional_aligned_malloc(std::size_t size) { check_that_malloc_is_allowed(); EIGEN_USING_STD(malloc) void *result = malloc(size); if(!result && size) throw_std_bad_alloc(); return result; } /** \internal Frees memory allocated with conditional_aligned_malloc */ template EIGEN_DEVICE_FUNC inline void conditional_aligned_free(void *ptr) { aligned_free(ptr); } template<> EIGEN_DEVICE_FUNC inline void conditional_aligned_free(void *ptr) { EIGEN_USING_STD(free) free(ptr); } template inline void* conditional_aligned_realloc(void* ptr, std::size_t new_size, std::size_t old_size) { return aligned_realloc(ptr, new_size, old_size); } template<> inline void* conditional_aligned_realloc(void* ptr, std::size_t new_size, std::size_t) { return std::realloc(ptr, new_size); } /***************************************************************************** *** Construction/destruction of array elements *** *****************************************************************************/ /** \internal Destructs the elements of an array. * The \a size parameters tells on how many objects to call the destructor of T. */ template EIGEN_DEVICE_FUNC inline void destruct_elements_of_array(T *ptr, std::size_t size) { // always destruct an array starting from the end. if(ptr) while(size) ptr[--size].~T(); } /** \internal Constructs the elements of an array. * The \a size parameter tells on how many objects to call the constructor of T. */ template EIGEN_DEVICE_FUNC inline T* construct_elements_of_array(T *ptr, std::size_t size) { std::size_t i; EIGEN_TRY { for (i = 0; i < size; ++i) ::new (ptr + i) T; return ptr; } EIGEN_CATCH(...) { destruct_elements_of_array(ptr, i); EIGEN_THROW; } return NULL; } /***************************************************************************** *** Implementation of aligned new/delete-like functions *** *****************************************************************************/ template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void check_size_for_overflow(std::size_t size) { if(size > std::size_t(-1) / sizeof(T)) throw_std_bad_alloc(); } /** \internal Allocates \a size objects of type T. The returned pointer is guaranteed to have 16 bytes alignment. * On allocation error, the returned pointer is undefined, but a std::bad_alloc is thrown. * The default constructor of T is called. */ template EIGEN_DEVICE_FUNC inline T* aligned_new(std::size_t size) { check_size_for_overflow(size); T *result = reinterpret_cast(aligned_malloc(sizeof(T)*size)); EIGEN_TRY { return construct_elements_of_array(result, size); } EIGEN_CATCH(...) { aligned_free(result); EIGEN_THROW; } return result; } template EIGEN_DEVICE_FUNC inline T* conditional_aligned_new(std::size_t size) { check_size_for_overflow(size); T *result = reinterpret_cast(conditional_aligned_malloc(sizeof(T)*size)); EIGEN_TRY { return construct_elements_of_array(result, size); } EIGEN_CATCH(...) { conditional_aligned_free(result); EIGEN_THROW; } return result; } /** \internal Deletes objects constructed with aligned_new * The \a size parameters tells on how many objects to call the destructor of T. */ template EIGEN_DEVICE_FUNC inline void aligned_delete(T *ptr, std::size_t size) { destruct_elements_of_array(ptr, size); Eigen::internal::aligned_free(ptr); } /** \internal Deletes objects constructed with conditional_aligned_new * The \a size parameters tells on how many objects to call the destructor of T. */ template EIGEN_DEVICE_FUNC inline void conditional_aligned_delete(T *ptr, std::size_t size) { destruct_elements_of_array(ptr, size); conditional_aligned_free(ptr); } template EIGEN_DEVICE_FUNC inline T* conditional_aligned_realloc_new(T* pts, std::size_t new_size, std::size_t old_size) { check_size_for_overflow(new_size); check_size_for_overflow(old_size); if(new_size < old_size) destruct_elements_of_array(pts+new_size, old_size-new_size); T *result = reinterpret_cast(conditional_aligned_realloc(reinterpret_cast(pts), sizeof(T)*new_size, sizeof(T)*old_size)); if(new_size > old_size) { EIGEN_TRY { construct_elements_of_array(result+old_size, new_size-old_size); } EIGEN_CATCH(...) { conditional_aligned_free(result); EIGEN_THROW; } } return result; } template EIGEN_DEVICE_FUNC inline T* conditional_aligned_new_auto(std::size_t size) { if(size==0) return 0; // short-cut. Also fixes Bug 884 check_size_for_overflow(size); T *result = reinterpret_cast(conditional_aligned_malloc(sizeof(T)*size)); if(NumTraits::RequireInitialization) { EIGEN_TRY { construct_elements_of_array(result, size); } EIGEN_CATCH(...) { conditional_aligned_free(result); EIGEN_THROW; } } return result; } template inline T* conditional_aligned_realloc_new_auto(T* pts, std::size_t new_size, std::size_t old_size) { check_size_for_overflow(new_size); check_size_for_overflow(old_size); if(NumTraits::RequireInitialization && (new_size < old_size)) destruct_elements_of_array(pts+new_size, old_size-new_size); T *result = reinterpret_cast(conditional_aligned_realloc(reinterpret_cast(pts), sizeof(T)*new_size, sizeof(T)*old_size)); if(NumTraits::RequireInitialization && (new_size > old_size)) { EIGEN_TRY { construct_elements_of_array(result+old_size, new_size-old_size); } EIGEN_CATCH(...) { conditional_aligned_free(result); EIGEN_THROW; } } return result; } template EIGEN_DEVICE_FUNC inline void conditional_aligned_delete_auto(T *ptr, std::size_t size) { if(NumTraits::RequireInitialization) destruct_elements_of_array(ptr, size); conditional_aligned_free(ptr); } /****************************************************************************/ /** \internal Returns the index of the first element of the array that is well aligned with respect to the requested \a Alignment. * * \tparam Alignment requested alignment in Bytes. * \param array the address of the start of the array * \param size the size of the array * * \note If no element of the array is well aligned or the requested alignment is not a multiple of a scalar, * the size of the array is returned. For example with SSE, the requested alignment is typically 16-bytes. If * packet size for the given scalar type is 1, then everything is considered well-aligned. * * \note Otherwise, if the Alignment is larger that the scalar size, we rely on the assumptions that sizeof(Scalar) is a * power of 2. On the other hand, we do not assume that the array address is a multiple of sizeof(Scalar), as that fails for * example with Scalar=double on certain 32-bit platforms, see bug #79. * * There is also the variant first_aligned(const MatrixBase&) defined in DenseCoeffsBase.h. * \sa first_default_aligned() */ template EIGEN_DEVICE_FUNC inline Index first_aligned(const Scalar* array, Index size) { const Index ScalarSize = sizeof(Scalar); const Index AlignmentSize = Alignment / ScalarSize; const Index AlignmentMask = AlignmentSize-1; if(AlignmentSize<=1) { // Either the requested alignment if smaller than a scalar, or it exactly match a 1 scalar // so that all elements of the array have the same alignment. return 0; } else if( (UIntPtr(array) & (sizeof(Scalar)-1)) || (Alignment%ScalarSize)!=0) { // The array is not aligned to the size of a single scalar, or the requested alignment is not a multiple of the scalar size. // Consequently, no element of the array is well aligned. return size; } else { Index first = (AlignmentSize - (Index((UIntPtr(array)/sizeof(Scalar))) & AlignmentMask)) & AlignmentMask; return (first < size) ? first : size; } } /** \internal Returns the index of the first element of the array that is well aligned with respect the largest packet requirement. * \sa first_aligned(Scalar*,Index) and first_default_aligned(DenseBase) */ template EIGEN_DEVICE_FUNC inline Index first_default_aligned(const Scalar* array, Index size) { typedef typename packet_traits::type DefaultPacketType; return first_aligned::alignment>(array, size); } /** \internal Returns the smallest integer multiple of \a base and greater or equal to \a size */ template inline Index first_multiple(Index size, Index base) { return ((size+base-1)/base)*base; } // std::copy is much slower than memcpy, so let's introduce a smart_copy which // use memcpy on trivial types, i.e., on types that does not require an initialization ctor. template struct smart_copy_helper; template EIGEN_DEVICE_FUNC void smart_copy(const T* start, const T* end, T* target) { smart_copy_helper::RequireInitialization>::run(start, end, target); } template struct smart_copy_helper { EIGEN_DEVICE_FUNC static inline void run(const T* start, const T* end, T* target) { IntPtr size = IntPtr(end)-IntPtr(start); if(size==0) return; eigen_internal_assert(start!=0 && end!=0 && target!=0); EIGEN_USING_STD(memcpy) memcpy(target, start, size); } }; template struct smart_copy_helper { EIGEN_DEVICE_FUNC static inline void run(const T* start, const T* end, T* target) { std::copy(start, end, target); } }; // intelligent memmove. falls back to std::memmove for POD types, uses std::copy otherwise. template struct smart_memmove_helper; template void smart_memmove(const T* start, const T* end, T* target) { smart_memmove_helper::RequireInitialization>::run(start, end, target); } template struct smart_memmove_helper { static inline void run(const T* start, const T* end, T* target) { IntPtr size = IntPtr(end)-IntPtr(start); if(size==0) return; eigen_internal_assert(start!=0 && end!=0 && target!=0); std::memmove(target, start, size); } }; template struct smart_memmove_helper { static inline void run(const T* start, const T* end, T* target) { if (UIntPtr(target) < UIntPtr(start)) { std::copy(start, end, target); } else { std::ptrdiff_t count = (std::ptrdiff_t(end)-std::ptrdiff_t(start)) / sizeof(T); std::copy_backward(start, end, target + count); } } }; #if EIGEN_HAS_RVALUE_REFERENCES template EIGEN_DEVICE_FUNC T* smart_move(T* start, T* end, T* target) { return std::move(start, end, target); } #else template EIGEN_DEVICE_FUNC T* smart_move(T* start, T* end, T* target) { return std::copy(start, end, target); } #endif /***************************************************************************** *** Implementation of runtime stack allocation (falling back to malloc) *** *****************************************************************************/ // you can overwrite Eigen's default behavior regarding alloca by defining EIGEN_ALLOCA // to the appropriate stack allocation function #if ! defined EIGEN_ALLOCA && ! defined EIGEN_GPU_COMPILE_PHASE #if EIGEN_OS_LINUX || EIGEN_OS_MAC || (defined alloca) #define EIGEN_ALLOCA alloca #elif EIGEN_COMP_MSVC #define EIGEN_ALLOCA _alloca #endif #endif // With clang -Oz -mthumb, alloca changes the stack pointer in a way that is // not allowed in Thumb2. -DEIGEN_STACK_ALLOCATION_LIMIT=0 doesn't work because // the compiler still emits bad code because stack allocation checks use "<=". // TODO: Eliminate after https://bugs.llvm.org/show_bug.cgi?id=23772 // is fixed. #if defined(__clang__) && defined(__thumb__) #undef EIGEN_ALLOCA #endif // This helper class construct the allocated memory, and takes care of destructing and freeing the handled data // at destruction time. In practice this helper class is mainly useful to avoid memory leak in case of exceptions. template class aligned_stack_memory_handler : noncopyable { public: /* Creates a stack_memory_handler responsible for the buffer \a ptr of size \a size. * Note that \a ptr can be 0 regardless of the other parameters. * This constructor takes care of constructing/initializing the elements of the buffer if required by the scalar type T (see NumTraits::RequireInitialization). * In this case, the buffer elements will also be destructed when this handler will be destructed. * Finally, if \a dealloc is true, then the pointer \a ptr is freed. **/ EIGEN_DEVICE_FUNC aligned_stack_memory_handler(T* ptr, std::size_t size, bool dealloc) : m_ptr(ptr), m_size(size), m_deallocate(dealloc) { if(NumTraits::RequireInitialization && m_ptr) Eigen::internal::construct_elements_of_array(m_ptr, size); } EIGEN_DEVICE_FUNC ~aligned_stack_memory_handler() { if(NumTraits::RequireInitialization && m_ptr) Eigen::internal::destruct_elements_of_array(m_ptr, m_size); if(m_deallocate) Eigen::internal::aligned_free(m_ptr); } protected: T* m_ptr; std::size_t m_size; bool m_deallocate; }; #ifdef EIGEN_ALLOCA template::Evaluate && Xpr::MaxSizeAtCompileTime==Dynamic > struct local_nested_eval_wrapper { static const bool NeedExternalBuffer = false; typedef typename Xpr::Scalar Scalar; typedef typename nested_eval::type ObjectType; ObjectType object; EIGEN_DEVICE_FUNC local_nested_eval_wrapper(const Xpr& xpr, Scalar* ptr) : object(xpr) { EIGEN_UNUSED_VARIABLE(ptr); eigen_internal_assert(ptr==0); } }; template struct local_nested_eval_wrapper { static const bool NeedExternalBuffer = true; typedef typename Xpr::Scalar Scalar; typedef typename plain_object_eval::type PlainObject; typedef Map ObjectType; ObjectType object; EIGEN_DEVICE_FUNC local_nested_eval_wrapper(const Xpr& xpr, Scalar* ptr) : object(ptr==0 ? reinterpret_cast(Eigen::internal::aligned_malloc(sizeof(Scalar)*xpr.size())) : ptr, xpr.rows(), xpr.cols()), m_deallocate(ptr==0) { if(NumTraits::RequireInitialization && object.data()) Eigen::internal::construct_elements_of_array(object.data(), object.size()); object = xpr; } EIGEN_DEVICE_FUNC ~local_nested_eval_wrapper() { if(NumTraits::RequireInitialization && object.data()) Eigen::internal::destruct_elements_of_array(object.data(), object.size()); if(m_deallocate) Eigen::internal::aligned_free(object.data()); } private: bool m_deallocate; }; #endif // EIGEN_ALLOCA template class scoped_array : noncopyable { T* m_ptr; public: explicit scoped_array(std::ptrdiff_t size) { m_ptr = new T[size]; } ~scoped_array() { delete[] m_ptr; } T& operator[](std::ptrdiff_t i) { return m_ptr[i]; } const T& operator[](std::ptrdiff_t i) const { return m_ptr[i]; } T* &ptr() { return m_ptr; } const T* ptr() const { return m_ptr; } operator const T*() const { return m_ptr; } }; template void swap(scoped_array &a,scoped_array &b) { std::swap(a.ptr(),b.ptr()); } } // end namespace internal /** \internal * * The macro ei_declare_aligned_stack_constructed_variable(TYPE,NAME,SIZE,BUFFER) declares, allocates, * and construct an aligned buffer named NAME of SIZE elements of type TYPE on the stack * if the size in bytes is smaller than EIGEN_STACK_ALLOCATION_LIMIT, and if stack allocation is supported by the platform * (currently, this is Linux, OSX and Visual Studio only). Otherwise the memory is allocated on the heap. * The allocated buffer is automatically deleted when exiting the scope of this declaration. * If BUFFER is non null, then the declared variable is simply an alias for BUFFER, and no allocation/deletion occurs. * Here is an example: * \code * { * ei_declare_aligned_stack_constructed_variable(float,data,size,0); * // use data[0] to data[size-1] * } * \endcode * The underlying stack allocation function can controlled with the EIGEN_ALLOCA preprocessor token. * * The macro ei_declare_local_nested_eval(XPR_T,XPR,N,NAME) is analogue to * \code * typename internal::nested_eval::type NAME(XPR); * \endcode * with the advantage of using aligned stack allocation even if the maximal size of XPR at compile time is unknown. * This is accomplished through alloca if this later is supported and if the required number of bytes * is below EIGEN_STACK_ALLOCATION_LIMIT. */ #ifdef EIGEN_ALLOCA #if EIGEN_DEFAULT_ALIGN_BYTES>0 // We always manually re-align the result of EIGEN_ALLOCA. // If alloca is already aligned, the compiler should be smart enough to optimize away the re-alignment. #define EIGEN_ALIGNED_ALLOCA(SIZE) reinterpret_cast((internal::UIntPtr(EIGEN_ALLOCA(SIZE+EIGEN_DEFAULT_ALIGN_BYTES-1)) + EIGEN_DEFAULT_ALIGN_BYTES-1) & ~(std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1))) #else #define EIGEN_ALIGNED_ALLOCA(SIZE) EIGEN_ALLOCA(SIZE) #endif #define ei_declare_aligned_stack_constructed_variable(TYPE,NAME,SIZE,BUFFER) \ Eigen::internal::check_size_for_overflow(SIZE); \ TYPE* NAME = (BUFFER)!=0 ? (BUFFER) \ : reinterpret_cast( \ (sizeof(TYPE)*SIZE<=EIGEN_STACK_ALLOCATION_LIMIT) ? EIGEN_ALIGNED_ALLOCA(sizeof(TYPE)*SIZE) \ : Eigen::internal::aligned_malloc(sizeof(TYPE)*SIZE) ); \ Eigen::internal::aligned_stack_memory_handler EIGEN_CAT(NAME,_stack_memory_destructor)((BUFFER)==0 ? NAME : 0,SIZE,sizeof(TYPE)*SIZE>EIGEN_STACK_ALLOCATION_LIMIT) #define ei_declare_local_nested_eval(XPR_T,XPR,N,NAME) \ Eigen::internal::local_nested_eval_wrapper EIGEN_CAT(NAME,_wrapper)(XPR, reinterpret_cast( \ ( (Eigen::internal::local_nested_eval_wrapper::NeedExternalBuffer) && ((sizeof(typename XPR_T::Scalar)*XPR.size())<=EIGEN_STACK_ALLOCATION_LIMIT) ) \ ? EIGEN_ALIGNED_ALLOCA( sizeof(typename XPR_T::Scalar)*XPR.size() ) : 0 ) ) ; \ typename Eigen::internal::local_nested_eval_wrapper::ObjectType NAME(EIGEN_CAT(NAME,_wrapper).object) #else #define ei_declare_aligned_stack_constructed_variable(TYPE,NAME,SIZE,BUFFER) \ Eigen::internal::check_size_for_overflow(SIZE); \ TYPE* NAME = (BUFFER)!=0 ? BUFFER : reinterpret_cast(Eigen::internal::aligned_malloc(sizeof(TYPE)*SIZE)); \ Eigen::internal::aligned_stack_memory_handler EIGEN_CAT(NAME,_stack_memory_destructor)((BUFFER)==0 ? NAME : 0,SIZE,true) #define ei_declare_local_nested_eval(XPR_T,XPR,N,NAME) typename Eigen::internal::nested_eval::type NAME(XPR) #endif /***************************************************************************** *** Implementation of EIGEN_MAKE_ALIGNED_OPERATOR_NEW [_IF] *** *****************************************************************************/ #if EIGEN_HAS_CXX17_OVERALIGN // C++17 -> no need to bother about alignment anymore :) #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign) #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar,Size) #else // HIP does not support new/delete on device. #if EIGEN_MAX_ALIGN_BYTES!=0 && !defined(EIGEN_HIP_DEVICE_COMPILE) #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \ EIGEN_DEVICE_FUNC \ void* operator new(std::size_t size, const std::nothrow_t&) EIGEN_NO_THROW { \ EIGEN_TRY { return Eigen::internal::conditional_aligned_malloc(size); } \ EIGEN_CATCH (...) { return 0; } \ } #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign) \ EIGEN_DEVICE_FUNC \ void *operator new(std::size_t size) { \ return Eigen::internal::conditional_aligned_malloc(size); \ } \ EIGEN_DEVICE_FUNC \ void *operator new[](std::size_t size) { \ return Eigen::internal::conditional_aligned_malloc(size); \ } \ EIGEN_DEVICE_FUNC \ void operator delete(void * ptr) EIGEN_NO_THROW { Eigen::internal::conditional_aligned_free(ptr); } \ EIGEN_DEVICE_FUNC \ void operator delete[](void * ptr) EIGEN_NO_THROW { Eigen::internal::conditional_aligned_free(ptr); } \ EIGEN_DEVICE_FUNC \ void operator delete(void * ptr, std::size_t /* sz */) EIGEN_NO_THROW { Eigen::internal::conditional_aligned_free(ptr); } \ EIGEN_DEVICE_FUNC \ void operator delete[](void * ptr, std::size_t /* sz */) EIGEN_NO_THROW { Eigen::internal::conditional_aligned_free(ptr); } \ /* in-place new and delete. since (at least afaik) there is no actual */ \ /* memory allocated we can safely let the default implementation handle */ \ /* this particular case. */ \ EIGEN_DEVICE_FUNC \ static void *operator new(std::size_t size, void *ptr) { return ::operator new(size,ptr); } \ EIGEN_DEVICE_FUNC \ static void *operator new[](std::size_t size, void* ptr) { return ::operator new[](size,ptr); } \ EIGEN_DEVICE_FUNC \ void operator delete(void * memory, void *ptr) EIGEN_NO_THROW { return ::operator delete(memory,ptr); } \ EIGEN_DEVICE_FUNC \ void operator delete[](void * memory, void *ptr) EIGEN_NO_THROW { return ::operator delete[](memory,ptr); } \ /* nothrow-new (returns zero instead of std::bad_alloc) */ \ EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \ EIGEN_DEVICE_FUNC \ void operator delete(void *ptr, const std::nothrow_t&) EIGEN_NO_THROW { \ Eigen::internal::conditional_aligned_free(ptr); \ } \ typedef void eigen_aligned_operator_new_marker_type; #else #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign) #endif #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(true) #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar,Size) \ EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(bool( \ ((Size)!=Eigen::Dynamic) && \ (((EIGEN_MAX_ALIGN_BYTES>=16) && ((sizeof(Scalar)*(Size))%(EIGEN_MAX_ALIGN_BYTES )==0)) || \ ((EIGEN_MAX_ALIGN_BYTES>=32) && ((sizeof(Scalar)*(Size))%(EIGEN_MAX_ALIGN_BYTES/2)==0)) || \ ((EIGEN_MAX_ALIGN_BYTES>=64) && ((sizeof(Scalar)*(Size))%(EIGEN_MAX_ALIGN_BYTES/4)==0)) ))) #endif /****************************************************************************/ /** \class aligned_allocator * \ingroup Core_Module * * \brief STL compatible allocator to use with types requiring a non standrad alignment. * * The memory is aligned as for dynamically aligned matrix/array types such as MatrixXd. * By default, it will thus provide at least 16 bytes alignment and more in following cases: * - 32 bytes alignment if AVX is enabled. * - 64 bytes alignment if AVX512 is enabled. * * This can be controlled using the \c EIGEN_MAX_ALIGN_BYTES macro as documented * \link TopicPreprocessorDirectivesPerformance there \endlink. * * Example: * \code * // Matrix4f requires 16 bytes alignment: * std::map< int, Matrix4f, std::less, * aligned_allocator > > my_map_mat4; * // Vector3f does not require 16 bytes alignment, no need to use Eigen's allocator: * std::map< int, Vector3f > my_map_vec3; * \endcode * * \sa \blank \ref TopicStlContainers. */ template class aligned_allocator : public std::allocator { public: typedef std::size_t size_type; typedef std::ptrdiff_t difference_type; typedef T* pointer; typedef const T* const_pointer; typedef T& reference; typedef const T& const_reference; typedef T value_type; template struct rebind { typedef aligned_allocator other; }; aligned_allocator() : std::allocator() {} aligned_allocator(const aligned_allocator& other) : std::allocator(other) {} template aligned_allocator(const aligned_allocator& other) : std::allocator(other) {} ~aligned_allocator() {} #if EIGEN_COMP_GNUC_STRICT && EIGEN_GNUC_AT_LEAST(7,0) // In gcc std::allocator::max_size() is bugged making gcc triggers a warning: // eigen/Eigen/src/Core/util/Memory.h:189:12: warning: argument 1 value '18446744073709551612' exceeds maximum object size 9223372036854775807 // See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87544 size_type max_size() const { return (std::numeric_limits::max)()/sizeof(T); } #endif pointer allocate(size_type num, const void* /*hint*/ = 0) { internal::check_size_for_overflow(num); return static_cast( internal::aligned_malloc(num * sizeof(T)) ); } void deallocate(pointer p, size_type /*num*/) { internal::aligned_free(p); } }; //---------- Cache sizes ---------- #if !defined(EIGEN_NO_CPUID) # if EIGEN_COMP_GNUC && EIGEN_ARCH_i386_OR_x86_64 # if defined(__PIC__) && EIGEN_ARCH_i386 // Case for x86 with PIC # define EIGEN_CPUID(abcd,func,id) \ __asm__ __volatile__ ("xchgl %%ebx, %k1;cpuid; xchgl %%ebx,%k1": "=a" (abcd[0]), "=&r" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "a" (func), "c" (id)); # elif defined(__PIC__) && EIGEN_ARCH_x86_64 // Case for x64 with PIC. In theory this is only a problem with recent gcc and with medium or large code model, not with the default small code model. // However, we cannot detect which code model is used, and the xchg overhead is negligible anyway. # define EIGEN_CPUID(abcd,func,id) \ __asm__ __volatile__ ("xchg{q}\t{%%}rbx, %q1; cpuid; xchg{q}\t{%%}rbx, %q1": "=a" (abcd[0]), "=&r" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "0" (func), "2" (id)); # else // Case for x86_64 or x86 w/o PIC # define EIGEN_CPUID(abcd,func,id) \ __asm__ __volatile__ ("cpuid": "=a" (abcd[0]), "=b" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "0" (func), "2" (id) ); # endif # elif EIGEN_COMP_MSVC # if (EIGEN_COMP_MSVC > 1500) && EIGEN_ARCH_i386_OR_x86_64 # define EIGEN_CPUID(abcd,func,id) __cpuidex((int*)abcd,func,id) # endif # endif #endif namespace internal { #ifdef EIGEN_CPUID inline bool cpuid_is_vendor(int abcd[4], const int vendor[3]) { return abcd[1]==vendor[0] && abcd[3]==vendor[1] && abcd[2]==vendor[2]; } inline void queryCacheSizes_intel_direct(int& l1, int& l2, int& l3) { int abcd[4]; l1 = l2 = l3 = 0; int cache_id = 0; int cache_type = 0; do { abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0; EIGEN_CPUID(abcd,0x4,cache_id); cache_type = (abcd[0] & 0x0F) >> 0; if(cache_type==1||cache_type==3) // data or unified cache { int cache_level = (abcd[0] & 0xE0) >> 5; // A[7:5] int ways = (abcd[1] & 0xFFC00000) >> 22; // B[31:22] int partitions = (abcd[1] & 0x003FF000) >> 12; // B[21:12] int line_size = (abcd[1] & 0x00000FFF) >> 0; // B[11:0] int sets = (abcd[2]); // C[31:0] int cache_size = (ways+1) * (partitions+1) * (line_size+1) * (sets+1); switch(cache_level) { case 1: l1 = cache_size; break; case 2: l2 = cache_size; break; case 3: l3 = cache_size; break; default: break; } } cache_id++; } while(cache_type>0 && cache_id<16); } inline void queryCacheSizes_intel_codes(int& l1, int& l2, int& l3) { int abcd[4]; abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0; l1 = l2 = l3 = 0; EIGEN_CPUID(abcd,0x00000002,0); unsigned char * bytes = reinterpret_cast(abcd)+2; bool check_for_p2_core2 = false; for(int i=0; i<14; ++i) { switch(bytes[i]) { case 0x0A: l1 = 8; break; // 0Ah data L1 cache, 8 KB, 2 ways, 32 byte lines case 0x0C: l1 = 16; break; // 0Ch data L1 cache, 16 KB, 4 ways, 32 byte lines case 0x0E: l1 = 24; break; // 0Eh data L1 cache, 24 KB, 6 ways, 64 byte lines case 0x10: l1 = 16; break; // 10h data L1 cache, 16 KB, 4 ways, 32 byte lines (IA-64) case 0x15: l1 = 16; break; // 15h code L1 cache, 16 KB, 4 ways, 32 byte lines (IA-64) case 0x2C: l1 = 32; break; // 2Ch data L1 cache, 32 KB, 8 ways, 64 byte lines case 0x30: l1 = 32; break; // 30h code L1 cache, 32 KB, 8 ways, 64 byte lines case 0x60: l1 = 16; break; // 60h data L1 cache, 16 KB, 8 ways, 64 byte lines, sectored case 0x66: l1 = 8; break; // 66h data L1 cache, 8 KB, 4 ways, 64 byte lines, sectored case 0x67: l1 = 16; break; // 67h data L1 cache, 16 KB, 4 ways, 64 byte lines, sectored case 0x68: l1 = 32; break; // 68h data L1 cache, 32 KB, 4 ways, 64 byte lines, sectored case 0x1A: l2 = 96; break; // code and data L2 cache, 96 KB, 6 ways, 64 byte lines (IA-64) case 0x22: l3 = 512; break; // code and data L3 cache, 512 KB, 4 ways (!), 64 byte lines, dual-sectored case 0x23: l3 = 1024; break; // code and data L3 cache, 1024 KB, 8 ways, 64 byte lines, dual-sectored case 0x25: l3 = 2048; break; // code and data L3 cache, 2048 KB, 8 ways, 64 byte lines, dual-sectored case 0x29: l3 = 4096; break; // code and data L3 cache, 4096 KB, 8 ways, 64 byte lines, dual-sectored case 0x39: l2 = 128; break; // code and data L2 cache, 128 KB, 4 ways, 64 byte lines, sectored case 0x3A: l2 = 192; break; // code and data L2 cache, 192 KB, 6 ways, 64 byte lines, sectored case 0x3B: l2 = 128; break; // code and data L2 cache, 128 KB, 2 ways, 64 byte lines, sectored case 0x3C: l2 = 256; break; // code and data L2 cache, 256 KB, 4 ways, 64 byte lines, sectored case 0x3D: l2 = 384; break; // code and data L2 cache, 384 KB, 6 ways, 64 byte lines, sectored case 0x3E: l2 = 512; break; // code and data L2 cache, 512 KB, 4 ways, 64 byte lines, sectored case 0x40: l2 = 0; break; // no integrated L2 cache (P6 core) or L3 cache (P4 core) case 0x41: l2 = 128; break; // code and data L2 cache, 128 KB, 4 ways, 32 byte lines case 0x42: l2 = 256; break; // code and data L2 cache, 256 KB, 4 ways, 32 byte lines case 0x43: l2 = 512; break; // code and data L2 cache, 512 KB, 4 ways, 32 byte lines case 0x44: l2 = 1024; break; // code and data L2 cache, 1024 KB, 4 ways, 32 byte lines case 0x45: l2 = 2048; break; // code and data L2 cache, 2048 KB, 4 ways, 32 byte lines case 0x46: l3 = 4096; break; // code and data L3 cache, 4096 KB, 4 ways, 64 byte lines case 0x47: l3 = 8192; break; // code and data L3 cache, 8192 KB, 8 ways, 64 byte lines case 0x48: l2 = 3072; break; // code and data L2 cache, 3072 KB, 12 ways, 64 byte lines case 0x49: if(l2!=0) l3 = 4096; else {check_for_p2_core2=true; l3 = l2 = 4096;} break;// code and data L3 cache, 4096 KB, 16 ways, 64 byte lines (P4) or L2 for core2 case 0x4A: l3 = 6144; break; // code and data L3 cache, 6144 KB, 12 ways, 64 byte lines case 0x4B: l3 = 8192; break; // code and data L3 cache, 8192 KB, 16 ways, 64 byte lines case 0x4C: l3 = 12288; break; // code and data L3 cache, 12288 KB, 12 ways, 64 byte lines case 0x4D: l3 = 16384; break; // code and data L3 cache, 16384 KB, 16 ways, 64 byte lines case 0x4E: l2 = 6144; break; // code and data L2 cache, 6144 KB, 24 ways, 64 byte lines case 0x78: l2 = 1024; break; // code and data L2 cache, 1024 KB, 4 ways, 64 byte lines case 0x79: l2 = 128; break; // code and data L2 cache, 128 KB, 8 ways, 64 byte lines, dual-sectored case 0x7A: l2 = 256; break; // code and data L2 cache, 256 KB, 8 ways, 64 byte lines, dual-sectored case 0x7B: l2 = 512; break; // code and data L2 cache, 512 KB, 8 ways, 64 byte lines, dual-sectored case 0x7C: l2 = 1024; break; // code and data L2 cache, 1024 KB, 8 ways, 64 byte lines, dual-sectored case 0x7D: l2 = 2048; break; // code and data L2 cache, 2048 KB, 8 ways, 64 byte lines case 0x7E: l2 = 256; break; // code and data L2 cache, 256 KB, 8 ways, 128 byte lines, sect. (IA-64) case 0x7F: l2 = 512; break; // code and data L2 cache, 512 KB, 2 ways, 64 byte lines case 0x80: l2 = 512; break; // code and data L2 cache, 512 KB, 8 ways, 64 byte lines case 0x81: l2 = 128; break; // code and data L2 cache, 128 KB, 8 ways, 32 byte lines case 0x82: l2 = 256; break; // code and data L2 cache, 256 KB, 8 ways, 32 byte lines case 0x83: l2 = 512; break; // code and data L2 cache, 512 KB, 8 ways, 32 byte lines case 0x84: l2 = 1024; break; // code and data L2 cache, 1024 KB, 8 ways, 32 byte lines case 0x85: l2 = 2048; break; // code and data L2 cache, 2048 KB, 8 ways, 32 byte lines case 0x86: l2 = 512; break; // code and data L2 cache, 512 KB, 4 ways, 64 byte lines case 0x87: l2 = 1024; break; // code and data L2 cache, 1024 KB, 8 ways, 64 byte lines case 0x88: l3 = 2048; break; // code and data L3 cache, 2048 KB, 4 ways, 64 byte lines (IA-64) case 0x89: l3 = 4096; break; // code and data L3 cache, 4096 KB, 4 ways, 64 byte lines (IA-64) case 0x8A: l3 = 8192; break; // code and data L3 cache, 8192 KB, 4 ways, 64 byte lines (IA-64) case 0x8D: l3 = 3072; break; // code and data L3 cache, 3072 KB, 12 ways, 128 byte lines (IA-64) default: break; } } if(check_for_p2_core2 && l2 == l3) l3 = 0; l1 *= 1024; l2 *= 1024; l3 *= 1024; } inline void queryCacheSizes_intel(int& l1, int& l2, int& l3, int max_std_funcs) { if(max_std_funcs>=4) queryCacheSizes_intel_direct(l1,l2,l3); else if(max_std_funcs>=2) queryCacheSizes_intel_codes(l1,l2,l3); else l1 = l2 = l3 = 0; } inline void queryCacheSizes_amd(int& l1, int& l2, int& l3) { int abcd[4]; abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0; // First query the max supported function. EIGEN_CPUID(abcd,0x80000000,0); if(static_cast(abcd[0]) >= static_cast(0x80000006)) { EIGEN_CPUID(abcd,0x80000005,0); l1 = (abcd[2] >> 24) * 1024; // C[31:24] = L1 size in KB abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0; EIGEN_CPUID(abcd,0x80000006,0); l2 = (abcd[2] >> 16) * 1024; // C[31;16] = l2 cache size in KB l3 = ((abcd[3] & 0xFFFC000) >> 18) * 512 * 1024; // D[31;18] = l3 cache size in 512KB } else { l1 = l2 = l3 = 0; } } #endif /** \internal * Queries and returns the cache sizes in Bytes of the L1, L2, and L3 data caches respectively */ inline void queryCacheSizes(int& l1, int& l2, int& l3) { #ifdef EIGEN_CPUID int abcd[4]; const int GenuineIntel[] = {0x756e6547, 0x49656e69, 0x6c65746e}; const int AuthenticAMD[] = {0x68747541, 0x69746e65, 0x444d4163}; const int AMDisbetter_[] = {0x69444d41, 0x74656273, 0x21726574}; // "AMDisbetter!" // identify the CPU vendor EIGEN_CPUID(abcd,0x0,0); int max_std_funcs = abcd[0]; if(cpuid_is_vendor(abcd,GenuineIntel)) queryCacheSizes_intel(l1,l2,l3,max_std_funcs); else if(cpuid_is_vendor(abcd,AuthenticAMD) || cpuid_is_vendor(abcd,AMDisbetter_)) queryCacheSizes_amd(l1,l2,l3); else // by default let's use Intel's API queryCacheSizes_intel(l1,l2,l3,max_std_funcs); // here is the list of other vendors: // ||cpuid_is_vendor(abcd,"VIA VIA VIA ") // ||cpuid_is_vendor(abcd,"CyrixInstead") // ||cpuid_is_vendor(abcd,"CentaurHauls") // ||cpuid_is_vendor(abcd,"GenuineTMx86") // ||cpuid_is_vendor(abcd,"TransmetaCPU") // ||cpuid_is_vendor(abcd,"RiseRiseRise") // ||cpuid_is_vendor(abcd,"Geode by NSC") // ||cpuid_is_vendor(abcd,"SiS SiS SiS ") // ||cpuid_is_vendor(abcd,"UMC UMC UMC ") // ||cpuid_is_vendor(abcd,"NexGenDriven") #else l1 = l2 = l3 = -1; #endif } /** \internal * \returns the size in Bytes of the L1 data cache */ inline int queryL1CacheSize() { int l1(-1), l2, l3; queryCacheSizes(l1,l2,l3); return l1; } /** \internal * \returns the size in Bytes of the L2 or L3 cache if this later is present */ inline int queryTopLevelCacheSize() { int l1, l2(-1), l3(-1); queryCacheSizes(l1,l2,l3); return (std::max)(l2,l3); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_MEMORY_H RcppEigen/inst/include/Eigen/src/Core/util/BlasUtil.h0000755000176200001440000005516414567757725022150 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2009-2010 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_BLASUTIL_H #define EIGEN_BLASUTIL_H // This file contains many lightweight helper classes used to // implement and control fast level 2 and level 3 BLAS-like routines. namespace Eigen { namespace internal { // forward declarations template struct gebp_kernel; template struct gemm_pack_rhs; template struct gemm_pack_lhs; template< typename Index, typename LhsScalar, int LhsStorageOrder, bool ConjugateLhs, typename RhsScalar, int RhsStorageOrder, bool ConjugateRhs, int ResStorageOrder, int ResInnerStride> struct general_matrix_matrix_product; template struct general_matrix_vector_product; template struct get_factor { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE To run(const From& x) { return To(x); } }; template struct get_factor::Real> { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE typename NumTraits::Real run(const Scalar& x) { return numext::real(x); } }; template class BlasVectorMapper { public: EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE BlasVectorMapper(Scalar *data) : m_data(data) {} EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar operator()(Index i) const { return m_data[i]; } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet load(Index i) const { return ploadt(m_data + i); } template EIGEN_DEVICE_FUNC bool aligned(Index i) const { return (UIntPtr(m_data+i)%sizeof(Packet))==0; } protected: Scalar* m_data; }; template class BlasLinearMapper; template class BlasLinearMapper { public: EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE BlasLinearMapper(Scalar *data, Index incr=1) : m_data(data) { EIGEN_ONLY_USED_FOR_DEBUG(incr); eigen_assert(incr==1); } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void prefetch(int i) const { internal::prefetch(&operator()(i)); } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar& operator()(Index i) const { return m_data[i]; } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i) const { return ploadt(m_data + i); } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacket(Index i, const PacketType &p) const { pstoret(m_data + i, p); } protected: Scalar *m_data; }; // Lightweight helper class to access matrix coefficients. template class blas_data_mapper; // TMP to help PacketBlock store implementation. // There's currently no known use case for PacketBlock load. // The default implementation assumes ColMajor order. // It always store each packet sequentially one `stride` apart. template struct PacketBlockManagement { PacketBlockManagement pbm; EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(Scalar *to, const Index stride, Index i, Index j, const PacketBlock &block) const { pbm.store(to, stride, i, j, block); pstoreu(to + i + (j + idx)*stride, block.packet[idx]); } }; // PacketBlockManagement specialization to take care of RowMajor order without ifs. template struct PacketBlockManagement { PacketBlockManagement pbm; EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(Scalar *to, const Index stride, Index i, Index j, const PacketBlock &block) const { pbm.store(to, stride, i, j, block); pstoreu(to + j + (i + idx)*stride, block.packet[idx]); } }; template struct PacketBlockManagement { EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(Scalar *to, const Index stride, Index i, Index j, const PacketBlock &block) const { EIGEN_UNUSED_VARIABLE(to); EIGEN_UNUSED_VARIABLE(stride); EIGEN_UNUSED_VARIABLE(i); EIGEN_UNUSED_VARIABLE(j); EIGEN_UNUSED_VARIABLE(block); } }; template struct PacketBlockManagement { EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(Scalar *to, const Index stride, Index i, Index j, const PacketBlock &block) const { EIGEN_UNUSED_VARIABLE(to); EIGEN_UNUSED_VARIABLE(stride); EIGEN_UNUSED_VARIABLE(i); EIGEN_UNUSED_VARIABLE(j); EIGEN_UNUSED_VARIABLE(block); } }; template class blas_data_mapper { public: typedef BlasLinearMapper LinearMapper; typedef BlasVectorMapper VectorMapper; EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE blas_data_mapper(Scalar* data, Index stride, Index incr=1) : m_data(data), m_stride(stride) { EIGEN_ONLY_USED_FOR_DEBUG(incr); eigen_assert(incr==1); } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE blas_data_mapper getSubMapper(Index i, Index j) const { return blas_data_mapper(&operator()(i, j), m_stride); } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE LinearMapper getLinearMapper(Index i, Index j) const { return LinearMapper(&operator()(i, j)); } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE VectorMapper getVectorMapper(Index i, Index j) const { return VectorMapper(&operator()(i, j)); } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar& operator()(Index i, Index j) const { return m_data[StorageOrder==RowMajor ? j + i*m_stride : i + j*m_stride]; } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i, Index j) const { return ploadt(&operator()(i, j)); } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketT load(Index i, Index j) const { return ploadt(&operator()(i, j)); } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void scatterPacket(Index i, Index j, const SubPacket &p) const { pscatter(&operator()(i, j), p, m_stride); } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE SubPacket gatherPacket(Index i, Index j) const { return pgather(&operator()(i, j), m_stride); } EIGEN_DEVICE_FUNC const Index stride() const { return m_stride; } EIGEN_DEVICE_FUNC const Scalar* data() const { return m_data; } EIGEN_DEVICE_FUNC Index firstAligned(Index size) const { if (UIntPtr(m_data)%sizeof(Scalar)) { return -1; } return internal::first_default_aligned(m_data, size); } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacketBlock(Index i, Index j, const PacketBlock &block) const { PacketBlockManagement pbm; pbm.store(m_data, m_stride, i, j, block); } protected: Scalar* EIGEN_RESTRICT m_data; const Index m_stride; }; // Implementation of non-natural increment (i.e. inner-stride != 1) // The exposed API is not complete yet compared to the Incr==1 case // because some features makes less sense in this case. template class BlasLinearMapper { public: EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE BlasLinearMapper(Scalar *data,Index incr) : m_data(data), m_incr(incr) {} EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void prefetch(int i) const { internal::prefetch(&operator()(i)); } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar& operator()(Index i) const { return m_data[i*m_incr.value()]; } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i) const { return pgather(m_data + i*m_incr.value(), m_incr.value()); } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacket(Index i, const PacketType &p) const { pscatter(m_data + i*m_incr.value(), p, m_incr.value()); } protected: Scalar *m_data; const internal::variable_if_dynamic m_incr; }; template class blas_data_mapper { public: typedef BlasLinearMapper LinearMapper; EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE blas_data_mapper(Scalar* data, Index stride, Index incr) : m_data(data), m_stride(stride), m_incr(incr) {} EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE blas_data_mapper getSubMapper(Index i, Index j) const { return blas_data_mapper(&operator()(i, j), m_stride, m_incr.value()); } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE LinearMapper getLinearMapper(Index i, Index j) const { return LinearMapper(&operator()(i, j), m_incr.value()); } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar& operator()(Index i, Index j) const { return m_data[StorageOrder==RowMajor ? j*m_incr.value() + i*m_stride : i*m_incr.value() + j*m_stride]; } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketType loadPacket(Index i, Index j) const { return pgather(&operator()(i, j),m_incr.value()); } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketT load(Index i, Index j) const { return pgather(&operator()(i, j),m_incr.value()); } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void scatterPacket(Index i, Index j, const SubPacket &p) const { pscatter(&operator()(i, j), p, m_stride); } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE SubPacket gatherPacket(Index i, Index j) const { return pgather(&operator()(i, j), m_stride); } // storePacketBlock_helper defines a way to access values inside the PacketBlock, this is essentially required by the Complex types. template struct storePacketBlock_helper { storePacketBlock_helper spbh; EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(const blas_data_mapper* sup, Index i, Index j, const PacketBlock& block) const { spbh.store(sup, i,j,block); for(int l = 0; l < unpacket_traits::size; l++) { ScalarT *v = &sup->operator()(i+l, j+idx); *v = block.packet[idx][l]; } } }; template struct storePacketBlock_helper, n, idx> { storePacketBlock_helper, n, idx-1> spbh; EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(const blas_data_mapper* sup, Index i, Index j, const PacketBlock& block) const { spbh.store(sup,i,j,block); for(int l = 0; l < unpacket_traits::size; l++) { std::complex *v = &sup->operator()(i+l, j+idx); v->real(block.packet[idx].v[2*l+0]); v->imag(block.packet[idx].v[2*l+1]); } } }; template struct storePacketBlock_helper, n, idx> { storePacketBlock_helper, n, idx-1> spbh; EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(const blas_data_mapper* sup, Index i, Index j, const PacketBlock& block) const { spbh.store(sup,i,j,block); for(int l = 0; l < unpacket_traits::size; l++) { std::complex *v = &sup->operator()(i+l, j+idx); v->real(block.packet[idx].v[2*l+0]); v->imag(block.packet[idx].v[2*l+1]); } } }; template struct storePacketBlock_helper { EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(const blas_data_mapper*, Index, Index, const PacketBlock& ) const { } }; template struct storePacketBlock_helper, n, -1> { EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(const blas_data_mapper*, Index, Index, const PacketBlock& ) const { } }; template struct storePacketBlock_helper, n, -1> { EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void store(const blas_data_mapper*, Index, Index, const PacketBlock& ) const { } }; // This function stores a PacketBlock on m_data, this approach is really quite slow compare to Incr=1 and should be avoided when possible. template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacketBlock(Index i, Index j, const PacketBlock&block) const { storePacketBlock_helper spb; spb.store(this, i,j,block); } protected: Scalar* EIGEN_RESTRICT m_data; const Index m_stride; const internal::variable_if_dynamic m_incr; }; // lightweight helper class to access matrix coefficients (const version) template class const_blas_data_mapper : public blas_data_mapper { public: EIGEN_ALWAYS_INLINE const_blas_data_mapper(const Scalar *data, Index stride) : blas_data_mapper(data, stride) {} EIGEN_ALWAYS_INLINE const_blas_data_mapper getSubMapper(Index i, Index j) const { return const_blas_data_mapper(&(this->operator()(i, j)), this->m_stride); } }; /* Helper class to analyze the factors of a Product expression. * In particular it allows to pop out operator-, scalar multiples, * and conjugate */ template struct blas_traits { typedef typename traits::Scalar Scalar; typedef const XprType& ExtractType; typedef XprType _ExtractType; enum { IsComplex = NumTraits::IsComplex, IsTransposed = false, NeedToConjugate = false, HasUsableDirectAccess = ( (int(XprType::Flags)&DirectAccessBit) && ( bool(XprType::IsVectorAtCompileTime) || int(inner_stride_at_compile_time::ret) == 1) ) ? 1 : 0, HasScalarFactor = false }; typedef typename conditional::type DirectLinearAccessType; static inline EIGEN_DEVICE_FUNC ExtractType extract(const XprType& x) { return x; } static inline EIGEN_DEVICE_FUNC const Scalar extractScalarFactor(const XprType&) { return Scalar(1); } }; // pop conjugate template struct blas_traits, NestedXpr> > : blas_traits { typedef blas_traits Base; typedef CwiseUnaryOp, NestedXpr> XprType; typedef typename Base::ExtractType ExtractType; enum { IsComplex = NumTraits::IsComplex, NeedToConjugate = Base::NeedToConjugate ? 0 : IsComplex }; static inline ExtractType extract(const XprType& x) { return Base::extract(x.nestedExpression()); } static inline Scalar extractScalarFactor(const XprType& x) { return conj(Base::extractScalarFactor(x.nestedExpression())); } }; // pop scalar multiple template struct blas_traits, const CwiseNullaryOp,Plain>, NestedXpr> > : blas_traits { enum { HasScalarFactor = true }; typedef blas_traits Base; typedef CwiseBinaryOp, const CwiseNullaryOp,Plain>, NestedXpr> XprType; typedef typename Base::ExtractType ExtractType; static inline EIGEN_DEVICE_FUNC ExtractType extract(const XprType& x) { return Base::extract(x.rhs()); } static inline EIGEN_DEVICE_FUNC Scalar extractScalarFactor(const XprType& x) { return x.lhs().functor().m_other * Base::extractScalarFactor(x.rhs()); } }; template struct blas_traits, NestedXpr, const CwiseNullaryOp,Plain> > > : blas_traits { enum { HasScalarFactor = true }; typedef blas_traits Base; typedef CwiseBinaryOp, NestedXpr, const CwiseNullaryOp,Plain> > XprType; typedef typename Base::ExtractType ExtractType; static inline ExtractType extract(const XprType& x) { return Base::extract(x.lhs()); } static inline Scalar extractScalarFactor(const XprType& x) { return Base::extractScalarFactor(x.lhs()) * x.rhs().functor().m_other; } }; template struct blas_traits, const CwiseNullaryOp,Plain1>, const CwiseNullaryOp,Plain2> > > : blas_traits,Plain1> > {}; // pop opposite template struct blas_traits, NestedXpr> > : blas_traits { enum { HasScalarFactor = true }; typedef blas_traits Base; typedef CwiseUnaryOp, NestedXpr> XprType; typedef typename Base::ExtractType ExtractType; static inline ExtractType extract(const XprType& x) { return Base::extract(x.nestedExpression()); } static inline Scalar extractScalarFactor(const XprType& x) { return - Base::extractScalarFactor(x.nestedExpression()); } }; // pop/push transpose template struct blas_traits > : blas_traits { typedef typename NestedXpr::Scalar Scalar; typedef blas_traits Base; typedef Transpose XprType; typedef Transpose ExtractType; // const to get rid of a compile error; anyway blas traits are only used on the RHS typedef Transpose _ExtractType; typedef typename conditional::type DirectLinearAccessType; enum { IsTransposed = Base::IsTransposed ? 0 : 1 }; static inline ExtractType extract(const XprType& x) { return ExtractType(Base::extract(x.nestedExpression())); } static inline Scalar extractScalarFactor(const XprType& x) { return Base::extractScalarFactor(x.nestedExpression()); } }; template struct blas_traits : blas_traits {}; template::HasUsableDirectAccess> struct extract_data_selector { EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static const typename T::Scalar* run(const T& m) { return blas_traits::extract(m).data(); } }; template struct extract_data_selector { static typename T::Scalar* run(const T&) { return 0; } }; template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE const typename T::Scalar* extract_data(const T& m) { return extract_data_selector::run(m); } /** * \c combine_scalar_factors extracts and multiplies factors from GEMM and GEMV products. * There is a specialization for booleans */ template struct combine_scalar_factors_impl { EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static ResScalar run(const Lhs& lhs, const Rhs& rhs) { return blas_traits::extractScalarFactor(lhs) * blas_traits::extractScalarFactor(rhs); } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static ResScalar run(const ResScalar& alpha, const Lhs& lhs, const Rhs& rhs) { return alpha * blas_traits::extractScalarFactor(lhs) * blas_traits::extractScalarFactor(rhs); } }; template struct combine_scalar_factors_impl { EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static bool run(const Lhs& lhs, const Rhs& rhs) { return blas_traits::extractScalarFactor(lhs) && blas_traits::extractScalarFactor(rhs); } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static bool run(const bool& alpha, const Lhs& lhs, const Rhs& rhs) { return alpha && blas_traits::extractScalarFactor(lhs) && blas_traits::extractScalarFactor(rhs); } }; template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE ResScalar combine_scalar_factors(const ResScalar& alpha, const Lhs& lhs, const Rhs& rhs) { return combine_scalar_factors_impl::run(alpha, lhs, rhs); } template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE ResScalar combine_scalar_factors(const Lhs& lhs, const Rhs& rhs) { return combine_scalar_factors_impl::run(lhs, rhs); } } // end namespace internal } // end namespace Eigen #endif // EIGEN_BLASUTIL_H RcppEigen/inst/include/Eigen/src/Core/MatrixBase.h0000644000176200001440000005646014567757725021510 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2006-2009 Benoit Jacob // Copyright (C) 2008 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_MATRIXBASE_H #define EIGEN_MATRIXBASE_H namespace Eigen { /** \class MatrixBase * \ingroup Core_Module * * \brief Base class for all dense matrices, vectors, and expressions * * This class is the base that is inherited by all matrix, vector, and related expression * types. Most of the Eigen API is contained in this class, and its base classes. Other important * classes for the Eigen API are Matrix, and VectorwiseOp. * * Note that some methods are defined in other modules such as the \ref LU_Module LU module * for all functions related to matrix inversions. * * \tparam Derived is the derived type, e.g. a matrix type, or an expression, etc. * * When writing a function taking Eigen objects as argument, if you want your function * to take as argument any matrix, vector, or expression, just let it take a * MatrixBase argument. As an example, here is a function printFirstRow which, given * a matrix, vector, or expression \a x, prints the first row of \a x. * * \code template void printFirstRow(const Eigen::MatrixBase& x) { cout << x.row(0) << endl; } * \endcode * * This class can be extended with the help of the plugin mechanism described on the page * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_MATRIXBASE_PLUGIN. * * \sa \blank \ref TopicClassHierarchy */ template class MatrixBase : public DenseBase { public: #ifndef EIGEN_PARSED_BY_DOXYGEN typedef MatrixBase StorageBaseType; typedef typename internal::traits::StorageKind StorageKind; typedef typename internal::traits::StorageIndex StorageIndex; typedef typename internal::traits::Scalar Scalar; typedef typename internal::packet_traits::type PacketScalar; typedef typename NumTraits::Real RealScalar; typedef DenseBase Base; using Base::RowsAtCompileTime; using Base::ColsAtCompileTime; using Base::SizeAtCompileTime; using Base::MaxRowsAtCompileTime; using Base::MaxColsAtCompileTime; using Base::MaxSizeAtCompileTime; using Base::IsVectorAtCompileTime; using Base::Flags; using Base::derived; using Base::const_cast_derived; using Base::rows; using Base::cols; using Base::size; using Base::coeff; using Base::coeffRef; using Base::lazyAssign; using Base::eval; using Base::operator-; using Base::operator+=; using Base::operator-=; using Base::operator*=; using Base::operator/=; typedef typename Base::CoeffReturnType CoeffReturnType; typedef typename Base::ConstTransposeReturnType ConstTransposeReturnType; typedef typename Base::RowXpr RowXpr; typedef typename Base::ColXpr ColXpr; #endif // not EIGEN_PARSED_BY_DOXYGEN #ifndef EIGEN_PARSED_BY_DOXYGEN /** type of the equivalent square matrix */ typedef Matrix SquareMatrixType; #endif // not EIGEN_PARSED_BY_DOXYGEN /** \returns the size of the main diagonal, which is min(rows(),cols()). * \sa rows(), cols(), SizeAtCompileTime. */ EIGEN_DEVICE_FUNC inline Index diagonalSize() const { return (numext::mini)(rows(),cols()); } typedef typename Base::PlainObject PlainObject; #ifndef EIGEN_PARSED_BY_DOXYGEN /** \internal Represents a matrix with all coefficients equal to one another*/ typedef CwiseNullaryOp,PlainObject> ConstantReturnType; /** \internal the return type of MatrixBase::adjoint() */ typedef typename internal::conditional::IsComplex, CwiseUnaryOp, ConstTransposeReturnType>, ConstTransposeReturnType >::type AdjointReturnType; /** \internal Return type of eigenvalues() */ typedef Matrix, internal::traits::ColsAtCompileTime, 1, ColMajor> EigenvaluesReturnType; /** \internal the return type of identity */ typedef CwiseNullaryOp,PlainObject> IdentityReturnType; /** \internal the return type of unit vectors */ typedef Block, SquareMatrixType>, internal::traits::RowsAtCompileTime, internal::traits::ColsAtCompileTime> BasisReturnType; #endif // not EIGEN_PARSED_BY_DOXYGEN #define EIGEN_CURRENT_STORAGE_BASE_CLASS Eigen::MatrixBase #define EIGEN_DOC_UNARY_ADDONS(X,Y) # include "../plugins/CommonCwiseBinaryOps.h" # include "../plugins/MatrixCwiseUnaryOps.h" # include "../plugins/MatrixCwiseBinaryOps.h" # ifdef EIGEN_MATRIXBASE_PLUGIN # include EIGEN_MATRIXBASE_PLUGIN # endif #undef EIGEN_CURRENT_STORAGE_BASE_CLASS #undef EIGEN_DOC_UNARY_ADDONS /** Special case of the template operator=, in order to prevent the compiler * from generating a default operator= (issue hit with g++ 4.1) */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const MatrixBase& other); // We cannot inherit here via Base::operator= since it is causing // trouble with MSVC. template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const DenseBase& other); template EIGEN_DEVICE_FUNC Derived& operator=(const EigenBase& other); template EIGEN_DEVICE_FUNC Derived& operator=(const ReturnByValue& other); template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator+=(const MatrixBase& other); template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator-=(const MatrixBase& other); template EIGEN_DEVICE_FUNC const Product operator*(const MatrixBase &other) const; template EIGEN_DEVICE_FUNC const Product lazyProduct(const MatrixBase &other) const; template Derived& operator*=(const EigenBase& other); template void applyOnTheLeft(const EigenBase& other); template void applyOnTheRight(const EigenBase& other); template EIGEN_DEVICE_FUNC const Product operator*(const DiagonalBase &diagonal) const; template EIGEN_DEVICE_FUNC typename ScalarBinaryOpTraits::Scalar,typename internal::traits::Scalar>::ReturnType dot(const MatrixBase& other) const; EIGEN_DEVICE_FUNC RealScalar squaredNorm() const; EIGEN_DEVICE_FUNC RealScalar norm() const; RealScalar stableNorm() const; RealScalar blueNorm() const; RealScalar hypotNorm() const; EIGEN_DEVICE_FUNC const PlainObject normalized() const; EIGEN_DEVICE_FUNC const PlainObject stableNormalized() const; EIGEN_DEVICE_FUNC void normalize(); EIGEN_DEVICE_FUNC void stableNormalize(); EIGEN_DEVICE_FUNC const AdjointReturnType adjoint() const; EIGEN_DEVICE_FUNC void adjointInPlace(); typedef Diagonal DiagonalReturnType; EIGEN_DEVICE_FUNC DiagonalReturnType diagonal(); typedef typename internal::add_const >::type ConstDiagonalReturnType; EIGEN_DEVICE_FUNC ConstDiagonalReturnType diagonal() const; template struct DiagonalIndexReturnType { typedef Diagonal Type; }; template struct ConstDiagonalIndexReturnType { typedef const Diagonal Type; }; template EIGEN_DEVICE_FUNC typename DiagonalIndexReturnType::Type diagonal(); template EIGEN_DEVICE_FUNC typename ConstDiagonalIndexReturnType::Type diagonal() const; typedef Diagonal DiagonalDynamicIndexReturnType; typedef typename internal::add_const >::type ConstDiagonalDynamicIndexReturnType; EIGEN_DEVICE_FUNC DiagonalDynamicIndexReturnType diagonal(Index index); EIGEN_DEVICE_FUNC ConstDiagonalDynamicIndexReturnType diagonal(Index index) const; template struct TriangularViewReturnType { typedef TriangularView Type; }; template struct ConstTriangularViewReturnType { typedef const TriangularView Type; }; template EIGEN_DEVICE_FUNC typename TriangularViewReturnType::Type triangularView(); template EIGEN_DEVICE_FUNC typename ConstTriangularViewReturnType::Type triangularView() const; template struct SelfAdjointViewReturnType { typedef SelfAdjointView Type; }; template struct ConstSelfAdjointViewReturnType { typedef const SelfAdjointView Type; }; template EIGEN_DEVICE_FUNC typename SelfAdjointViewReturnType::Type selfadjointView(); template EIGEN_DEVICE_FUNC typename ConstSelfAdjointViewReturnType::Type selfadjointView() const; const SparseView sparseView(const Scalar& m_reference = Scalar(0), const typename NumTraits::Real& m_epsilon = NumTraits::dummy_precision()) const; EIGEN_DEVICE_FUNC static const IdentityReturnType Identity(); EIGEN_DEVICE_FUNC static const IdentityReturnType Identity(Index rows, Index cols); EIGEN_DEVICE_FUNC static const BasisReturnType Unit(Index size, Index i); EIGEN_DEVICE_FUNC static const BasisReturnType Unit(Index i); EIGEN_DEVICE_FUNC static const BasisReturnType UnitX(); EIGEN_DEVICE_FUNC static const BasisReturnType UnitY(); EIGEN_DEVICE_FUNC static const BasisReturnType UnitZ(); EIGEN_DEVICE_FUNC static const BasisReturnType UnitW(); EIGEN_DEVICE_FUNC const DiagonalWrapper asDiagonal() const; const PermutationWrapper asPermutation() const; EIGEN_DEVICE_FUNC Derived& setIdentity(); EIGEN_DEVICE_FUNC Derived& setIdentity(Index rows, Index cols); EIGEN_DEVICE_FUNC Derived& setUnit(Index i); EIGEN_DEVICE_FUNC Derived& setUnit(Index newSize, Index i); bool isIdentity(const RealScalar& prec = NumTraits::dummy_precision()) const; bool isDiagonal(const RealScalar& prec = NumTraits::dummy_precision()) const; bool isUpperTriangular(const RealScalar& prec = NumTraits::dummy_precision()) const; bool isLowerTriangular(const RealScalar& prec = NumTraits::dummy_precision()) const; template bool isOrthogonal(const MatrixBase& other, const RealScalar& prec = NumTraits::dummy_precision()) const; bool isUnitary(const RealScalar& prec = NumTraits::dummy_precision()) const; /** \returns true if each coefficients of \c *this and \a other are all exactly equal. * \warning When using floating point scalar values you probably should rather use a * fuzzy comparison such as isApprox() * \sa isApprox(), operator!= */ template EIGEN_DEVICE_FUNC inline bool operator==(const MatrixBase& other) const { return cwiseEqual(other).all(); } /** \returns true if at least one pair of coefficients of \c *this and \a other are not exactly equal to each other. * \warning When using floating point scalar values you probably should rather use a * fuzzy comparison such as isApprox() * \sa isApprox(), operator== */ template EIGEN_DEVICE_FUNC inline bool operator!=(const MatrixBase& other) const { return cwiseNotEqual(other).any(); } NoAlias EIGEN_DEVICE_FUNC noalias(); // TODO forceAlignedAccess is temporarily disabled // Need to find a nicer workaround. inline const Derived& forceAlignedAccess() const { return derived(); } inline Derived& forceAlignedAccess() { return derived(); } template inline const Derived& forceAlignedAccessIf() const { return derived(); } template inline Derived& forceAlignedAccessIf() { return derived(); } EIGEN_DEVICE_FUNC Scalar trace() const; template EIGEN_DEVICE_FUNC RealScalar lpNorm() const; EIGEN_DEVICE_FUNC MatrixBase& matrix() { return *this; } EIGEN_DEVICE_FUNC const MatrixBase& matrix() const { return *this; } /** \returns an \link Eigen::ArrayBase Array \endlink expression of this matrix * \sa ArrayBase::matrix() */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ArrayWrapper array() { return ArrayWrapper(derived()); } /** \returns a const \link Eigen::ArrayBase Array \endlink expression of this matrix * \sa ArrayBase::matrix() */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const ArrayWrapper array() const { return ArrayWrapper(derived()); } /////////// LU module /////////// inline const FullPivLU fullPivLu() const; inline const PartialPivLU partialPivLu() const; inline const PartialPivLU lu() const; EIGEN_DEVICE_FUNC inline const Inverse inverse() const; template inline void computeInverseAndDetWithCheck( ResultType& inverse, typename ResultType::Scalar& determinant, bool& invertible, const RealScalar& absDeterminantThreshold = NumTraits::dummy_precision() ) const; template inline void computeInverseWithCheck( ResultType& inverse, bool& invertible, const RealScalar& absDeterminantThreshold = NumTraits::dummy_precision() ) const; EIGEN_DEVICE_FUNC Scalar determinant() const; /////////// Cholesky module /////////// inline const LLT llt() const; inline const LDLT ldlt() const; /////////// QR module /////////// inline const HouseholderQR householderQr() const; inline const ColPivHouseholderQR colPivHouseholderQr() const; inline const FullPivHouseholderQR fullPivHouseholderQr() const; inline const CompleteOrthogonalDecomposition completeOrthogonalDecomposition() const; /////////// Eigenvalues module /////////// inline EigenvaluesReturnType eigenvalues() const; inline RealScalar operatorNorm() const; /////////// SVD module /////////// inline JacobiSVD jacobiSvd(unsigned int computationOptions = 0) const; inline BDCSVD bdcSvd(unsigned int computationOptions = 0) const; /////////// Geometry module /////////// #ifndef EIGEN_PARSED_BY_DOXYGEN /// \internal helper struct to form the return type of the cross product template struct cross_product_return_type { typedef typename ScalarBinaryOpTraits::Scalar,typename internal::traits::Scalar>::ReturnType Scalar; typedef Matrix type; }; #endif // EIGEN_PARSED_BY_DOXYGEN template EIGEN_DEVICE_FUNC #ifndef EIGEN_PARSED_BY_DOXYGEN inline typename cross_product_return_type::type #else inline PlainObject #endif cross(const MatrixBase& other) const; template EIGEN_DEVICE_FUNC inline PlainObject cross3(const MatrixBase& other) const; EIGEN_DEVICE_FUNC inline PlainObject unitOrthogonal(void) const; EIGEN_DEVICE_FUNC inline Matrix eulerAngles(Index a0, Index a1, Index a2) const; // put this as separate enum value to work around possible GCC 4.3 bug (?) enum { HomogeneousReturnTypeDirection = ColsAtCompileTime==1&&RowsAtCompileTime==1 ? ((internal::traits::Flags&RowMajorBit)==RowMajorBit ? Horizontal : Vertical) : ColsAtCompileTime==1 ? Vertical : Horizontal }; typedef Homogeneous HomogeneousReturnType; EIGEN_DEVICE_FUNC inline HomogeneousReturnType homogeneous() const; enum { SizeMinusOne = SizeAtCompileTime==Dynamic ? Dynamic : SizeAtCompileTime-1 }; typedef Block::ColsAtCompileTime==1 ? SizeMinusOne : 1, internal::traits::ColsAtCompileTime==1 ? 1 : SizeMinusOne> ConstStartMinusOne; typedef EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(ConstStartMinusOne,Scalar,quotient) HNormalizedReturnType; EIGEN_DEVICE_FUNC inline const HNormalizedReturnType hnormalized() const; ////////// Householder module /////////// EIGEN_DEVICE_FUNC void makeHouseholderInPlace(Scalar& tau, RealScalar& beta); template EIGEN_DEVICE_FUNC void makeHouseholder(EssentialPart& essential, Scalar& tau, RealScalar& beta) const; template EIGEN_DEVICE_FUNC void applyHouseholderOnTheLeft(const EssentialPart& essential, const Scalar& tau, Scalar* workspace); template EIGEN_DEVICE_FUNC void applyHouseholderOnTheRight(const EssentialPart& essential, const Scalar& tau, Scalar* workspace); ///////// Jacobi module ///////// template EIGEN_DEVICE_FUNC void applyOnTheLeft(Index p, Index q, const JacobiRotation& j); template EIGEN_DEVICE_FUNC void applyOnTheRight(Index p, Index q, const JacobiRotation& j); ///////// SparseCore module ///////// template EIGEN_STRONG_INLINE const typename SparseMatrixBase::template CwiseProductDenseReturnType::Type cwiseProduct(const SparseMatrixBase &other) const { return other.cwiseProduct(derived()); } ///////// MatrixFunctions module ///////// typedef typename internal::stem_function::type StemFunction; #define EIGEN_MATRIX_FUNCTION(ReturnType, Name, Description) \ /** \returns an expression of the matrix Description of \c *this. \brief This function requires the unsupported MatrixFunctions module. To compute the coefficient-wise Description use ArrayBase::##Name . */ \ const ReturnType Name() const; #define EIGEN_MATRIX_FUNCTION_1(ReturnType, Name, Description, Argument) \ /** \returns an expression of the matrix Description of \c *this. \brief This function requires the unsupported MatrixFunctions module. To compute the coefficient-wise Description use ArrayBase::##Name . */ \ const ReturnType Name(Argument) const; EIGEN_MATRIX_FUNCTION(MatrixExponentialReturnValue, exp, exponential) /** \brief Helper function for the unsupported MatrixFunctions module.*/ const MatrixFunctionReturnValue matrixFunction(StemFunction f) const; EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, cosh, hyperbolic cosine) EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, sinh, hyperbolic sine) #if EIGEN_HAS_CXX11_MATH EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, atanh, inverse hyperbolic cosine) EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, acosh, inverse hyperbolic cosine) EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, asinh, inverse hyperbolic sine) #endif EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, cos, cosine) EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, sin, sine) EIGEN_MATRIX_FUNCTION(MatrixSquareRootReturnValue, sqrt, square root) EIGEN_MATRIX_FUNCTION(MatrixLogarithmReturnValue, log, logarithm) EIGEN_MATRIX_FUNCTION_1(MatrixPowerReturnValue, pow, power to \c p, const RealScalar& p) EIGEN_MATRIX_FUNCTION_1(MatrixComplexPowerReturnValue, pow, power to \c p, const std::complex& p) protected: EIGEN_DEFAULT_COPY_CONSTRUCTOR(MatrixBase) EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(MatrixBase) private: EIGEN_DEVICE_FUNC explicit MatrixBase(int); EIGEN_DEVICE_FUNC MatrixBase(int,int); template EIGEN_DEVICE_FUNC explicit MatrixBase(const MatrixBase&); protected: // mixing arrays and matrices is not legal template Derived& operator+=(const ArrayBase& ) {EIGEN_STATIC_ASSERT(std::ptrdiff_t(sizeof(typename OtherDerived::Scalar))==-1,YOU_CANNOT_MIX_ARRAYS_AND_MATRICES); return *this;} // mixing arrays and matrices is not legal template Derived& operator-=(const ArrayBase& ) {EIGEN_STATIC_ASSERT(std::ptrdiff_t(sizeof(typename OtherDerived::Scalar))==-1,YOU_CANNOT_MIX_ARRAYS_AND_MATRICES); return *this;} }; /*************************************************************************** * Implementation of matrix base methods ***************************************************************************/ /** replaces \c *this by \c *this * \a other. * * \returns a reference to \c *this * * Example: \include MatrixBase_applyOnTheRight.cpp * Output: \verbinclude MatrixBase_applyOnTheRight.out */ template template inline Derived& MatrixBase::operator*=(const EigenBase &other) { other.derived().applyThisOnTheRight(derived()); return derived(); } /** replaces \c *this by \c *this * \a other. It is equivalent to MatrixBase::operator*=(). * * Example: \include MatrixBase_applyOnTheRight.cpp * Output: \verbinclude MatrixBase_applyOnTheRight.out */ template template inline void MatrixBase::applyOnTheRight(const EigenBase &other) { other.derived().applyThisOnTheRight(derived()); } /** replaces \c *this by \a other * \c *this. * * Example: \include MatrixBase_applyOnTheLeft.cpp * Output: \verbinclude MatrixBase_applyOnTheLeft.out */ template template inline void MatrixBase::applyOnTheLeft(const EigenBase &other) { other.derived().applyThisOnTheLeft(derived()); } } // end namespace Eigen #endif // EIGEN_MATRIXBASE_H RcppEigen/inst/include/Eigen/src/Core/CwiseUnaryView.h0000644000176200001440000001265714567757725022375 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2009-2010 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_CWISE_UNARY_VIEW_H #define EIGEN_CWISE_UNARY_VIEW_H namespace Eigen { namespace internal { template struct traits > : traits { typedef typename result_of< ViewOp(const typename traits::Scalar&) >::type Scalar; typedef typename MatrixType::Nested MatrixTypeNested; typedef typename remove_all::type _MatrixTypeNested; enum { FlagsLvalueBit = is_lvalue::value ? LvalueBit : 0, Flags = traits<_MatrixTypeNested>::Flags & (RowMajorBit | FlagsLvalueBit | DirectAccessBit), // FIXME DirectAccessBit should not be handled by expressions MatrixTypeInnerStride = inner_stride_at_compile_time::ret, // need to cast the sizeof's from size_t to int explicitly, otherwise: // "error: no integral type can represent all of the enumerator values InnerStrideAtCompileTime = MatrixTypeInnerStride == Dynamic ? int(Dynamic) : int(MatrixTypeInnerStride) * int(sizeof(typename traits::Scalar) / sizeof(Scalar)), OuterStrideAtCompileTime = outer_stride_at_compile_time::ret == Dynamic ? int(Dynamic) : outer_stride_at_compile_time::ret * int(sizeof(typename traits::Scalar) / sizeof(Scalar)) }; }; } template class CwiseUnaryViewImpl; /** \class CwiseUnaryView * \ingroup Core_Module * * \brief Generic lvalue expression of a coefficient-wise unary operator of a matrix or a vector * * \tparam ViewOp template functor implementing the view * \tparam MatrixType the type of the matrix we are applying the unary operator * * This class represents a lvalue expression of a generic unary view operator of a matrix or a vector. * It is the return type of real() and imag(), and most of the time this is the only way it is used. * * \sa MatrixBase::unaryViewExpr(const CustomUnaryOp &) const, class CwiseUnaryOp */ template class CwiseUnaryView : public CwiseUnaryViewImpl::StorageKind> { public: typedef typename CwiseUnaryViewImpl::StorageKind>::Base Base; EIGEN_GENERIC_PUBLIC_INTERFACE(CwiseUnaryView) typedef typename internal::ref_selector::non_const_type MatrixTypeNested; typedef typename internal::remove_all::type NestedExpression; explicit EIGEN_DEVICE_FUNC inline CwiseUnaryView(MatrixType& mat, const ViewOp& func = ViewOp()) : m_matrix(mat), m_functor(func) {} EIGEN_INHERIT_ASSIGNMENT_OPERATORS(CwiseUnaryView) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_matrix.rows(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_matrix.cols(); } /** \returns the functor representing unary operation */ EIGEN_DEVICE_FUNC const ViewOp& functor() const { return m_functor; } /** \returns the nested expression */ EIGEN_DEVICE_FUNC const typename internal::remove_all::type& nestedExpression() const { return m_matrix; } /** \returns the nested expression */ EIGEN_DEVICE_FUNC typename internal::remove_reference::type& nestedExpression() { return m_matrix; } protected: MatrixTypeNested m_matrix; ViewOp m_functor; }; // Generic API dispatcher template class CwiseUnaryViewImpl : public internal::generic_xpr_base >::type { public: typedef typename internal::generic_xpr_base >::type Base; }; template class CwiseUnaryViewImpl : public internal::dense_xpr_base< CwiseUnaryView >::type { public: typedef CwiseUnaryView Derived; typedef typename internal::dense_xpr_base< CwiseUnaryView >::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(Derived) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(CwiseUnaryViewImpl) EIGEN_DEVICE_FUNC inline Scalar* data() { return &(this->coeffRef(0)); } EIGEN_DEVICE_FUNC inline const Scalar* data() const { return &(this->coeff(0)); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const { return derived().nestedExpression().innerStride() * sizeof(typename internal::traits::Scalar) / sizeof(Scalar); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const { return derived().nestedExpression().outerStride() * sizeof(typename internal::traits::Scalar) / sizeof(Scalar); } protected: EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(CwiseUnaryViewImpl) }; } // end namespace Eigen #endif // EIGEN_CWISE_UNARY_VIEW_H RcppEigen/inst/include/Eigen/src/Core/functors/0000755000176200001440000000000014567757725021130 5ustar liggesusersRcppEigen/inst/include/Eigen/src/Core/functors/NullaryFunctors.h0000644000176200001440000002021614567757725024454 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2016 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_NULLARY_FUNCTORS_H #define EIGEN_NULLARY_FUNCTORS_H namespace Eigen { namespace internal { template struct scalar_constant_op { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE scalar_constant_op(const scalar_constant_op& other) : m_other(other.m_other) { } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE scalar_constant_op(const Scalar& other) : m_other(other) { } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() () const { return m_other; } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const PacketType packetOp() const { return internal::pset1(m_other); } const Scalar m_other; }; template struct functor_traits > { enum { Cost = 0 /* as the constant value should be loaded in register only once for the whole expression */, PacketAccess = packet_traits::Vectorizable, IsRepeatable = true }; }; template struct scalar_identity_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_identity_op) template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (IndexType row, IndexType col) const { return row==col ? Scalar(1) : Scalar(0); } }; template struct functor_traits > { enum { Cost = NumTraits::AddCost, PacketAccess = false, IsRepeatable = true }; }; template struct linspaced_op_impl; template struct linspaced_op_impl { typedef typename NumTraits::Real RealScalar; EIGEN_DEVICE_FUNC linspaced_op_impl(const Scalar& low, const Scalar& high, Index num_steps) : m_low(low), m_high(high), m_size1(num_steps==1 ? 1 : num_steps-1), m_step(num_steps==1 ? Scalar() : Scalar((high-low)/RealScalar(num_steps-1))), m_flip(numext::abs(high) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (IndexType i) const { if(m_flip) return (i==0)? m_low : Scalar(m_high - RealScalar(m_size1-i)*m_step); else return (i==m_size1)? m_high : Scalar(m_low + RealScalar(i)*m_step); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(IndexType i) const { // Principle: // [low, ..., low] + ( [step, ..., step] * ( [i, ..., i] + [0, ..., size] ) ) if(m_flip) { Packet pi = plset(Scalar(i-m_size1)); Packet res = padd(pset1(m_high), pmul(pset1(m_step), pi)); if (EIGEN_PREDICT_TRUE(i != 0)) return res; Packet mask = pcmp_lt(pset1(0), plset(0)); return pselect(mask, res, pset1(m_low)); } else { Packet pi = plset(Scalar(i)); Packet res = padd(pset1(m_low), pmul(pset1(m_step), pi)); if(EIGEN_PREDICT_TRUE(i != m_size1-unpacket_traits::size+1)) return res; Packet mask = pcmp_lt(plset(0), pset1(unpacket_traits::size-1)); return pselect(mask, res, pset1(m_high)); } } const Scalar m_low; const Scalar m_high; const Index m_size1; const Scalar m_step; const bool m_flip; }; template struct linspaced_op_impl { EIGEN_DEVICE_FUNC linspaced_op_impl(const Scalar& low, const Scalar& high, Index num_steps) : m_low(low), m_multiplier((high-low)/convert_index(num_steps<=1 ? 1 : num_steps-1)), m_divisor(convert_index((high>=low?num_steps:-num_steps)+(high-low))/((numext::abs(high-low)+1)==0?1:(numext::abs(high-low)+1))), m_use_divisor(num_steps>1 && (numext::abs(high-low)+1) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (IndexType i) const { if(m_use_divisor) return m_low + convert_index(i)/m_divisor; else return m_low + convert_index(i)*m_multiplier; } const Scalar m_low; const Scalar m_multiplier; const Scalar m_divisor; const bool m_use_divisor; }; // ----- Linspace functor ---------------------------------------------------------------- // Forward declaration (we default to random access which does not really give // us a speed gain when using packet access but it allows to use the functor in // nested expressions). template struct linspaced_op; template struct functor_traits< linspaced_op > { enum { Cost = 1, PacketAccess = (!NumTraits::IsInteger) && packet_traits::HasSetLinear && packet_traits::HasBlend, /*&& ((!NumTraits::IsInteger) || packet_traits::HasDiv),*/ // <- vectorization for integer is currently disabled IsRepeatable = true }; }; template struct linspaced_op { EIGEN_DEVICE_FUNC linspaced_op(const Scalar& low, const Scalar& high, Index num_steps) : impl((num_steps==1 ? high : low),high,num_steps) {} template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (IndexType i) const { return impl(i); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(IndexType i) const { return impl.template packetOp(i); } // This proxy object handles the actual required temporaries and the different // implementations (integer vs. floating point). const linspaced_op_impl::IsInteger> impl; }; // Linear access is automatically determined from the operator() prototypes available for the given functor. // If it exposes an operator()(i,j), then we assume the i and j coefficients are required independently // and linear access is not possible. In all other cases, linear access is enabled. // Users should not have to deal with this structure. template struct functor_has_linear_access { enum { ret = !has_binary_operator::value }; }; // For unreliable compilers, let's specialize the has_*ary_operator // helpers so that at least built-in nullary functors work fine. #if !( (EIGEN_COMP_MSVC>1600) || (EIGEN_GNUC_AT_LEAST(4,8)) || (EIGEN_COMP_ICC>=1600)) template struct has_nullary_operator,IndexType> { enum { value = 1}; }; template struct has_unary_operator,IndexType> { enum { value = 0}; }; template struct has_binary_operator,IndexType> { enum { value = 0}; }; template struct has_nullary_operator,IndexType> { enum { value = 0}; }; template struct has_unary_operator,IndexType> { enum { value = 0}; }; template struct has_binary_operator,IndexType> { enum { value = 1}; }; template struct has_nullary_operator,IndexType> { enum { value = 0}; }; template struct has_unary_operator,IndexType> { enum { value = 1}; }; template struct has_binary_operator,IndexType> { enum { value = 0}; }; template struct has_nullary_operator,IndexType> { enum { value = 1}; }; template struct has_unary_operator,IndexType> { enum { value = 0}; }; template struct has_binary_operator,IndexType> { enum { value = 0}; }; #endif } // end namespace internal } // end namespace Eigen #endif // EIGEN_NULLARY_FUNCTORS_H RcppEigen/inst/include/Eigen/src/Core/functors/TernaryFunctors.h0000644000176200001440000000113714107270226024423 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2016 Eugene Brevdo // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_TERNARY_FUNCTORS_H #define EIGEN_TERNARY_FUNCTORS_H namespace Eigen { namespace internal { //---------- associative ternary functors ---------- } // end namespace internal } // end namespace Eigen #endif // EIGEN_TERNARY_FUNCTORS_H RcppEigen/inst/include/Eigen/src/Core/functors/UnaryFunctors.h0000644000176200001440000011632214567757725024130 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2016 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_UNARY_FUNCTORS_H #define EIGEN_UNARY_FUNCTORS_H namespace Eigen { namespace internal { /** \internal * \brief Template functor to compute the opposite of a scalar * * \sa class CwiseUnaryOp, MatrixBase::operator- */ template struct scalar_opposite_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_opposite_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return -a; } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const { return internal::pnegate(a); } }; template struct functor_traits > { enum { Cost = NumTraits::AddCost, PacketAccess = packet_traits::HasNegate }; }; /** \internal * \brief Template functor to compute the absolute value of a scalar * * \sa class CwiseUnaryOp, Cwise::abs */ template struct scalar_abs_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_abs_op) typedef typename NumTraits::Real result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const Scalar& a) const { return numext::abs(a); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const { return internal::pabs(a); } }; template struct functor_traits > { enum { Cost = NumTraits::AddCost, PacketAccess = packet_traits::HasAbs }; }; /** \internal * \brief Template functor to compute the score of a scalar, to chose a pivot * * \sa class CwiseUnaryOp */ template struct scalar_score_coeff_op : scalar_abs_op { typedef void Score_is_abs; }; template struct functor_traits > : functor_traits > {}; /* Avoid recomputing abs when we know the score and they are the same. Not a true Eigen functor. */ template struct abs_knowing_score { EIGEN_EMPTY_STRUCT_CTOR(abs_knowing_score) typedef typename NumTraits::Real result_type; template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const Scalar& a, const Score&) const { return numext::abs(a); } }; template struct abs_knowing_score::Score_is_abs> { EIGEN_EMPTY_STRUCT_CTOR(abs_knowing_score) typedef typename NumTraits::Real result_type; template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const Scal&, const result_type& a) const { return a; } }; /** \internal * \brief Template functor to compute the squared absolute value of a scalar * * \sa class CwiseUnaryOp, Cwise::abs2 */ template struct scalar_abs2_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_abs2_op) typedef typename NumTraits::Real result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const Scalar& a) const { return numext::abs2(a); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const { return internal::pmul(a,a); } }; template struct functor_traits > { enum { Cost = NumTraits::MulCost, PacketAccess = packet_traits::HasAbs2 }; }; /** \internal * \brief Template functor to compute the conjugate of a complex value * * \sa class CwiseUnaryOp, MatrixBase::conjugate() */ template struct scalar_conjugate_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_conjugate_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return numext::conj(a); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const { return internal::pconj(a); } }; template struct functor_traits > { enum { Cost = 0, // Yes the cost is zero even for complexes because in most cases for which // the cost is used, conjugation turns to be a no-op. Some examples: // cost(a*conj(b)) == cost(a*b) // cost(a+conj(b)) == cost(a+b) // ::HasConj }; }; /** \internal * \brief Template functor to compute the phase angle of a complex * * \sa class CwiseUnaryOp, Cwise::arg */ template struct scalar_arg_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_arg_op) typedef typename NumTraits::Real result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const Scalar& a) const { return numext::arg(a); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const { return internal::parg(a); } }; template struct functor_traits > { enum { Cost = NumTraits::IsComplex ? 5 * NumTraits::MulCost : NumTraits::AddCost, PacketAccess = packet_traits::HasArg }; }; /** \internal * \brief Template functor to cast a scalar to another type * * \sa class CwiseUnaryOp, MatrixBase::cast() */ template struct scalar_cast_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) typedef NewType result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const NewType operator() (const Scalar& a) const { return cast(a); } }; template struct functor_traits > { enum { Cost = is_same::value ? 0 : NumTraits::AddCost, PacketAccess = false }; }; /** \internal * \brief Template functor to arithmetically shift a scalar right by a number of bits * * \sa class CwiseUnaryOp, MatrixBase::shift_right() */ template struct scalar_shift_right_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_shift_right_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return a >> N; } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const { return internal::parithmetic_shift_right(a); } }; template struct functor_traits > { enum { Cost = NumTraits::AddCost, PacketAccess = packet_traits::HasShift }; }; /** \internal * \brief Template functor to logically shift a scalar left by a number of bits * * \sa class CwiseUnaryOp, MatrixBase::shift_left() */ template struct scalar_shift_left_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_shift_left_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return a << N; } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const { return internal::plogical_shift_left(a); } }; template struct functor_traits > { enum { Cost = NumTraits::AddCost, PacketAccess = packet_traits::HasShift }; }; /** \internal * \brief Template functor to extract the real part of a complex * * \sa class CwiseUnaryOp, MatrixBase::real() */ template struct scalar_real_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_real_op) typedef typename NumTraits::Real result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { return numext::real(a); } }; template struct functor_traits > { enum { Cost = 0, PacketAccess = false }; }; /** \internal * \brief Template functor to extract the imaginary part of a complex * * \sa class CwiseUnaryOp, MatrixBase::imag() */ template struct scalar_imag_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_imag_op) typedef typename NumTraits::Real result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { return numext::imag(a); } }; template struct functor_traits > { enum { Cost = 0, PacketAccess = false }; }; /** \internal * \brief Template functor to extract the real part of a complex as a reference * * \sa class CwiseUnaryOp, MatrixBase::real() */ template struct scalar_real_ref_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_real_ref_op) typedef typename NumTraits::Real result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type& operator() (const Scalar& a) const { return numext::real_ref(*const_cast(&a)); } }; template struct functor_traits > { enum { Cost = 0, PacketAccess = false }; }; /** \internal * \brief Template functor to extract the imaginary part of a complex as a reference * * \sa class CwiseUnaryOp, MatrixBase::imag() */ template struct scalar_imag_ref_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_imag_ref_op) typedef typename NumTraits::Real result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type& operator() (const Scalar& a) const { return numext::imag_ref(*const_cast(&a)); } }; template struct functor_traits > { enum { Cost = 0, PacketAccess = false }; }; /** \internal * * \brief Template functor to compute the exponential of a scalar * * \sa class CwiseUnaryOp, Cwise::exp() */ template struct scalar_exp_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_exp_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::exp(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pexp(a); } }; template struct functor_traits > { enum { PacketAccess = packet_traits::HasExp, // The following numbers are based on the AVX implementation. #ifdef EIGEN_VECTORIZE_FMA // Haswell can issue 2 add/mul/madd per cycle. Cost = (sizeof(Scalar) == 4 // float: 8 pmadd, 4 pmul, 2 padd/psub, 6 other ? (8 * NumTraits::AddCost + 6 * NumTraits::MulCost) // double: 7 pmadd, 5 pmul, 3 padd/psub, 1 div, 13 other : (14 * NumTraits::AddCost + 6 * NumTraits::MulCost + scalar_div_cost::HasDiv>::value)) #else Cost = (sizeof(Scalar) == 4 // float: 7 pmadd, 6 pmul, 4 padd/psub, 10 other ? (21 * NumTraits::AddCost + 13 * NumTraits::MulCost) // double: 7 pmadd, 5 pmul, 3 padd/psub, 1 div, 13 other : (23 * NumTraits::AddCost + 12 * NumTraits::MulCost + scalar_div_cost::HasDiv>::value)) #endif }; }; /** \internal * * \brief Template functor to compute the exponential of a scalar - 1. * * \sa class CwiseUnaryOp, ArrayBase::expm1() */ template struct scalar_expm1_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_expm1_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::expm1(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pexpm1(a); } }; template struct functor_traits > { enum { PacketAccess = packet_traits::HasExpm1, Cost = functor_traits >::Cost // TODO measure cost of expm1 }; }; /** \internal * * \brief Template functor to compute the logarithm of a scalar * * \sa class CwiseUnaryOp, ArrayBase::log() */ template struct scalar_log_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_log_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::log(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::plog(a); } }; template struct functor_traits > { enum { PacketAccess = packet_traits::HasLog, Cost = (PacketAccess // The following numbers are based on the AVX implementation. #ifdef EIGEN_VECTORIZE_FMA // 8 pmadd, 6 pmul, 8 padd/psub, 16 other, can issue 2 add/mul/madd per cycle. ? (20 * NumTraits::AddCost + 7 * NumTraits::MulCost) #else // 8 pmadd, 6 pmul, 8 padd/psub, 20 other ? (36 * NumTraits::AddCost + 14 * NumTraits::MulCost) #endif // Measured cost of std::log. : sizeof(Scalar)==4 ? 40 : 85) }; }; /** \internal * * \brief Template functor to compute the logarithm of 1 plus a scalar value * * \sa class CwiseUnaryOp, ArrayBase::log1p() */ template struct scalar_log1p_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_log1p_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::log1p(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::plog1p(a); } }; template struct functor_traits > { enum { PacketAccess = packet_traits::HasLog1p, Cost = functor_traits >::Cost // TODO measure cost of log1p }; }; /** \internal * * \brief Template functor to compute the base-10 logarithm of a scalar * * \sa class CwiseUnaryOp, Cwise::log10() */ template struct scalar_log10_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_log10_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { EIGEN_USING_STD(log10) return log10(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::plog10(a); } }; template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = packet_traits::HasLog10 }; }; /** \internal * * \brief Template functor to compute the base-2 logarithm of a scalar * * \sa class CwiseUnaryOp, Cwise::log2() */ template struct scalar_log2_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_log2_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return Scalar(EIGEN_LOG2E) * numext::log(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::plog2(a); } }; template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = packet_traits::HasLog }; }; /** \internal * \brief Template functor to compute the square root of a scalar * \sa class CwiseUnaryOp, Cwise::sqrt() */ template struct scalar_sqrt_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_sqrt_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::sqrt(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psqrt(a); } }; template struct functor_traits > { enum { #if EIGEN_FAST_MATH // The following numbers are based on the AVX implementation. Cost = (sizeof(Scalar) == 8 ? 28 // 4 pmul, 1 pmadd, 3 other : (3 * NumTraits::AddCost + 5 * NumTraits::MulCost)), #else // The following numbers are based on min VSQRT throughput on Haswell. Cost = (sizeof(Scalar) == 8 ? 28 : 14), #endif PacketAccess = packet_traits::HasSqrt }; }; // Boolean specialization to eliminate -Wimplicit-conversion-floating-point-to-bool warnings. template<> struct scalar_sqrt_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_sqrt_op) EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline bool operator() (const bool& a) const { return a; } template EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return a; } }; template <> struct functor_traits > { enum { Cost = 1, PacketAccess = packet_traits::Vectorizable }; }; /** \internal * \brief Template functor to compute the reciprocal square root of a scalar * \sa class CwiseUnaryOp, Cwise::rsqrt() */ template struct scalar_rsqrt_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_rsqrt_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::rsqrt(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::prsqrt(a); } }; template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = packet_traits::HasRsqrt }; }; /** \internal * \brief Template functor to compute the cosine of a scalar * \sa class CwiseUnaryOp, ArrayBase::cos() */ template struct scalar_cos_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_cos_op) EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return numext::cos(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pcos(a); } }; template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = packet_traits::HasCos }; }; /** \internal * \brief Template functor to compute the sine of a scalar * \sa class CwiseUnaryOp, ArrayBase::sin() */ template struct scalar_sin_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_sin_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::sin(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psin(a); } }; template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = packet_traits::HasSin }; }; /** \internal * \brief Template functor to compute the tan of a scalar * \sa class CwiseUnaryOp, ArrayBase::tan() */ template struct scalar_tan_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_tan_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::tan(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::ptan(a); } }; template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = packet_traits::HasTan }; }; /** \internal * \brief Template functor to compute the arc cosine of a scalar * \sa class CwiseUnaryOp, ArrayBase::acos() */ template struct scalar_acos_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_acos_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::acos(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pacos(a); } }; template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = packet_traits::HasACos }; }; /** \internal * \brief Template functor to compute the arc sine of a scalar * \sa class CwiseUnaryOp, ArrayBase::asin() */ template struct scalar_asin_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_asin_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::asin(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pasin(a); } }; template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = packet_traits::HasASin }; }; /** \internal * \brief Template functor to compute the atan of a scalar * \sa class CwiseUnaryOp, ArrayBase::atan() */ template struct scalar_atan_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_atan_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::atan(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::patan(a); } }; template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = packet_traits::HasATan }; }; /** \internal * \brief Template functor to compute the tanh of a scalar * \sa class CwiseUnaryOp, ArrayBase::tanh() */ template struct scalar_tanh_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_tanh_op) EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::tanh(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& x) const { return ptanh(x); } }; template struct functor_traits > { enum { PacketAccess = packet_traits::HasTanh, Cost = ( (EIGEN_FAST_MATH && is_same::value) // The following numbers are based on the AVX implementation, #ifdef EIGEN_VECTORIZE_FMA // Haswell can issue 2 add/mul/madd per cycle. // 9 pmadd, 2 pmul, 1 div, 2 other ? (2 * NumTraits::AddCost + 6 * NumTraits::MulCost + scalar_div_cost::HasDiv>::value) #else ? (11 * NumTraits::AddCost + 11 * NumTraits::MulCost + scalar_div_cost::HasDiv>::value) #endif // This number assumes a naive implementation of tanh : (6 * NumTraits::AddCost + 3 * NumTraits::MulCost + 2 * scalar_div_cost::HasDiv>::value + functor_traits >::Cost)) }; }; #if EIGEN_HAS_CXX11_MATH /** \internal * \brief Template functor to compute the atanh of a scalar * \sa class CwiseUnaryOp, ArrayBase::atanh() */ template struct scalar_atanh_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_atanh_op) EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::atanh(a); } }; template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = false }; }; #endif /** \internal * \brief Template functor to compute the sinh of a scalar * \sa class CwiseUnaryOp, ArrayBase::sinh() */ template struct scalar_sinh_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_sinh_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::sinh(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psinh(a); } }; template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = packet_traits::HasSinh }; }; #if EIGEN_HAS_CXX11_MATH /** \internal * \brief Template functor to compute the asinh of a scalar * \sa class CwiseUnaryOp, ArrayBase::asinh() */ template struct scalar_asinh_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_asinh_op) EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::asinh(a); } }; template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = false }; }; #endif /** \internal * \brief Template functor to compute the cosh of a scalar * \sa class CwiseUnaryOp, ArrayBase::cosh() */ template struct scalar_cosh_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_cosh_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::cosh(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pcosh(a); } }; template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = packet_traits::HasCosh }; }; #if EIGEN_HAS_CXX11_MATH /** \internal * \brief Template functor to compute the acosh of a scalar * \sa class CwiseUnaryOp, ArrayBase::acosh() */ template struct scalar_acosh_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_acosh_op) EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::acosh(a); } }; template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = false }; }; #endif /** \internal * \brief Template functor to compute the inverse of a scalar * \sa class CwiseUnaryOp, Cwise::inverse() */ template struct scalar_inverse_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_inverse_op) EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return Scalar(1)/a; } template EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const { return internal::pdiv(pset1(Scalar(1)),a); } }; template struct functor_traits > { enum { PacketAccess = packet_traits::HasDiv, Cost = scalar_div_cost::value }; }; /** \internal * \brief Template functor to compute the square of a scalar * \sa class CwiseUnaryOp, Cwise::square() */ template struct scalar_square_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_square_op) EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return a*a; } template EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const { return internal::pmul(a,a); } }; template struct functor_traits > { enum { Cost = NumTraits::MulCost, PacketAccess = packet_traits::HasMul }; }; // Boolean specialization to avoid -Wint-in-bool-context warnings on GCC. template<> struct scalar_square_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_square_op) EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline bool operator() (const bool& a) const { return a; } template EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const { return a; } }; template<> struct functor_traits > { enum { Cost = 0, PacketAccess = packet_traits::Vectorizable }; }; /** \internal * \brief Template functor to compute the cube of a scalar * \sa class CwiseUnaryOp, Cwise::cube() */ template struct scalar_cube_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_cube_op) EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return a*a*a; } template EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const { return internal::pmul(a,pmul(a,a)); } }; template struct functor_traits > { enum { Cost = 2*NumTraits::MulCost, PacketAccess = packet_traits::HasMul }; }; // Boolean specialization to avoid -Wint-in-bool-context warnings on GCC. template<> struct scalar_cube_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_cube_op) EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline bool operator() (const bool& a) const { return a; } template EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const { return a; } }; template<> struct functor_traits > { enum { Cost = 0, PacketAccess = packet_traits::Vectorizable }; }; /** \internal * \brief Template functor to compute the rounded value of a scalar * \sa class CwiseUnaryOp, ArrayBase::round() */ template struct scalar_round_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_round_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return numext::round(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pround(a); } }; template struct functor_traits > { enum { Cost = NumTraits::MulCost, PacketAccess = packet_traits::HasRound }; }; /** \internal * \brief Template functor to compute the floor of a scalar * \sa class CwiseUnaryOp, ArrayBase::floor() */ template struct scalar_floor_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_floor_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return numext::floor(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pfloor(a); } }; template struct functor_traits > { enum { Cost = NumTraits::MulCost, PacketAccess = packet_traits::HasFloor }; }; /** \internal * \brief Template functor to compute the rounded (with current rounding mode) value of a scalar * \sa class CwiseUnaryOp, ArrayBase::rint() */ template struct scalar_rint_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_rint_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return numext::rint(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::print(a); } }; template struct functor_traits > { enum { Cost = NumTraits::MulCost, PacketAccess = packet_traits::HasRint }; }; /** \internal * \brief Template functor to compute the ceil of a scalar * \sa class CwiseUnaryOp, ArrayBase::ceil() */ template struct scalar_ceil_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_ceil_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return numext::ceil(a); } template EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pceil(a); } }; template struct functor_traits > { enum { Cost = NumTraits::MulCost, PacketAccess = packet_traits::HasCeil }; }; /** \internal * \brief Template functor to compute whether a scalar is NaN * \sa class CwiseUnaryOp, ArrayBase::isnan() */ template struct scalar_isnan_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_isnan_op) typedef bool result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { #if defined(SYCL_DEVICE_ONLY) return numext::isnan(a); #else return (numext::isnan)(a); #endif } }; template struct functor_traits > { enum { Cost = NumTraits::MulCost, PacketAccess = false }; }; /** \internal * \brief Template functor to check whether a scalar is +/-inf * \sa class CwiseUnaryOp, ArrayBase::isinf() */ template struct scalar_isinf_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_isinf_op) typedef bool result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { #if defined(SYCL_DEVICE_ONLY) return numext::isinf(a); #else return (numext::isinf)(a); #endif } }; template struct functor_traits > { enum { Cost = NumTraits::MulCost, PacketAccess = false }; }; /** \internal * \brief Template functor to check whether a scalar has a finite value * \sa class CwiseUnaryOp, ArrayBase::isfinite() */ template struct scalar_isfinite_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_isfinite_op) typedef bool result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { #if defined(SYCL_DEVICE_ONLY) return numext::isfinite(a); #else return (numext::isfinite)(a); #endif } }; template struct functor_traits > { enum { Cost = NumTraits::MulCost, PacketAccess = false }; }; /** \internal * \brief Template functor to compute the logical not of a boolean * * \sa class CwiseUnaryOp, ArrayBase::operator! */ template struct scalar_boolean_not_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_boolean_not_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator() (const bool& a) const { return !a; } }; template struct functor_traits > { enum { Cost = NumTraits::AddCost, PacketAccess = false }; }; /** \internal * \brief Template functor to compute the signum of a scalar * \sa class CwiseUnaryOp, Cwise::sign() */ template::IsComplex!=0), bool is_integer=(NumTraits::IsInteger!=0) > struct scalar_sign_op; template struct scalar_sign_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_sign_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return Scalar( (a>Scalar(0)) - (a //EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psign(a); } }; template struct scalar_sign_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_sign_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return (numext::isnan)(a) ? a : Scalar( (a>Scalar(0)) - (a //EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psign(a); } }; template struct scalar_sign_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_sign_op) EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { typedef typename NumTraits::Real real_type; real_type aa = numext::abs(a); if (aa==real_type(0)) return Scalar(0); aa = real_type(1)/aa; return Scalar(a.real()*aa, a.imag()*aa ); } //TODO //template //EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psign(a); } }; template struct functor_traits > { enum { Cost = NumTraits::IsComplex ? ( 8*NumTraits::MulCost ) // roughly : ( 3*NumTraits::AddCost), PacketAccess = packet_traits::HasSign }; }; /** \internal * \brief Template functor to compute the logistic function of a scalar * \sa class CwiseUnaryOp, ArrayBase::logistic() */ template struct scalar_logistic_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_logistic_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T operator()(const T& x) const { return packetOp(x); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& x) const { const Packet one = pset1(T(1)); return pdiv(one, padd(one, pexp(pnegate(x)))); } }; #ifndef EIGEN_GPU_COMPILE_PHASE /** \internal * \brief Template specialization of the logistic function for float. * * Uses just a 9/10-degree rational interpolant which * interpolates 1/(1+exp(-x)) - 0.5 up to a couple of ulps in the range * [-9, 18]. Below -9 we use the more accurate approximation * 1/(1+exp(-x)) ~= exp(x), and above 18 the logistic function is 1 withing * one ulp. The shifted logistic is interpolated because it was easier to * make the fit converge. * */ template <> struct scalar_logistic_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_logistic_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float operator()(const float& x) const { return packetOp(x); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& _x) const { const Packet cutoff_lower = pset1(-9.f); const Packet lt_mask = pcmp_lt(_x, cutoff_lower); const bool any_small = predux_any(lt_mask); // The upper cut-off is the smallest x for which the rational approximation evaluates to 1. // Choosing this value saves us a few instructions clamping the results at the end. #ifdef EIGEN_VECTORIZE_FMA const Packet cutoff_upper = pset1(15.7243833541870117f); #else const Packet cutoff_upper = pset1(15.6437711715698242f); #endif const Packet x = pmin(_x, cutoff_upper); // The monomial coefficients of the numerator polynomial (odd). const Packet alpha_1 = pset1(2.48287947061529e-01f); const Packet alpha_3 = pset1(8.51377133304701e-03f); const Packet alpha_5 = pset1(6.08574864600143e-05f); const Packet alpha_7 = pset1(1.15627324459942e-07f); const Packet alpha_9 = pset1(4.37031012579801e-11f); // The monomial coefficients of the denominator polynomial (even). const Packet beta_0 = pset1(9.93151921023180e-01f); const Packet beta_2 = pset1(1.16817656904453e-01f); const Packet beta_4 = pset1(1.70198817374094e-03f); const Packet beta_6 = pset1(6.29106785017040e-06f); const Packet beta_8 = pset1(5.76102136993427e-09f); const Packet beta_10 = pset1(6.10247389755681e-13f); // Since the polynomials are odd/even, we need x^2. const Packet x2 = pmul(x, x); // Evaluate the numerator polynomial p. Packet p = pmadd(x2, alpha_9, alpha_7); p = pmadd(x2, p, alpha_5); p = pmadd(x2, p, alpha_3); p = pmadd(x2, p, alpha_1); p = pmul(x, p); // Evaluate the denominator polynomial q. Packet q = pmadd(x2, beta_10, beta_8); q = pmadd(x2, q, beta_6); q = pmadd(x2, q, beta_4); q = pmadd(x2, q, beta_2); q = pmadd(x2, q, beta_0); // Divide the numerator by the denominator and shift it up. const Packet logistic = padd(pdiv(p, q), pset1(0.5f)); if (EIGEN_PREDICT_FALSE(any_small)) { const Packet exponential = pexp(_x); return pselect(lt_mask, exponential, logistic); } else { return logistic; } } }; #endif // #ifndef EIGEN_GPU_COMPILE_PHASE template struct functor_traits > { enum { // The cost estimate for float here here is for the common(?) case where // all arguments are greater than -9. Cost = scalar_div_cost::HasDiv>::value + (internal::is_same::value ? NumTraits::AddCost * 15 + NumTraits::MulCost * 11 : NumTraits::AddCost * 2 + functor_traits >::Cost), PacketAccess = packet_traits::HasAdd && packet_traits::HasDiv && (internal::is_same::value ? packet_traits::HasMul && packet_traits::HasMax && packet_traits::HasMin : packet_traits::HasNegate && packet_traits::HasExp) }; }; } // end namespace internal } // end namespace Eigen #endif // EIGEN_FUNCTORS_H RcppEigen/inst/include/Eigen/src/Core/functors/BinaryFunctors.h0000644000176200001440000005067114567757725024262 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2010 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_BINARY_FUNCTORS_H #define EIGEN_BINARY_FUNCTORS_H namespace Eigen { namespace internal { //---------- associative binary functors ---------- template struct binary_op_base { typedef Arg1 first_argument_type; typedef Arg2 second_argument_type; }; /** \internal * \brief Template functor to compute the sum of two scalars * * \sa class CwiseBinaryOp, MatrixBase::operator+, class VectorwiseOp, DenseBase::sum() */ template struct scalar_sum_op : binary_op_base { typedef typename ScalarBinaryOpTraits::ReturnType result_type; #ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN EIGEN_EMPTY_STRUCT_CTOR(scalar_sum_op) #else scalar_sum_op() { EIGEN_SCALAR_BINARY_OP_PLUGIN } #endif EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return a + b; } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { return internal::padd(a,b); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type predux(const Packet& a) const { return internal::predux(a); } }; template struct functor_traits > { enum { Cost = (int(NumTraits::AddCost) + int(NumTraits::AddCost)) / 2, // rough estimate! PacketAccess = is_same::value && packet_traits::HasAdd && packet_traits::HasAdd // TODO vectorize mixed sum }; }; template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool scalar_sum_op::operator() (const bool& a, const bool& b) const { return a || b; } /** \internal * \brief Template functor to compute the product of two scalars * * \sa class CwiseBinaryOp, Cwise::operator*(), class VectorwiseOp, MatrixBase::redux() */ template struct scalar_product_op : binary_op_base { typedef typename ScalarBinaryOpTraits::ReturnType result_type; #ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN EIGEN_EMPTY_STRUCT_CTOR(scalar_product_op) #else scalar_product_op() { EIGEN_SCALAR_BINARY_OP_PLUGIN } #endif EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return a * b; } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { return internal::pmul(a,b); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type predux(const Packet& a) const { return internal::predux_mul(a); } }; template struct functor_traits > { enum { Cost = (int(NumTraits::MulCost) + int(NumTraits::MulCost))/2, // rough estimate! PacketAccess = is_same::value && packet_traits::HasMul && packet_traits::HasMul // TODO vectorize mixed product }; }; template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool scalar_product_op::operator() (const bool& a, const bool& b) const { return a && b; } /** \internal * \brief Template functor to compute the conjugate product of two scalars * * This is a short cut for conj(x) * y which is needed for optimization purpose; in Eigen2 support mode, this becomes x * conj(y) */ template struct scalar_conj_product_op : binary_op_base { enum { Conj = NumTraits::IsComplex }; typedef typename ScalarBinaryOpTraits::ReturnType result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_conj_product_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return conj_helper().pmul(a,b); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { return conj_helper().pmul(a,b); } }; template struct functor_traits > { enum { Cost = NumTraits::MulCost, PacketAccess = internal::is_same::value && packet_traits::HasMul }; }; /** \internal * \brief Template functor to compute the min of two scalars * * \sa class CwiseBinaryOp, MatrixBase::cwiseMin, class VectorwiseOp, MatrixBase::minCoeff() */ template struct scalar_min_op : binary_op_base { typedef typename ScalarBinaryOpTraits::ReturnType result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_min_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return internal::pmin(a, b); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { return internal::pmin(a,b); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type predux(const Packet& a) const { return internal::predux_min(a); } }; template struct functor_traits > { enum { Cost = (NumTraits::AddCost+NumTraits::AddCost)/2, PacketAccess = internal::is_same::value && packet_traits::HasMin }; }; /** \internal * \brief Template functor to compute the max of two scalars * * \sa class CwiseBinaryOp, MatrixBase::cwiseMax, class VectorwiseOp, MatrixBase::maxCoeff() */ template struct scalar_max_op : binary_op_base { typedef typename ScalarBinaryOpTraits::ReturnType result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_max_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return internal::pmax(a,b); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { return internal::pmax(a,b); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type predux(const Packet& a) const { return internal::predux_max(a); } }; template struct functor_traits > { enum { Cost = (NumTraits::AddCost+NumTraits::AddCost)/2, PacketAccess = internal::is_same::value && packet_traits::HasMax }; }; /** \internal * \brief Template functors for comparison of two scalars * \todo Implement packet-comparisons */ template struct scalar_cmp_op; template struct functor_traits > { enum { Cost = (NumTraits::AddCost+NumTraits::AddCost)/2, PacketAccess = false }; }; template struct result_of(LhsScalar,RhsScalar)> { typedef bool type; }; template struct scalar_cmp_op : binary_op_base { typedef bool result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a==b;} }; template struct scalar_cmp_op : binary_op_base { typedef bool result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a struct scalar_cmp_op : binary_op_base { typedef bool result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a<=b;} }; template struct scalar_cmp_op : binary_op_base { typedef bool result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a>b;} }; template struct scalar_cmp_op : binary_op_base { typedef bool result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a>=b;} }; template struct scalar_cmp_op : binary_op_base { typedef bool result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return !(a<=b || b<=a);} }; template struct scalar_cmp_op : binary_op_base { typedef bool result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_cmp_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a!=b;} }; /** \internal * \brief Template functor to compute the hypot of two \b positive \b and \b real scalars * * \sa MatrixBase::stableNorm(), class Redux */ template struct scalar_hypot_op : binary_op_base { EIGEN_EMPTY_STRUCT_CTOR(scalar_hypot_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar &x, const Scalar &y) const { // This functor is used by hypotNorm only for which it is faster to first apply abs // on all coefficients prior to reduction through hypot. // This way we avoid calling abs on positive and real entries, and this also permits // to seamlessly handle complexes. Otherwise we would have to handle both real and complexes // through the same functor... return internal::positive_real_hypot(x,y); } }; template struct functor_traits > { enum { Cost = 3 * NumTraits::AddCost + 2 * NumTraits::MulCost + 2 * scalar_div_cost::value, PacketAccess = false }; }; /** \internal * \brief Template functor to compute the pow of two scalars * See the specification of pow in https://en.cppreference.com/w/cpp/numeric/math/pow */ template struct scalar_pow_op : binary_op_base { typedef typename ScalarBinaryOpTraits::ReturnType result_type; #ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN EIGEN_EMPTY_STRUCT_CTOR(scalar_pow_op) #else scalar_pow_op() { typedef Scalar LhsScalar; typedef Exponent RhsScalar; EIGEN_SCALAR_BINARY_OP_PLUGIN } #endif EIGEN_DEVICE_FUNC inline result_type operator() (const Scalar& a, const Exponent& b) const { return numext::pow(a, b); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const { return generic_pow(a,b); } }; template struct functor_traits > { enum { Cost = 5 * NumTraits::MulCost, PacketAccess = (!NumTraits::IsComplex && !NumTraits::IsInteger && packet_traits::HasExp && packet_traits::HasLog && packet_traits::HasRound && packet_traits::HasCmp && // Temporarly disable packet access for half/bfloat16 until // accuracy is improved. !is_same::value && !is_same::value ) }; }; //---------- non associative binary functors ---------- /** \internal * \brief Template functor to compute the difference of two scalars * * \sa class CwiseBinaryOp, MatrixBase::operator- */ template struct scalar_difference_op : binary_op_base { typedef typename ScalarBinaryOpTraits::ReturnType result_type; #ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN EIGEN_EMPTY_STRUCT_CTOR(scalar_difference_op) #else scalar_difference_op() { EIGEN_SCALAR_BINARY_OP_PLUGIN } #endif EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return a - b; } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const { return internal::psub(a,b); } }; template struct functor_traits > { enum { Cost = (int(NumTraits::AddCost) + int(NumTraits::AddCost)) / 2, PacketAccess = is_same::value && packet_traits::HasSub && packet_traits::HasSub }; }; /** \internal * \brief Template functor to compute the quotient of two scalars * * \sa class CwiseBinaryOp, Cwise::operator/() */ template struct scalar_quotient_op : binary_op_base { typedef typename ScalarBinaryOpTraits::ReturnType result_type; #ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN EIGEN_EMPTY_STRUCT_CTOR(scalar_quotient_op) #else scalar_quotient_op() { EIGEN_SCALAR_BINARY_OP_PLUGIN } #endif EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return a / b; } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const { return internal::pdiv(a,b); } }; template struct functor_traits > { typedef typename scalar_quotient_op::result_type result_type; enum { PacketAccess = is_same::value && packet_traits::HasDiv && packet_traits::HasDiv, Cost = scalar_div_cost::value }; }; /** \internal * \brief Template functor to compute the and of two booleans * * \sa class CwiseBinaryOp, ArrayBase::operator&& */ struct scalar_boolean_and_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_boolean_and_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator() (const bool& a, const bool& b) const { return a && b; } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const { return internal::pand(a,b); } }; template<> struct functor_traits { enum { Cost = NumTraits::AddCost, PacketAccess = true }; }; /** \internal * \brief Template functor to compute the or of two booleans * * \sa class CwiseBinaryOp, ArrayBase::operator|| */ struct scalar_boolean_or_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_boolean_or_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator() (const bool& a, const bool& b) const { return a || b; } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const { return internal::por(a,b); } }; template<> struct functor_traits { enum { Cost = NumTraits::AddCost, PacketAccess = true }; }; /** \internal * \brief Template functor to compute the xor of two booleans * * \sa class CwiseBinaryOp, ArrayBase::operator^ */ struct scalar_boolean_xor_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_boolean_xor_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator() (const bool& a, const bool& b) const { return a ^ b; } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const { return internal::pxor(a,b); } }; template<> struct functor_traits { enum { Cost = NumTraits::AddCost, PacketAccess = true }; }; /** \internal * \brief Template functor to compute the absolute difference of two scalars * * \sa class CwiseBinaryOp, MatrixBase::absolute_difference */ template struct scalar_absolute_difference_op : binary_op_base { typedef typename ScalarBinaryOpTraits::ReturnType result_type; #ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN EIGEN_EMPTY_STRUCT_CTOR(scalar_absolute_difference_op) #else scalar_absolute_difference_op() { EIGEN_SCALAR_BINARY_OP_PLUGIN } #endif EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return numext::absdiff(a,b); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const { return internal::pabsdiff(a,b); } }; template struct functor_traits > { enum { Cost = (NumTraits::AddCost+NumTraits::AddCost)/2, PacketAccess = is_same::value && packet_traits::HasAbsDiff }; }; //---------- binary functors bound to a constant, thus appearing as a unary functor ---------- // The following two classes permits to turn any binary functor into a unary one with one argument bound to a constant value. // They are analogues to std::binder1st/binder2nd but with the following differences: // - they are compatible with packetOp // - they are portable across C++ versions (the std::binder* are deprecated in C++11) template struct bind1st_op : BinaryOp { typedef typename BinaryOp::first_argument_type first_argument_type; typedef typename BinaryOp::second_argument_type second_argument_type; typedef typename BinaryOp::result_type result_type; EIGEN_DEVICE_FUNC explicit bind1st_op(const first_argument_type &val) : m_value(val) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const second_argument_type& b) const { return BinaryOp::operator()(m_value,b); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& b) const { return BinaryOp::packetOp(internal::pset1(m_value), b); } first_argument_type m_value; }; template struct functor_traits > : functor_traits {}; template struct bind2nd_op : BinaryOp { typedef typename BinaryOp::first_argument_type first_argument_type; typedef typename BinaryOp::second_argument_type second_argument_type; typedef typename BinaryOp::result_type result_type; EIGEN_DEVICE_FUNC explicit bind2nd_op(const second_argument_type &val) : m_value(val) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const first_argument_type& a) const { return BinaryOp::operator()(a,m_value); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const { return BinaryOp::packetOp(a,internal::pset1(m_value)); } second_argument_type m_value; }; template struct functor_traits > : functor_traits {}; } // end namespace internal } // end namespace Eigen #endif // EIGEN_BINARY_FUNCTORS_H RcppEigen/inst/include/Eigen/src/Core/functors/AssignmentFunctors.h0000644000176200001440000001503614567757725025142 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2010 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_ASSIGNMENT_FUNCTORS_H #define EIGEN_ASSIGNMENT_FUNCTORS_H namespace Eigen { namespace internal { /** \internal * \brief Template functor for scalar/packet assignment * */ template struct assign_op { EIGEN_EMPTY_STRUCT_CTOR(assign_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(DstScalar& a, const SrcScalar& b) const { a = b; } template EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const { internal::pstoret(a,b); } }; // Empty overload for void type (used by PermutationMatrix) template struct assign_op {}; template struct functor_traits > { enum { Cost = NumTraits::ReadCost, PacketAccess = is_same::value && packet_traits::Vectorizable && packet_traits::Vectorizable }; }; /** \internal * \brief Template functor for scalar/packet assignment with addition * */ template struct add_assign_op { EIGEN_EMPTY_STRUCT_CTOR(add_assign_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(DstScalar& a, const SrcScalar& b) const { a += b; } template EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const { internal::pstoret(a,internal::padd(internal::ploadt(a),b)); } }; template struct functor_traits > { enum { Cost = NumTraits::ReadCost + NumTraits::AddCost, PacketAccess = is_same::value && packet_traits::HasAdd }; }; /** \internal * \brief Template functor for scalar/packet assignment with subtraction * */ template struct sub_assign_op { EIGEN_EMPTY_STRUCT_CTOR(sub_assign_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(DstScalar& a, const SrcScalar& b) const { a -= b; } template EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const { internal::pstoret(a,internal::psub(internal::ploadt(a),b)); } }; template struct functor_traits > { enum { Cost = NumTraits::ReadCost + NumTraits::AddCost, PacketAccess = is_same::value && packet_traits::HasSub }; }; /** \internal * \brief Template functor for scalar/packet assignment with multiplication * */ template struct mul_assign_op { EIGEN_EMPTY_STRUCT_CTOR(mul_assign_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(DstScalar& a, const SrcScalar& b) const { a *= b; } template EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const { internal::pstoret(a,internal::pmul(internal::ploadt(a),b)); } }; template struct functor_traits > { enum { Cost = NumTraits::ReadCost + NumTraits::MulCost, PacketAccess = is_same::value && packet_traits::HasMul }; }; /** \internal * \brief Template functor for scalar/packet assignment with diviving * */ template struct div_assign_op { EIGEN_EMPTY_STRUCT_CTOR(div_assign_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(DstScalar& a, const SrcScalar& b) const { a /= b; } template EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const { internal::pstoret(a,internal::pdiv(internal::ploadt(a),b)); } }; template struct functor_traits > { enum { Cost = NumTraits::ReadCost + NumTraits::MulCost, PacketAccess = is_same::value && packet_traits::HasDiv }; }; /** \internal * \brief Template functor for scalar/packet assignment with swapping * * It works as follow. For a non-vectorized evaluation loop, we have: * for(i) func(A.coeffRef(i), B.coeff(i)); * where B is a SwapWrapper expression. The trick is to make SwapWrapper::coeff behaves like a non-const coeffRef. * Actually, SwapWrapper might not even be needed since even if B is a plain expression, since it has to be writable * B.coeff already returns a const reference to the underlying scalar value. * * The case of a vectorized loop is more tricky: * for(i,j) func.assignPacket(&A.coeffRef(i,j), B.packet(i,j)); * Here, B must be a SwapWrapper whose packet function actually returns a proxy object holding a Scalar*, * the actual alignment and Packet type. * */ template struct swap_assign_op { EIGEN_EMPTY_STRUCT_CTOR(swap_assign_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Scalar& a, const Scalar& b) const { #ifdef EIGEN_GPUCC // FIXME is there some kind of cuda::swap? Scalar t=b; const_cast(b)=a; a=t; #else using std::swap; swap(a,const_cast(b)); #endif } }; template struct functor_traits > { enum { Cost = 3 * NumTraits::ReadCost, PacketAccess = #if defined(EIGEN_VECTORIZE_AVX) && EIGEN_COMP_CLANG && (EIGEN_COMP_CLANG<800 || defined(__apple_build_version__)) // This is a partial workaround for a bug in clang generating bad code // when mixing 256/512 bits loads and 128 bits moves. // See http://eigen.tuxfamily.org/bz/show_bug.cgi?id=1684 // https://bugs.llvm.org/show_bug.cgi?id=40815 0 #else packet_traits::Vectorizable #endif }; }; } // namespace internal } // namespace Eigen #endif // EIGEN_ASSIGNMENT_FUNCTORS_H RcppEigen/inst/include/Eigen/src/Core/functors/StlFunctors.h0000644000176200001440000001160614567757725023573 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2010 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_STL_FUNCTORS_H #define EIGEN_STL_FUNCTORS_H namespace Eigen { // Portable replacements for certain functors. namespace numext { template struct equal_to { typedef bool result_type; EIGEN_DEVICE_FUNC bool operator()(const T& lhs, const T& rhs) const { return lhs == rhs; } }; template struct not_equal_to { typedef bool result_type; EIGEN_DEVICE_FUNC bool operator()(const T& lhs, const T& rhs) const { return lhs != rhs; } }; } namespace internal { // default functor traits for STL functors: template struct functor_traits > { enum { Cost = NumTraits::MulCost, PacketAccess = false }; }; template struct functor_traits > { enum { Cost = NumTraits::MulCost, PacketAccess = false }; }; template struct functor_traits > { enum { Cost = NumTraits::AddCost, PacketAccess = false }; }; template struct functor_traits > { enum { Cost = NumTraits::AddCost, PacketAccess = false }; }; template struct functor_traits > { enum { Cost = NumTraits::AddCost, PacketAccess = false }; }; template struct functor_traits > { enum { Cost = 1, PacketAccess = false }; }; template struct functor_traits > { enum { Cost = 1, PacketAccess = false }; }; template struct functor_traits > { enum { Cost = 1, PacketAccess = false }; }; template struct functor_traits > { enum { Cost = 1, PacketAccess = false }; }; template struct functor_traits > { enum { Cost = 1, PacketAccess = false }; }; template struct functor_traits > { enum { Cost = 1, PacketAccess = false }; }; template struct functor_traits > { enum { Cost = 1, PacketAccess = false }; }; template struct functor_traits > { enum { Cost = 1, PacketAccess = false }; }; template struct functor_traits > : functor_traits > {}; template struct functor_traits > { enum { Cost = 1, PacketAccess = false }; }; template struct functor_traits > : functor_traits > {}; #if (EIGEN_COMP_CXXVER < 11) // std::binder* are deprecated since c++11 and will be removed in c++17 template struct functor_traits > { enum { Cost = functor_traits::Cost, PacketAccess = false }; }; template struct functor_traits > { enum { Cost = functor_traits::Cost, PacketAccess = false }; }; #endif #if (EIGEN_COMP_CXXVER < 17) // std::unary_negate is deprecated since c++17 and will be removed in c++20 template struct functor_traits > { enum { Cost = 1 + functor_traits::Cost, PacketAccess = false }; }; // std::binary_negate is deprecated since c++17 and will be removed in c++20 template struct functor_traits > { enum { Cost = 1 + functor_traits::Cost, PacketAccess = false }; }; #endif #ifdef EIGEN_STDEXT_SUPPORT template struct functor_traits > { enum { Cost = 0, PacketAccess = false }; }; template struct functor_traits > { enum { Cost = 0, PacketAccess = false }; }; template struct functor_traits > > { enum { Cost = 0, PacketAccess = false }; }; template struct functor_traits > > { enum { Cost = 0, PacketAccess = false }; }; template struct functor_traits > { enum { Cost = functor_traits::Cost + functor_traits::Cost, PacketAccess = false }; }; template struct functor_traits > { enum { Cost = functor_traits::Cost + functor_traits::Cost + functor_traits::Cost, PacketAccess = false }; }; #endif // EIGEN_STDEXT_SUPPORT // allow to add new functors and specializations of functor_traits from outside Eigen. // this macro is really needed because functor_traits must be specialized after it is declared but before it is used... #ifdef EIGEN_FUNCTORS_PLUGIN #include EIGEN_FUNCTORS_PLUGIN #endif } // end namespace internal } // end namespace Eigen #endif // EIGEN_STL_FUNCTORS_H RcppEigen/inst/include/Eigen/src/Core/Matrix.h0000644000176200001440000005742714567757725020721 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2006-2010 Benoit Jacob // Copyright (C) 2008-2009 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_MATRIX_H #define EIGEN_MATRIX_H namespace Eigen { namespace internal { template struct traits > { private: enum { size = internal::size_at_compile_time<_Rows,_Cols>::ret }; typedef typename find_best_packet<_Scalar,size>::type PacketScalar; enum { row_major_bit = _Options&RowMajor ? RowMajorBit : 0, is_dynamic_size_storage = _MaxRows==Dynamic || _MaxCols==Dynamic, max_size = is_dynamic_size_storage ? Dynamic : _MaxRows*_MaxCols, default_alignment = compute_default_alignment<_Scalar,max_size>::value, actual_alignment = ((_Options&DontAlign)==0) ? default_alignment : 0, required_alignment = unpacket_traits::alignment, packet_access_bit = (packet_traits<_Scalar>::Vectorizable && (EIGEN_UNALIGNED_VECTORIZE || (actual_alignment>=required_alignment))) ? PacketAccessBit : 0 }; public: typedef _Scalar Scalar; typedef Dense StorageKind; typedef Eigen::Index StorageIndex; typedef MatrixXpr XprKind; enum { RowsAtCompileTime = _Rows, ColsAtCompileTime = _Cols, MaxRowsAtCompileTime = _MaxRows, MaxColsAtCompileTime = _MaxCols, Flags = compute_matrix_flags<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols>::ret, Options = _Options, InnerStrideAtCompileTime = 1, OuterStrideAtCompileTime = (Options&RowMajor) ? ColsAtCompileTime : RowsAtCompileTime, // FIXME, the following flag in only used to define NeedsToAlign in PlainObjectBase EvaluatorFlags = LinearAccessBit | DirectAccessBit | packet_access_bit | row_major_bit, Alignment = actual_alignment }; }; } /** \class Matrix * \ingroup Core_Module * * \brief The matrix class, also used for vectors and row-vectors * * The %Matrix class is the work-horse for all \em dense (\ref dense "note") matrices and vectors within Eigen. * Vectors are matrices with one column, and row-vectors are matrices with one row. * * The %Matrix class encompasses \em both fixed-size and dynamic-size objects (\ref fixedsize "note"). * * The first three template parameters are required: * \tparam _Scalar Numeric type, e.g. float, double, int or std::complex. * User defined scalar types are supported as well (see \ref user_defined_scalars "here"). * \tparam _Rows Number of rows, or \b Dynamic * \tparam _Cols Number of columns, or \b Dynamic * * The remaining template parameters are optional -- in most cases you don't have to worry about them. * \tparam _Options A combination of either \b #RowMajor or \b #ColMajor, and of either * \b #AutoAlign or \b #DontAlign. * The former controls \ref TopicStorageOrders "storage order", and defaults to column-major. The latter controls alignment, which is required * for vectorization. It defaults to aligning matrices except for fixed sizes that aren't a multiple of the packet size. * \tparam _MaxRows Maximum number of rows. Defaults to \a _Rows (\ref maxrows "note"). * \tparam _MaxCols Maximum number of columns. Defaults to \a _Cols (\ref maxrows "note"). * * Eigen provides a number of typedefs covering the usual cases. Here are some examples: * * \li \c Matrix2d is a 2x2 square matrix of doubles (\c Matrix) * \li \c Vector4f is a vector of 4 floats (\c Matrix) * \li \c RowVector3i is a row-vector of 3 ints (\c Matrix) * * \li \c MatrixXf is a dynamic-size matrix of floats (\c Matrix) * \li \c VectorXf is a dynamic-size vector of floats (\c Matrix) * * \li \c Matrix2Xf is a partially fixed-size (dynamic-size) matrix of floats (\c Matrix) * \li \c MatrixX3d is a partially dynamic-size (fixed-size) matrix of double (\c Matrix) * * See \link matrixtypedefs this page \endlink for a complete list of predefined \em %Matrix and \em Vector typedefs. * * You can access elements of vectors and matrices using normal subscripting: * * \code * Eigen::VectorXd v(10); * v[0] = 0.1; * v[1] = 0.2; * v(0) = 0.3; * v(1) = 0.4; * * Eigen::MatrixXi m(10, 10); * m(0, 1) = 1; * m(0, 2) = 2; * m(0, 3) = 3; * \endcode * * This class can be extended with the help of the plugin mechanism described on the page * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_MATRIX_PLUGIN. * * Some notes: * *
*
\anchor dense Dense versus sparse:
*
This %Matrix class handles dense, not sparse matrices and vectors. For sparse matrices and vectors, see the Sparse module. * * Dense matrices and vectors are plain usual arrays of coefficients. All the coefficients are stored, in an ordinary contiguous array. * This is unlike Sparse matrices and vectors where the coefficients are stored as a list of nonzero coefficients.
* *
\anchor fixedsize Fixed-size versus dynamic-size:
*
Fixed-size means that the numbers of rows and columns are known are compile-time. In this case, Eigen allocates the array * of coefficients as a fixed-size array, as a class member. This makes sense for very small matrices, typically up to 4x4, sometimes up * to 16x16. Larger matrices should be declared as dynamic-size even if one happens to know their size at compile-time. * * Dynamic-size means that the numbers of rows or columns are not necessarily known at compile-time. In this case they are runtime * variables, and the array of coefficients is allocated dynamically on the heap. * * Note that \em dense matrices, be they Fixed-size or Dynamic-size, do not expand dynamically in the sense of a std::map. * If you want this behavior, see the Sparse module.
* *
\anchor maxrows _MaxRows and _MaxCols:
*
In most cases, one just leaves these parameters to the default values. * These parameters mean the maximum size of rows and columns that the matrix may have. They are useful in cases * when the exact numbers of rows and columns are not known are compile-time, but it is known at compile-time that they cannot * exceed a certain value. This happens when taking dynamic-size blocks inside fixed-size matrices: in this case _MaxRows and _MaxCols * are the dimensions of the original matrix, while _Rows and _Cols are Dynamic.
*
* * ABI and storage layout * * The table below summarizes the ABI of some possible Matrix instances which is fixed thorough the lifetime of Eigen 3. * * * * * * *
Matrix typeEquivalent C structure
\code Matrix \endcode\code * struct { * T *data; // with (size_t(data)%EIGEN_MAX_ALIGN_BYTES)==0 * Eigen::Index rows, cols; * }; * \endcode
\code * Matrix * Matrix \endcode\code * struct { * T *data; // with (size_t(data)%EIGEN_MAX_ALIGN_BYTES)==0 * Eigen::Index size; * }; * \endcode
\code Matrix \endcode\code * struct { * T data[Rows*Cols]; // with (size_t(data)%A(Rows*Cols*sizeof(T)))==0 * }; * \endcode
\code Matrix \endcode\code * struct { * T data[MaxRows*MaxCols]; // with (size_t(data)%A(MaxRows*MaxCols*sizeof(T)))==0 * Eigen::Index rows, cols; * }; * \endcode
* Note that in this table Rows, Cols, MaxRows and MaxCols are all positive integers. A(S) is defined to the largest possible power-of-two * smaller to EIGEN_MAX_STATIC_ALIGN_BYTES. * * \see MatrixBase for the majority of the API methods for matrices, \ref TopicClassHierarchy, * \ref TopicStorageOrders */ template class Matrix : public PlainObjectBase > { public: /** \brief Base class typedef. * \sa PlainObjectBase */ typedef PlainObjectBase Base; enum { Options = _Options }; EIGEN_DENSE_PUBLIC_INTERFACE(Matrix) typedef typename Base::PlainObject PlainObject; using Base::base; using Base::coeffRef; /** * \brief Assigns matrices to each other. * * \note This is a special case of the templated operator=. Its purpose is * to prevent a default operator= from hiding the templated operator=. * * \callgraph */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix& operator=(const Matrix& other) { return Base::_set(other); } /** \internal * \brief Copies the value of the expression \a other into \c *this with automatic resizing. * * *this might be resized to match the dimensions of \a other. If *this was a null matrix (not already initialized), * it will be initialized. * * Note that copying a row-vector into a vector (and conversely) is allowed. * The resizing, if any, is then done in the appropriate way so that row-vectors * remain row-vectors and vectors remain vectors. */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix& operator=(const DenseBase& other) { return Base::_set(other); } /* Here, doxygen failed to copy the brief information when using \copydoc */ /** * \brief Copies the generic expression \a other into *this. * \copydetails DenseBase::operator=(const EigenBase &other) */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix& operator=(const EigenBase &other) { return Base::operator=(other); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix& operator=(const ReturnByValue& func) { return Base::operator=(func); } /** \brief Default constructor. * * For fixed-size matrices, does nothing. * * For dynamic-size matrices, creates an empty matrix of size 0. Does not allocate any array. Such a matrix * is called a null matrix. This constructor is the unique way to create null matrices: resizing * a matrix to 0 is not supported. * * \sa resize(Index,Index) */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix() : Base() { Base::_check_template_params(); EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } // FIXME is it still needed EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Matrix(internal::constructor_without_unaligned_array_assert) : Base(internal::constructor_without_unaligned_array_assert()) { Base::_check_template_params(); EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } #if EIGEN_HAS_RVALUE_REFERENCES EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(Matrix&& other) EIGEN_NOEXCEPT_IF(std::is_nothrow_move_constructible::value) : Base(std::move(other)) { Base::_check_template_params(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix& operator=(Matrix&& other) EIGEN_NOEXCEPT_IF(std::is_nothrow_move_assignable::value) { Base::operator=(std::move(other)); return *this; } #endif #if EIGEN_HAS_CXX11 /** \copydoc PlainObjectBase(const Scalar&, const Scalar&, const Scalar&, const Scalar&, const ArgTypes&... args) * * Example: \include Matrix_variadic_ctor_cxx11.cpp * Output: \verbinclude Matrix_variadic_ctor_cxx11.out * * \sa Matrix(const std::initializer_list>&) */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) : Base(a0, a1, a2, a3, args...) {} /** \brief Constructs a Matrix and initializes it from the coefficients given as initializer-lists grouped by row. \cpp11 * * In the general case, the constructor takes a list of rows, each row being represented as a list of coefficients: * * Example: \include Matrix_initializer_list_23_cxx11.cpp * Output: \verbinclude Matrix_initializer_list_23_cxx11.out * * Each of the inner initializer lists must contain the exact same number of elements, otherwise an assertion is triggered. * * In the case of a compile-time column vector, implicit transposition from a single row is allowed. * Therefore VectorXd{{1,2,3,4,5}} is legal and the more verbose syntax * RowVectorXd{{1},{2},{3},{4},{5}} can be avoided: * * Example: \include Matrix_initializer_list_vector_cxx11.cpp * Output: \verbinclude Matrix_initializer_list_vector_cxx11.out * * In the case of fixed-sized matrices, the initializer list sizes must exactly match the matrix sizes, * and implicit transposition is allowed for compile-time vectors only. * * \sa Matrix(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) */ EIGEN_DEVICE_FUNC explicit EIGEN_STRONG_INLINE Matrix(const std::initializer_list>& list) : Base(list) {} #endif // end EIGEN_HAS_CXX11 #ifndef EIGEN_PARSED_BY_DOXYGEN // This constructor is for both 1x1 matrices and dynamic vectors template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Matrix(const T& x) { Base::_check_template_params(); Base::template _init1(x); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(const T0& x, const T1& y) { Base::_check_template_params(); Base::template _init2(x, y); } #else /** \brief Constructs a fixed-sized matrix initialized with coefficients starting at \a data */ EIGEN_DEVICE_FUNC explicit Matrix(const Scalar *data); /** \brief Constructs a vector or row-vector with given dimension. \only_for_vectors * * This is useful for dynamic-size vectors. For fixed-size vectors, * it is redundant to pass these parameters, so one should use the default constructor * Matrix() instead. * * \warning This constructor is disabled for fixed-size \c 1x1 matrices. For instance, * calling Matrix(1) will call the initialization constructor: Matrix(const Scalar&). * For fixed-size \c 1x1 matrices it is therefore recommended to use the default * constructor Matrix() instead, especially when using one of the non standard * \c EIGEN_INITIALIZE_MATRICES_BY_{ZERO,\c NAN} macros (see \ref TopicPreprocessorDirectives). */ EIGEN_STRONG_INLINE explicit Matrix(Index dim); /** \brief Constructs an initialized 1x1 matrix with the given coefficient * \sa Matrix(const Scalar&, const Scalar&, const Scalar&, const Scalar&, const ArgTypes&...) */ Matrix(const Scalar& x); /** \brief Constructs an uninitialized matrix with \a rows rows and \a cols columns. * * This is useful for dynamic-size matrices. For fixed-size matrices, * it is redundant to pass these parameters, so one should use the default constructor * Matrix() instead. * * \warning This constructor is disabled for fixed-size \c 1x2 and \c 2x1 vectors. For instance, * calling Matrix2f(2,1) will call the initialization constructor: Matrix(const Scalar& x, const Scalar& y). * For fixed-size \c 1x2 or \c 2x1 vectors it is therefore recommended to use the default * constructor Matrix() instead, especially when using one of the non standard * \c EIGEN_INITIALIZE_MATRICES_BY_{ZERO,\c NAN} macros (see \ref TopicPreprocessorDirectives). */ EIGEN_DEVICE_FUNC Matrix(Index rows, Index cols); /** \brief Constructs an initialized 2D vector with given coefficients * \sa Matrix(const Scalar&, const Scalar&, const Scalar&, const Scalar&, const ArgTypes&...) */ Matrix(const Scalar& x, const Scalar& y); #endif // end EIGEN_PARSED_BY_DOXYGEN /** \brief Constructs an initialized 3D vector with given coefficients * \sa Matrix(const Scalar&, const Scalar&, const Scalar&, const Scalar&, const ArgTypes&...) */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(const Scalar& x, const Scalar& y, const Scalar& z) { Base::_check_template_params(); EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Matrix, 3) m_storage.data()[0] = x; m_storage.data()[1] = y; m_storage.data()[2] = z; } /** \brief Constructs an initialized 4D vector with given coefficients * \sa Matrix(const Scalar&, const Scalar&, const Scalar&, const Scalar&, const ArgTypes&...) */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(const Scalar& x, const Scalar& y, const Scalar& z, const Scalar& w) { Base::_check_template_params(); EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Matrix, 4) m_storage.data()[0] = x; m_storage.data()[1] = y; m_storage.data()[2] = z; m_storage.data()[3] = w; } /** \brief Copy constructor */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(const Matrix& other) : Base(other) { } /** \brief Copy constructor for generic expressions. * \sa MatrixBase::operator=(const EigenBase&) */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(const EigenBase &other) : Base(other.derived()) { } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT { return 1; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT { return this->innerSize(); } /////////// Geometry module /////////// template EIGEN_DEVICE_FUNC explicit Matrix(const RotationBase& r); template EIGEN_DEVICE_FUNC Matrix& operator=(const RotationBase& r); // allow to extend Matrix outside Eigen #ifdef EIGEN_MATRIX_PLUGIN #include EIGEN_MATRIX_PLUGIN #endif protected: template friend struct internal::conservative_resize_like_impl; using Base::m_storage; }; /** \defgroup matrixtypedefs Global matrix typedefs * * \ingroup Core_Module * * %Eigen defines several typedef shortcuts for most common matrix and vector types. * * The general patterns are the following: * * \c MatrixSizeType where \c Size can be \c 2,\c 3,\c 4 for fixed size square matrices or \c X for dynamic size, * and where \c Type can be \c i for integer, \c f for float, \c d for double, \c cf for complex float, \c cd * for complex double. * * For example, \c Matrix3d is a fixed-size 3x3 matrix type of doubles, and \c MatrixXf is a dynamic-size matrix of floats. * * There are also \c VectorSizeType and \c RowVectorSizeType which are self-explanatory. For example, \c Vector4cf is * a fixed-size vector of 4 complex floats. * * With \cpp11, template alias are also defined for common sizes. * They follow the same pattern as above except that the scalar type suffix is replaced by a * template parameter, i.e.: * - `MatrixSize` where `Size` can be \c 2,\c 3,\c 4 for fixed size square matrices or \c X for dynamic size. * - `MatrixXSize` and `MatrixSizeX` where `Size` can be \c 2,\c 3,\c 4 for hybrid dynamic/fixed matrices. * - `VectorSize` and `RowVectorSize` for column and row vectors. * * With \cpp11, you can also use fully generic column and row vector types: `Vector` and `RowVector`. * * \sa class Matrix */ #define EIGEN_MAKE_TYPEDEFS(Type, TypeSuffix, Size, SizeSuffix) \ /** \ingroup matrixtypedefs */ \ typedef Matrix Matrix##SizeSuffix##TypeSuffix; \ /** \ingroup matrixtypedefs */ \ typedef Matrix Vector##SizeSuffix##TypeSuffix; \ /** \ingroup matrixtypedefs */ \ typedef Matrix RowVector##SizeSuffix##TypeSuffix; #define EIGEN_MAKE_FIXED_TYPEDEFS(Type, TypeSuffix, Size) \ /** \ingroup matrixtypedefs */ \ typedef Matrix Matrix##Size##X##TypeSuffix; \ /** \ingroup matrixtypedefs */ \ typedef Matrix Matrix##X##Size##TypeSuffix; #define EIGEN_MAKE_TYPEDEFS_ALL_SIZES(Type, TypeSuffix) \ EIGEN_MAKE_TYPEDEFS(Type, TypeSuffix, 2, 2) \ EIGEN_MAKE_TYPEDEFS(Type, TypeSuffix, 3, 3) \ EIGEN_MAKE_TYPEDEFS(Type, TypeSuffix, 4, 4) \ EIGEN_MAKE_TYPEDEFS(Type, TypeSuffix, Dynamic, X) \ EIGEN_MAKE_FIXED_TYPEDEFS(Type, TypeSuffix, 2) \ EIGEN_MAKE_FIXED_TYPEDEFS(Type, TypeSuffix, 3) \ EIGEN_MAKE_FIXED_TYPEDEFS(Type, TypeSuffix, 4) EIGEN_MAKE_TYPEDEFS_ALL_SIZES(int, i) EIGEN_MAKE_TYPEDEFS_ALL_SIZES(float, f) EIGEN_MAKE_TYPEDEFS_ALL_SIZES(double, d) EIGEN_MAKE_TYPEDEFS_ALL_SIZES(std::complex, cf) EIGEN_MAKE_TYPEDEFS_ALL_SIZES(std::complex, cd) #undef EIGEN_MAKE_TYPEDEFS_ALL_SIZES #undef EIGEN_MAKE_TYPEDEFS #undef EIGEN_MAKE_FIXED_TYPEDEFS #if EIGEN_HAS_CXX11 #define EIGEN_MAKE_TYPEDEFS(Size, SizeSuffix) \ /** \ingroup matrixtypedefs */ \ /** \brief \cpp11 */ \ template \ using Matrix##SizeSuffix = Matrix; \ /** \ingroup matrixtypedefs */ \ /** \brief \cpp11 */ \ template \ using Vector##SizeSuffix = Matrix; \ /** \ingroup matrixtypedefs */ \ /** \brief \cpp11 */ \ template \ using RowVector##SizeSuffix = Matrix; #define EIGEN_MAKE_FIXED_TYPEDEFS(Size) \ /** \ingroup matrixtypedefs */ \ /** \brief \cpp11 */ \ template \ using Matrix##Size##X = Matrix; \ /** \ingroup matrixtypedefs */ \ /** \brief \cpp11 */ \ template \ using Matrix##X##Size = Matrix; EIGEN_MAKE_TYPEDEFS(2, 2) EIGEN_MAKE_TYPEDEFS(3, 3) EIGEN_MAKE_TYPEDEFS(4, 4) EIGEN_MAKE_TYPEDEFS(Dynamic, X) EIGEN_MAKE_FIXED_TYPEDEFS(2) EIGEN_MAKE_FIXED_TYPEDEFS(3) EIGEN_MAKE_FIXED_TYPEDEFS(4) /** \ingroup matrixtypedefs * \brief \cpp11 */ template using Vector = Matrix; /** \ingroup matrixtypedefs * \brief \cpp11 */ template using RowVector = Matrix; #undef EIGEN_MAKE_TYPEDEFS #undef EIGEN_MAKE_FIXED_TYPEDEFS #endif // EIGEN_HAS_CXX11 } // end namespace Eigen #endif // EIGEN_MATRIX_H RcppEigen/inst/include/Eigen/src/Core/Stride.h0000644000176200001440000001016414567757725020672 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2010 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_STRIDE_H #define EIGEN_STRIDE_H namespace Eigen { /** \class Stride * \ingroup Core_Module * * \brief Holds strides information for Map * * This class holds the strides information for mapping arrays with strides with class Map. * * It holds two values: the inner stride and the outer stride. * * The inner stride is the pointer increment between two consecutive entries within a given row of a * row-major matrix or within a given column of a column-major matrix. * * The outer stride is the pointer increment between two consecutive rows of a row-major matrix or * between two consecutive columns of a column-major matrix. * * These two values can be passed either at compile-time as template parameters, or at runtime as * arguments to the constructor. * * Indeed, this class takes two template parameters: * \tparam _OuterStrideAtCompileTime the outer stride, or Dynamic if you want to specify it at runtime. * \tparam _InnerStrideAtCompileTime the inner stride, or Dynamic if you want to specify it at runtime. * * Here is an example: * \include Map_general_stride.cpp * Output: \verbinclude Map_general_stride.out * * Both strides can be negative, however, a negative stride of -1 cannot be specified at compiletime * because of the ambiguity with Dynamic which is defined to -1 (historically, negative strides were * not allowed). * * \sa class InnerStride, class OuterStride, \ref TopicStorageOrders */ template class Stride { public: typedef Eigen::Index Index; ///< \deprecated since Eigen 3.3 enum { InnerStrideAtCompileTime = _InnerStrideAtCompileTime, OuterStrideAtCompileTime = _OuterStrideAtCompileTime }; /** Default constructor, for use when strides are fixed at compile time */ EIGEN_DEVICE_FUNC Stride() : m_outer(OuterStrideAtCompileTime), m_inner(InnerStrideAtCompileTime) { // FIXME: for Eigen 4 we should use DynamicIndex instead of Dynamic. // FIXME: for Eigen 4 we should also unify this API with fix<> eigen_assert(InnerStrideAtCompileTime != Dynamic && OuterStrideAtCompileTime != Dynamic); } /** Constructor allowing to pass the strides at runtime */ EIGEN_DEVICE_FUNC Stride(Index outerStride, Index innerStride) : m_outer(outerStride), m_inner(innerStride) { } /** Copy constructor */ EIGEN_DEVICE_FUNC Stride(const Stride& other) : m_outer(other.outer()), m_inner(other.inner()) {} /** \returns the outer stride */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outer() const { return m_outer.value(); } /** \returns the inner stride */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index inner() const { return m_inner.value(); } protected: internal::variable_if_dynamic m_outer; internal::variable_if_dynamic m_inner; }; /** \brief Convenience specialization of Stride to specify only an inner stride * See class Map for some examples */ template class InnerStride : public Stride<0, Value> { typedef Stride<0, Value> Base; public: EIGEN_DEVICE_FUNC InnerStride() : Base() {} EIGEN_DEVICE_FUNC InnerStride(Index v) : Base(0, v) {} // FIXME making this explicit could break valid code }; /** \brief Convenience specialization of Stride to specify only an outer stride * See class Map for some examples */ template class OuterStride : public Stride { typedef Stride Base; public: EIGEN_DEVICE_FUNC OuterStride() : Base() {} EIGEN_DEVICE_FUNC OuterStride(Index v) : Base(v,0) {} // FIXME making this explicit could break valid code }; } // end namespace Eigen #endif // EIGEN_STRIDE_H RcppEigen/inst/include/Eigen/src/Core/Fuzzy.h0000644000176200001440000001317714567757725020576 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2006-2008 Benoit Jacob // Copyright (C) 2008 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_FUZZY_H #define EIGEN_FUZZY_H namespace Eigen { namespace internal { template::IsInteger> struct isApprox_selector { EIGEN_DEVICE_FUNC static bool run(const Derived& x, const OtherDerived& y, const typename Derived::RealScalar& prec) { typename internal::nested_eval::type nested(x); typename internal::nested_eval::type otherNested(y); return (nested - otherNested).cwiseAbs2().sum() <= prec * prec * numext::mini(nested.cwiseAbs2().sum(), otherNested.cwiseAbs2().sum()); } }; template struct isApprox_selector { EIGEN_DEVICE_FUNC static bool run(const Derived& x, const OtherDerived& y, const typename Derived::RealScalar&) { return x.matrix() == y.matrix(); } }; template::IsInteger> struct isMuchSmallerThan_object_selector { EIGEN_DEVICE_FUNC static bool run(const Derived& x, const OtherDerived& y, const typename Derived::RealScalar& prec) { return x.cwiseAbs2().sum() <= numext::abs2(prec) * y.cwiseAbs2().sum(); } }; template struct isMuchSmallerThan_object_selector { EIGEN_DEVICE_FUNC static bool run(const Derived& x, const OtherDerived&, const typename Derived::RealScalar&) { return x.matrix() == Derived::Zero(x.rows(), x.cols()).matrix(); } }; template::IsInteger> struct isMuchSmallerThan_scalar_selector { EIGEN_DEVICE_FUNC static bool run(const Derived& x, const typename Derived::RealScalar& y, const typename Derived::RealScalar& prec) { return x.cwiseAbs2().sum() <= numext::abs2(prec * y); } }; template struct isMuchSmallerThan_scalar_selector { EIGEN_DEVICE_FUNC static bool run(const Derived& x, const typename Derived::RealScalar&, const typename Derived::RealScalar&) { return x.matrix() == Derived::Zero(x.rows(), x.cols()).matrix(); } }; } // end namespace internal /** \returns \c true if \c *this is approximately equal to \a other, within the precision * determined by \a prec. * * \note The fuzzy compares are done multiplicatively. Two vectors \f$ v \f$ and \f$ w \f$ * are considered to be approximately equal within precision \f$ p \f$ if * \f[ \Vert v - w \Vert \leqslant p\,\min(\Vert v\Vert, \Vert w\Vert). \f] * For matrices, the comparison is done using the Hilbert-Schmidt norm (aka Frobenius norm * L2 norm). * * \note Because of the multiplicativeness of this comparison, one can't use this function * to check whether \c *this is approximately equal to the zero matrix or vector. * Indeed, \c isApprox(zero) returns false unless \c *this itself is exactly the zero matrix * or vector. If you want to test whether \c *this is zero, use internal::isMuchSmallerThan(const * RealScalar&, RealScalar) instead. * * \sa internal::isMuchSmallerThan(const RealScalar&, RealScalar) const */ template template EIGEN_DEVICE_FUNC bool DenseBase::isApprox( const DenseBase& other, const RealScalar& prec ) const { return internal::isApprox_selector::run(derived(), other.derived(), prec); } /** \returns \c true if the norm of \c *this is much smaller than \a other, * within the precision determined by \a prec. * * \note The fuzzy compares are done multiplicatively. A vector \f$ v \f$ is * considered to be much smaller than \f$ x \f$ within precision \f$ p \f$ if * \f[ \Vert v \Vert \leqslant p\,\vert x\vert. \f] * * For matrices, the comparison is done using the Hilbert-Schmidt norm. For this reason, * the value of the reference scalar \a other should come from the Hilbert-Schmidt norm * of a reference matrix of same dimensions. * * \sa isApprox(), isMuchSmallerThan(const DenseBase&, RealScalar) const */ template EIGEN_DEVICE_FUNC bool DenseBase::isMuchSmallerThan( const typename NumTraits::Real& other, const RealScalar& prec ) const { return internal::isMuchSmallerThan_scalar_selector::run(derived(), other, prec); } /** \returns \c true if the norm of \c *this is much smaller than the norm of \a other, * within the precision determined by \a prec. * * \note The fuzzy compares are done multiplicatively. A vector \f$ v \f$ is * considered to be much smaller than a vector \f$ w \f$ within precision \f$ p \f$ if * \f[ \Vert v \Vert \leqslant p\,\Vert w\Vert. \f] * For matrices, the comparison is done using the Hilbert-Schmidt norm. * * \sa isApprox(), isMuchSmallerThan(const RealScalar&, RealScalar) const */ template template EIGEN_DEVICE_FUNC bool DenseBase::isMuchSmallerThan( const DenseBase& other, const RealScalar& prec ) const { return internal::isMuchSmallerThan_object_selector::run(derived(), other.derived(), prec); } } // end namespace Eigen #endif // EIGEN_FUZZY_H RcppEigen/inst/include/Eigen/src/Core/CwiseBinaryOp.h0000644000176200001440000001734514567757725022166 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2014 Gael Guennebaud // Copyright (C) 2006-2008 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_CWISE_BINARY_OP_H #define EIGEN_CWISE_BINARY_OP_H namespace Eigen { namespace internal { template struct traits > { // we must not inherit from traits since it has // the potential to cause problems with MSVC typedef typename remove_all::type Ancestor; typedef typename traits::XprKind XprKind; enum { RowsAtCompileTime = traits::RowsAtCompileTime, ColsAtCompileTime = traits::ColsAtCompileTime, MaxRowsAtCompileTime = traits::MaxRowsAtCompileTime, MaxColsAtCompileTime = traits::MaxColsAtCompileTime }; // even though we require Lhs and Rhs to have the same scalar type (see CwiseBinaryOp constructor), // we still want to handle the case when the result type is different. typedef typename result_of< BinaryOp( const typename Lhs::Scalar&, const typename Rhs::Scalar& ) >::type Scalar; typedef typename cwise_promote_storage_type::StorageKind, typename traits::StorageKind, BinaryOp>::ret StorageKind; typedef typename promote_index_type::StorageIndex, typename traits::StorageIndex>::type StorageIndex; typedef typename Lhs::Nested LhsNested; typedef typename Rhs::Nested RhsNested; typedef typename remove_reference::type _LhsNested; typedef typename remove_reference::type _RhsNested; enum { Flags = cwise_promote_storage_order::StorageKind,typename traits::StorageKind,_LhsNested::Flags & RowMajorBit,_RhsNested::Flags & RowMajorBit>::value }; }; } // end namespace internal template class CwiseBinaryOpImpl; /** \class CwiseBinaryOp * \ingroup Core_Module * * \brief Generic expression where a coefficient-wise binary operator is applied to two expressions * * \tparam BinaryOp template functor implementing the operator * \tparam LhsType the type of the left-hand side * \tparam RhsType the type of the right-hand side * * This class represents an expression where a coefficient-wise binary operator is applied to two expressions. * It is the return type of binary operators, by which we mean only those binary operators where * both the left-hand side and the right-hand side are Eigen expressions. * For example, the return type of matrix1+matrix2 is a CwiseBinaryOp. * * Most of the time, this is the only way that it is used, so you typically don't have to name * CwiseBinaryOp types explicitly. * * \sa MatrixBase::binaryExpr(const MatrixBase &,const CustomBinaryOp &) const, class CwiseUnaryOp, class CwiseNullaryOp */ template class CwiseBinaryOp : public CwiseBinaryOpImpl< BinaryOp, LhsType, RhsType, typename internal::cwise_promote_storage_type::StorageKind, typename internal::traits::StorageKind, BinaryOp>::ret>, internal::no_assignment_operator { public: typedef typename internal::remove_all::type Functor; typedef typename internal::remove_all::type Lhs; typedef typename internal::remove_all::type Rhs; typedef typename CwiseBinaryOpImpl< BinaryOp, LhsType, RhsType, typename internal::cwise_promote_storage_type::StorageKind, typename internal::traits::StorageKind, BinaryOp>::ret>::Base Base; EIGEN_GENERIC_PUBLIC_INTERFACE(CwiseBinaryOp) typedef typename internal::ref_selector::type LhsNested; typedef typename internal::ref_selector::type RhsNested; typedef typename internal::remove_reference::type _LhsNested; typedef typename internal::remove_reference::type _RhsNested; #if EIGEN_COMP_MSVC && EIGEN_HAS_CXX11 //Required for Visual Studio or the Copy constructor will probably not get inlined! EIGEN_STRONG_INLINE CwiseBinaryOp(const CwiseBinaryOp&) = default; #endif EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CwiseBinaryOp(const Lhs& aLhs, const Rhs& aRhs, const BinaryOp& func = BinaryOp()) : m_lhs(aLhs), m_rhs(aRhs), m_functor(func) { EIGEN_CHECK_BINARY_COMPATIBILIY(BinaryOp,typename Lhs::Scalar,typename Rhs::Scalar); // require the sizes to match EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Lhs, Rhs) eigen_assert(aLhs.rows() == aRhs.rows() && aLhs.cols() == aRhs.cols()); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { // return the fixed size type if available to enable compile time optimizations return internal::traits::type>::RowsAtCompileTime==Dynamic ? m_rhs.rows() : m_lhs.rows(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { // return the fixed size type if available to enable compile time optimizations return internal::traits::type>::ColsAtCompileTime==Dynamic ? m_rhs.cols() : m_lhs.cols(); } /** \returns the left hand side nested expression */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const _LhsNested& lhs() const { return m_lhs; } /** \returns the right hand side nested expression */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const _RhsNested& rhs() const { return m_rhs; } /** \returns the functor representing the binary operation */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const BinaryOp& functor() const { return m_functor; } protected: LhsNested m_lhs; RhsNested m_rhs; const BinaryOp m_functor; }; // Generic API dispatcher template class CwiseBinaryOpImpl : public internal::generic_xpr_base >::type { public: typedef typename internal::generic_xpr_base >::type Base; }; /** replaces \c *this by \c *this - \a other. * * \returns a reference to \c *this */ template template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived & MatrixBase::operator-=(const MatrixBase &other) { call_assignment(derived(), other.derived(), internal::sub_assign_op()); return derived(); } /** replaces \c *this by \c *this + \a other. * * \returns a reference to \c *this */ template template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived & MatrixBase::operator+=(const MatrixBase& other) { call_assignment(derived(), other.derived(), internal::add_assign_op()); return derived(); } } // end namespace Eigen #endif // EIGEN_CWISE_BINARY_OP_H RcppEigen/inst/include/Eigen/src/Core/Inverse.h0000644000176200001440000000665714567757725021067 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2014-2019 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_INVERSE_H #define EIGEN_INVERSE_H namespace Eigen { template class InverseImpl; namespace internal { template struct traits > : traits { typedef typename XprType::PlainObject PlainObject; typedef traits BaseTraits; enum { Flags = BaseTraits::Flags & RowMajorBit }; }; } // end namespace internal /** \class Inverse * * \brief Expression of the inverse of another expression * * \tparam XprType the type of the expression we are taking the inverse * * This class represents an abstract expression of A.inverse() * and most of the time this is the only way it is used. * */ template class Inverse : public InverseImpl::StorageKind> { public: typedef typename XprType::StorageIndex StorageIndex; typedef typename XprType::Scalar Scalar; typedef typename internal::ref_selector::type XprTypeNested; typedef typename internal::remove_all::type XprTypeNestedCleaned; typedef typename internal::ref_selector::type Nested; typedef typename internal::remove_all::type NestedExpression; explicit EIGEN_DEVICE_FUNC Inverse(const XprType &xpr) : m_xpr(xpr) {} EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_xpr.cols(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_xpr.rows(); } EIGEN_DEVICE_FUNC const XprTypeNestedCleaned& nestedExpression() const { return m_xpr; } protected: XprTypeNested m_xpr; }; // Generic API dispatcher template class InverseImpl : public internal::generic_xpr_base >::type { public: typedef typename internal::generic_xpr_base >::type Base; typedef typename XprType::Scalar Scalar; private: Scalar coeff(Index row, Index col) const; Scalar coeff(Index i) const; }; namespace internal { /** \internal * \brief Default evaluator for Inverse expression. * * This default evaluator for Inverse expression simply evaluate the inverse into a temporary * by a call to internal::call_assignment_no_alias. * Therefore, inverse implementers only have to specialize Assignment, ...> for * there own nested expression. * * \sa class Inverse */ template struct unary_evaluator > : public evaluator::PlainObject> { typedef Inverse InverseType; typedef typename InverseType::PlainObject PlainObject; typedef evaluator Base; enum { Flags = Base::Flags | EvalBeforeNestingBit }; unary_evaluator(const InverseType& inv_xpr) : m_result(inv_xpr.rows(), inv_xpr.cols()) { ::new (static_cast(this)) Base(m_result); internal::call_assignment_no_alias(m_result, inv_xpr); } protected: PlainObject m_result; }; } // end namespace internal } // end namespace Eigen #endif // EIGEN_INVERSE_H RcppEigen/inst/include/Eigen/src/Core/VectorwiseOp.h0000644000176200001440000010454014567757725022073 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2019 Gael Guennebaud // Copyright (C) 2006-2008 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_PARTIAL_REDUX_H #define EIGEN_PARTIAL_REDUX_H namespace Eigen { /** \class PartialReduxExpr * \ingroup Core_Module * * \brief Generic expression of a partially reduxed matrix * * \tparam MatrixType the type of the matrix we are applying the redux operation * \tparam MemberOp type of the member functor * \tparam Direction indicates the direction of the redux (#Vertical or #Horizontal) * * This class represents an expression of a partial redux operator of a matrix. * It is the return type of some VectorwiseOp functions, * and most of the time this is the only way it is used. * * \sa class VectorwiseOp */ template< typename MatrixType, typename MemberOp, int Direction> class PartialReduxExpr; namespace internal { template struct traits > : traits { typedef typename MemberOp::result_type Scalar; typedef typename traits::StorageKind StorageKind; typedef typename traits::XprKind XprKind; typedef typename MatrixType::Scalar InputScalar; enum { RowsAtCompileTime = Direction==Vertical ? 1 : MatrixType::RowsAtCompileTime, ColsAtCompileTime = Direction==Horizontal ? 1 : MatrixType::ColsAtCompileTime, MaxRowsAtCompileTime = Direction==Vertical ? 1 : MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = Direction==Horizontal ? 1 : MatrixType::MaxColsAtCompileTime, Flags = RowsAtCompileTime == 1 ? RowMajorBit : 0, TraversalSize = Direction==Vertical ? MatrixType::RowsAtCompileTime : MatrixType::ColsAtCompileTime }; }; } template< typename MatrixType, typename MemberOp, int Direction> class PartialReduxExpr : public internal::dense_xpr_base< PartialReduxExpr >::type, internal::no_assignment_operator { public: typedef typename internal::dense_xpr_base::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(PartialReduxExpr) EIGEN_DEVICE_FUNC explicit PartialReduxExpr(const MatrixType& mat, const MemberOp& func = MemberOp()) : m_matrix(mat), m_functor(func) {} EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return (Direction==Vertical ? 1 : m_matrix.rows()); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return (Direction==Horizontal ? 1 : m_matrix.cols()); } EIGEN_DEVICE_FUNC typename MatrixType::Nested nestedExpression() const { return m_matrix; } EIGEN_DEVICE_FUNC const MemberOp& functor() const { return m_functor; } protected: typename MatrixType::Nested m_matrix; const MemberOp m_functor; }; template struct partial_redux_dummy_func; #define EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(MEMBER,COST,VECTORIZABLE,BINARYOP) \ template \ struct member_##MEMBER { \ EIGEN_EMPTY_STRUCT_CTOR(member_##MEMBER) \ typedef ResultType result_type; \ typedef BINARYOP BinaryOp; \ template struct Cost { enum { value = COST }; }; \ enum { Vectorizable = VECTORIZABLE }; \ template \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ ResultType operator()(const XprType& mat) const \ { return mat.MEMBER(); } \ BinaryOp binaryFunc() const { return BinaryOp(); } \ } #define EIGEN_MEMBER_FUNCTOR(MEMBER,COST) \ EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(MEMBER,COST,0,partial_redux_dummy_func) namespace internal { EIGEN_MEMBER_FUNCTOR(norm, (Size+5) * NumTraits::MulCost + (Size-1)*NumTraits::AddCost); EIGEN_MEMBER_FUNCTOR(stableNorm, (Size+5) * NumTraits::MulCost + (Size-1)*NumTraits::AddCost); EIGEN_MEMBER_FUNCTOR(blueNorm, (Size+5) * NumTraits::MulCost + (Size-1)*NumTraits::AddCost); EIGEN_MEMBER_FUNCTOR(hypotNorm, (Size-1) * functor_traits >::Cost ); EIGEN_MEMBER_FUNCTOR(all, (Size-1)*NumTraits::AddCost); EIGEN_MEMBER_FUNCTOR(any, (Size-1)*NumTraits::AddCost); EIGEN_MEMBER_FUNCTOR(count, (Size-1)*NumTraits::AddCost); EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(sum, (Size-1)*NumTraits::AddCost, 1, internal::scalar_sum_op); EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(minCoeff, (Size-1)*NumTraits::AddCost, 1, internal::scalar_min_op); EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(maxCoeff, (Size-1)*NumTraits::AddCost, 1, internal::scalar_max_op); EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(prod, (Size-1)*NumTraits::MulCost, 1, internal::scalar_product_op); template struct member_lpnorm { typedef ResultType result_type; enum { Vectorizable = 0 }; template struct Cost { enum { value = (Size+5) * NumTraits::MulCost + (Size-1)*NumTraits::AddCost }; }; EIGEN_DEVICE_FUNC member_lpnorm() {} template EIGEN_DEVICE_FUNC inline ResultType operator()(const XprType& mat) const { return mat.template lpNorm

(); } }; template struct member_redux { typedef BinaryOpT BinaryOp; typedef typename result_of< BinaryOp(const Scalar&,const Scalar&) >::type result_type; enum { Vectorizable = functor_traits::PacketAccess }; template struct Cost { enum { value = (Size-1) * functor_traits::Cost }; }; EIGEN_DEVICE_FUNC explicit member_redux(const BinaryOp func) : m_functor(func) {} template EIGEN_DEVICE_FUNC inline result_type operator()(const DenseBase& mat) const { return mat.redux(m_functor); } const BinaryOp& binaryFunc() const { return m_functor; } const BinaryOp m_functor; }; } /** \class VectorwiseOp * \ingroup Core_Module * * \brief Pseudo expression providing broadcasting and partial reduction operations * * \tparam ExpressionType the type of the object on which to do partial reductions * \tparam Direction indicates whether to operate on columns (#Vertical) or rows (#Horizontal) * * This class represents a pseudo expression with broadcasting and partial reduction features. * It is the return type of DenseBase::colwise() and DenseBase::rowwise() * and most of the time this is the only way it is explicitly used. * * To understand the logic of rowwise/colwise expression, let's consider a generic case `A.colwise().foo()` * where `foo` is any method of `VectorwiseOp`. This expression is equivalent to applying `foo()` to each * column of `A` and then re-assemble the outputs in a matrix expression: * \code [A.col(0).foo(), A.col(1).foo(), ..., A.col(A.cols()-1).foo()] \endcode * * Example: \include MatrixBase_colwise.cpp * Output: \verbinclude MatrixBase_colwise.out * * The begin() and end() methods are obviously exceptions to the previous rule as they * return STL-compatible begin/end iterators to the rows or columns of the nested expression. * Typical use cases include for-range-loop and calls to STL algorithms: * * Example: \include MatrixBase_colwise_iterator_cxx11.cpp * Output: \verbinclude MatrixBase_colwise_iterator_cxx11.out * * For a partial reduction on an empty input, some rules apply. * For the sake of clarity, let's consider a vertical reduction: * - If the number of columns is zero, then a 1x0 row-major vector expression is returned. * - Otherwise, if the number of rows is zero, then * - a row vector of zeros is returned for sum-like reductions (sum, squaredNorm, norm, etc.) * - a row vector of ones is returned for a product reduction (e.g., MatrixXd(n,0).colwise().prod()) * - an assert is triggered for all other reductions (minCoeff,maxCoeff,redux(bin_op)) * * \sa DenseBase::colwise(), DenseBase::rowwise(), class PartialReduxExpr */ template class VectorwiseOp { public: typedef typename ExpressionType::Scalar Scalar; typedef typename ExpressionType::RealScalar RealScalar; typedef Eigen::Index Index; ///< \deprecated since Eigen 3.3 typedef typename internal::ref_selector::non_const_type ExpressionTypeNested; typedef typename internal::remove_all::type ExpressionTypeNestedCleaned; template class Functor, typename ReturnScalar=Scalar> struct ReturnType { typedef PartialReduxExpr, Direction > Type; }; template struct ReduxReturnType { typedef PartialReduxExpr, Direction > Type; }; enum { isVertical = (Direction==Vertical) ? 1 : 0, isHorizontal = (Direction==Horizontal) ? 1 : 0 }; protected: template struct ExtendedType { typedef Replicate Type; }; /** \internal * Replicates a vector to match the size of \c *this */ template EIGEN_DEVICE_FUNC typename ExtendedType::Type extendedTo(const DenseBase& other) const { EIGEN_STATIC_ASSERT(EIGEN_IMPLIES(isVertical, OtherDerived::MaxColsAtCompileTime==1), YOU_PASSED_A_ROW_VECTOR_BUT_A_COLUMN_VECTOR_WAS_EXPECTED) EIGEN_STATIC_ASSERT(EIGEN_IMPLIES(isHorizontal, OtherDerived::MaxRowsAtCompileTime==1), YOU_PASSED_A_COLUMN_VECTOR_BUT_A_ROW_VECTOR_WAS_EXPECTED) return typename ExtendedType::Type (other.derived(), isVertical ? 1 : m_matrix.rows(), isHorizontal ? 1 : m_matrix.cols()); } template struct OppositeExtendedType { typedef Replicate Type; }; /** \internal * Replicates a vector in the opposite direction to match the size of \c *this */ template EIGEN_DEVICE_FUNC typename OppositeExtendedType::Type extendedToOpposite(const DenseBase& other) const { EIGEN_STATIC_ASSERT(EIGEN_IMPLIES(isHorizontal, OtherDerived::MaxColsAtCompileTime==1), YOU_PASSED_A_ROW_VECTOR_BUT_A_COLUMN_VECTOR_WAS_EXPECTED) EIGEN_STATIC_ASSERT(EIGEN_IMPLIES(isVertical, OtherDerived::MaxRowsAtCompileTime==1), YOU_PASSED_A_COLUMN_VECTOR_BUT_A_ROW_VECTOR_WAS_EXPECTED) return typename OppositeExtendedType::Type (other.derived(), isHorizontal ? 1 : m_matrix.rows(), isVertical ? 1 : m_matrix.cols()); } public: EIGEN_DEVICE_FUNC explicit inline VectorwiseOp(ExpressionType& matrix) : m_matrix(matrix) {} /** \internal */ EIGEN_DEVICE_FUNC inline const ExpressionType& _expression() const { return m_matrix; } #ifdef EIGEN_PARSED_BY_DOXYGEN /** STL-like RandomAccessIterator * iterator type over the columns or rows as returned by the begin() and end() methods. */ random_access_iterator_type iterator; /** This is the const version of iterator (aka read-only) */ random_access_iterator_type const_iterator; #else typedef internal::subvector_stl_iterator iterator; typedef internal::subvector_stl_iterator const_iterator; typedef internal::subvector_stl_reverse_iterator reverse_iterator; typedef internal::subvector_stl_reverse_iterator const_reverse_iterator; #endif /** returns an iterator to the first row (rowwise) or column (colwise) of the nested expression. * \sa end(), cbegin() */ iterator begin() { return iterator (m_matrix, 0); } /** const version of begin() */ const_iterator begin() const { return const_iterator(m_matrix, 0); } /** const version of begin() */ const_iterator cbegin() const { return const_iterator(m_matrix, 0); } /** returns a reverse iterator to the last row (rowwise) or column (colwise) of the nested expression. * \sa rend(), crbegin() */ reverse_iterator rbegin() { return reverse_iterator (m_matrix, m_matrix.template subVectors()-1); } /** const version of rbegin() */ const_reverse_iterator rbegin() const { return const_reverse_iterator (m_matrix, m_matrix.template subVectors()-1); } /** const version of rbegin() */ const_reverse_iterator crbegin() const { return const_reverse_iterator (m_matrix, m_matrix.template subVectors()-1); } /** returns an iterator to the row (resp. column) following the last row (resp. column) of the nested expression * \sa begin(), cend() */ iterator end() { return iterator (m_matrix, m_matrix.template subVectors()); } /** const version of end() */ const_iterator end() const { return const_iterator(m_matrix, m_matrix.template subVectors()); } /** const version of end() */ const_iterator cend() const { return const_iterator(m_matrix, m_matrix.template subVectors()); } /** returns a reverse iterator to the row (resp. column) before the first row (resp. column) of the nested expression * \sa begin(), cend() */ reverse_iterator rend() { return reverse_iterator (m_matrix, -1); } /** const version of rend() */ const_reverse_iterator rend() const { return const_reverse_iterator (m_matrix, -1); } /** const version of rend() */ const_reverse_iterator crend() const { return const_reverse_iterator (m_matrix, -1); } /** \returns a row or column vector expression of \c *this reduxed by \a func * * The template parameter \a BinaryOp is the type of the functor * of the custom redux operator. Note that func must be an associative operator. * * \warning the size along the reduction direction must be strictly positive, * otherwise an assertion is triggered. * * \sa class VectorwiseOp, DenseBase::colwise(), DenseBase::rowwise() */ template EIGEN_DEVICE_FUNC const typename ReduxReturnType::Type redux(const BinaryOp& func = BinaryOp()) const { eigen_assert(redux_length()>0 && "you are using an empty matrix"); return typename ReduxReturnType::Type(_expression(), internal::member_redux(func)); } typedef typename ReturnType::Type MinCoeffReturnType; typedef typename ReturnType::Type MaxCoeffReturnType; typedef PartialReduxExpr, const ExpressionTypeNestedCleaned>,internal::member_sum,Direction> SquaredNormReturnType; typedef CwiseUnaryOp, const SquaredNormReturnType> NormReturnType; typedef typename ReturnType::Type BlueNormReturnType; typedef typename ReturnType::Type StableNormReturnType; typedef typename ReturnType::Type HypotNormReturnType; typedef typename ReturnType::Type SumReturnType; typedef EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(SumReturnType,Scalar,quotient) MeanReturnType; typedef typename ReturnType::Type AllReturnType; typedef typename ReturnType::Type AnyReturnType; typedef PartialReduxExpr, Direction> CountReturnType; typedef typename ReturnType::Type ProdReturnType; typedef Reverse ConstReverseReturnType; typedef Reverse ReverseReturnType; template struct LpNormReturnType { typedef PartialReduxExpr,Direction> Type; }; /** \returns a row (or column) vector expression of the smallest coefficient * of each column (or row) of the referenced expression. * * \warning the size along the reduction direction must be strictly positive, * otherwise an assertion is triggered. * * \warning the result is undefined if \c *this contains NaN. * * Example: \include PartialRedux_minCoeff.cpp * Output: \verbinclude PartialRedux_minCoeff.out * * \sa DenseBase::minCoeff() */ EIGEN_DEVICE_FUNC const MinCoeffReturnType minCoeff() const { eigen_assert(redux_length()>0 && "you are using an empty matrix"); return MinCoeffReturnType(_expression()); } /** \returns a row (or column) vector expression of the largest coefficient * of each column (or row) of the referenced expression. * * \warning the size along the reduction direction must be strictly positive, * otherwise an assertion is triggered. * * \warning the result is undefined if \c *this contains NaN. * * Example: \include PartialRedux_maxCoeff.cpp * Output: \verbinclude PartialRedux_maxCoeff.out * * \sa DenseBase::maxCoeff() */ EIGEN_DEVICE_FUNC const MaxCoeffReturnType maxCoeff() const { eigen_assert(redux_length()>0 && "you are using an empty matrix"); return MaxCoeffReturnType(_expression()); } /** \returns a row (or column) vector expression of the squared norm * of each column (or row) of the referenced expression. * This is a vector with real entries, even if the original matrix has complex entries. * * Example: \include PartialRedux_squaredNorm.cpp * Output: \verbinclude PartialRedux_squaredNorm.out * * \sa DenseBase::squaredNorm() */ EIGEN_DEVICE_FUNC const SquaredNormReturnType squaredNorm() const { return SquaredNormReturnType(m_matrix.cwiseAbs2()); } /** \returns a row (or column) vector expression of the norm * of each column (or row) of the referenced expression. * This is a vector with real entries, even if the original matrix has complex entries. * * Example: \include PartialRedux_norm.cpp * Output: \verbinclude PartialRedux_norm.out * * \sa DenseBase::norm() */ EIGEN_DEVICE_FUNC const NormReturnType norm() const { return NormReturnType(squaredNorm()); } /** \returns a row (or column) vector expression of the norm * of each column (or row) of the referenced expression. * This is a vector with real entries, even if the original matrix has complex entries. * * Example: \include PartialRedux_norm.cpp * Output: \verbinclude PartialRedux_norm.out * * \sa DenseBase::norm() */ template EIGEN_DEVICE_FUNC const typename LpNormReturnType

::Type lpNorm() const { return typename LpNormReturnType

::Type(_expression()); } /** \returns a row (or column) vector expression of the norm * of each column (or row) of the referenced expression, using * Blue's algorithm. * This is a vector with real entries, even if the original matrix has complex entries. * * \sa DenseBase::blueNorm() */ EIGEN_DEVICE_FUNC const BlueNormReturnType blueNorm() const { return BlueNormReturnType(_expression()); } /** \returns a row (or column) vector expression of the norm * of each column (or row) of the referenced expression, avoiding * underflow and overflow. * This is a vector with real entries, even if the original matrix has complex entries. * * \sa DenseBase::stableNorm() */ EIGEN_DEVICE_FUNC const StableNormReturnType stableNorm() const { return StableNormReturnType(_expression()); } /** \returns a row (or column) vector expression of the norm * of each column (or row) of the referenced expression, avoiding * underflow and overflow using a concatenation of hypot() calls. * This is a vector with real entries, even if the original matrix has complex entries. * * \sa DenseBase::hypotNorm() */ EIGEN_DEVICE_FUNC const HypotNormReturnType hypotNorm() const { return HypotNormReturnType(_expression()); } /** \returns a row (or column) vector expression of the sum * of each column (or row) of the referenced expression. * * Example: \include PartialRedux_sum.cpp * Output: \verbinclude PartialRedux_sum.out * * \sa DenseBase::sum() */ EIGEN_DEVICE_FUNC const SumReturnType sum() const { return SumReturnType(_expression()); } /** \returns a row (or column) vector expression of the mean * of each column (or row) of the referenced expression. * * \sa DenseBase::mean() */ EIGEN_DEVICE_FUNC const MeanReturnType mean() const { return sum() / Scalar(Direction==Vertical?m_matrix.rows():m_matrix.cols()); } /** \returns a row (or column) vector expression representing * whether \b all coefficients of each respective column (or row) are \c true. * This expression can be assigned to a vector with entries of type \c bool. * * \sa DenseBase::all() */ EIGEN_DEVICE_FUNC const AllReturnType all() const { return AllReturnType(_expression()); } /** \returns a row (or column) vector expression representing * whether \b at \b least one coefficient of each respective column (or row) is \c true. * This expression can be assigned to a vector with entries of type \c bool. * * \sa DenseBase::any() */ EIGEN_DEVICE_FUNC const AnyReturnType any() const { return AnyReturnType(_expression()); } /** \returns a row (or column) vector expression representing * the number of \c true coefficients of each respective column (or row). * This expression can be assigned to a vector whose entries have the same type as is used to * index entries of the original matrix; for dense matrices, this is \c std::ptrdiff_t . * * Example: \include PartialRedux_count.cpp * Output: \verbinclude PartialRedux_count.out * * \sa DenseBase::count() */ EIGEN_DEVICE_FUNC const CountReturnType count() const { return CountReturnType(_expression()); } /** \returns a row (or column) vector expression of the product * of each column (or row) of the referenced expression. * * Example: \include PartialRedux_prod.cpp * Output: \verbinclude PartialRedux_prod.out * * \sa DenseBase::prod() */ EIGEN_DEVICE_FUNC const ProdReturnType prod() const { return ProdReturnType(_expression()); } /** \returns a matrix expression * where each column (or row) are reversed. * * Example: \include Vectorwise_reverse.cpp * Output: \verbinclude Vectorwise_reverse.out * * \sa DenseBase::reverse() */ EIGEN_DEVICE_FUNC const ConstReverseReturnType reverse() const { return ConstReverseReturnType( _expression() ); } /** \returns a writable matrix expression * where each column (or row) are reversed. * * \sa reverse() const */ EIGEN_DEVICE_FUNC ReverseReturnType reverse() { return ReverseReturnType( _expression() ); } typedef Replicate ReplicateReturnType; EIGEN_DEVICE_FUNC const ReplicateReturnType replicate(Index factor) const; /** * \return an expression of the replication of each column (or row) of \c *this * * Example: \include DirectionWise_replicate.cpp * Output: \verbinclude DirectionWise_replicate.out * * \sa VectorwiseOp::replicate(Index), DenseBase::replicate(), class Replicate */ // NOTE implemented here because of sunstudio's compilation errors // isVertical*Factor+isHorizontal instead of (isVertical?Factor:1) to handle CUDA bug with ternary operator template const Replicate EIGEN_DEVICE_FUNC replicate(Index factor = Factor) const { return Replicate (_expression(),isVertical?factor:1,isHorizontal?factor:1); } /////////// Artithmetic operators /////////// /** Copies the vector \a other to each subvector of \c *this */ template EIGEN_DEVICE_FUNC ExpressionType& operator=(const DenseBase& other) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived) //eigen_assert((m_matrix.isNull()) == (other.isNull())); FIXME return m_matrix = extendedTo(other.derived()); } /** Adds the vector \a other to each subvector of \c *this */ template EIGEN_DEVICE_FUNC ExpressionType& operator+=(const DenseBase& other) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived) return m_matrix += extendedTo(other.derived()); } /** Substracts the vector \a other to each subvector of \c *this */ template EIGEN_DEVICE_FUNC ExpressionType& operator-=(const DenseBase& other) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived) return m_matrix -= extendedTo(other.derived()); } /** Multiples each subvector of \c *this by the vector \a other */ template EIGEN_DEVICE_FUNC ExpressionType& operator*=(const DenseBase& other) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) EIGEN_STATIC_ASSERT_ARRAYXPR(ExpressionType) EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived) m_matrix *= extendedTo(other.derived()); return m_matrix; } /** Divides each subvector of \c *this by the vector \a other */ template EIGEN_DEVICE_FUNC ExpressionType& operator/=(const DenseBase& other) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) EIGEN_STATIC_ASSERT_ARRAYXPR(ExpressionType) EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived) m_matrix /= extendedTo(other.derived()); return m_matrix; } /** Returns the expression of the sum of the vector \a other to each subvector of \c *this */ template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC CwiseBinaryOp, const ExpressionTypeNestedCleaned, const typename ExtendedType::Type> operator+(const DenseBase& other) const { EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived) return m_matrix + extendedTo(other.derived()); } /** Returns the expression of the difference between each subvector of \c *this and the vector \a other */ template EIGEN_DEVICE_FUNC CwiseBinaryOp, const ExpressionTypeNestedCleaned, const typename ExtendedType::Type> operator-(const DenseBase& other) const { EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived) return m_matrix - extendedTo(other.derived()); } /** Returns the expression where each subvector is the product of the vector \a other * by the corresponding subvector of \c *this */ template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC CwiseBinaryOp, const ExpressionTypeNestedCleaned, const typename ExtendedType::Type> EIGEN_DEVICE_FUNC operator*(const DenseBase& other) const { EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) EIGEN_STATIC_ASSERT_ARRAYXPR(ExpressionType) EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived) return m_matrix * extendedTo(other.derived()); } /** Returns the expression where each subvector is the quotient of the corresponding * subvector of \c *this by the vector \a other */ template EIGEN_DEVICE_FUNC CwiseBinaryOp, const ExpressionTypeNestedCleaned, const typename ExtendedType::Type> operator/(const DenseBase& other) const { EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) EIGEN_STATIC_ASSERT_ARRAYXPR(ExpressionType) EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived) return m_matrix / extendedTo(other.derived()); } /** \returns an expression where each column (or row) of the referenced matrix are normalized. * The referenced matrix is \b not modified. * \sa MatrixBase::normalized(), normalize() */ EIGEN_DEVICE_FUNC CwiseBinaryOp, const ExpressionTypeNestedCleaned, const typename OppositeExtendedType::Type> normalized() const { return m_matrix.cwiseQuotient(extendedToOpposite(this->norm())); } /** Normalize in-place each row or columns of the referenced matrix. * \sa MatrixBase::normalize(), normalized() */ EIGEN_DEVICE_FUNC void normalize() { m_matrix = this->normalized(); } EIGEN_DEVICE_FUNC inline void reverseInPlace(); /////////// Geometry module /////////// typedef Homogeneous HomogeneousReturnType; EIGEN_DEVICE_FUNC HomogeneousReturnType homogeneous() const; typedef typename ExpressionType::PlainObject CrossReturnType; template EIGEN_DEVICE_FUNC const CrossReturnType cross(const MatrixBase& other) const; enum { HNormalized_Size = Direction==Vertical ? internal::traits::RowsAtCompileTime : internal::traits::ColsAtCompileTime, HNormalized_SizeMinusOne = HNormalized_Size==Dynamic ? Dynamic : HNormalized_Size-1 }; typedef Block::RowsAtCompileTime), Direction==Horizontal ? int(HNormalized_SizeMinusOne) : int(internal::traits::ColsAtCompileTime)> HNormalized_Block; typedef Block::RowsAtCompileTime), Direction==Horizontal ? 1 : int(internal::traits::ColsAtCompileTime)> HNormalized_Factors; typedef CwiseBinaryOp::Scalar>, const HNormalized_Block, const Replicate > HNormalizedReturnType; EIGEN_DEVICE_FUNC const HNormalizedReturnType hnormalized() const; # ifdef EIGEN_VECTORWISEOP_PLUGIN # include EIGEN_VECTORWISEOP_PLUGIN # endif protected: Index redux_length() const { return Direction==Vertical ? m_matrix.rows() : m_matrix.cols(); } ExpressionTypeNested m_matrix; }; //const colwise moved to DenseBase.h due to CUDA compiler bug /** \returns a writable VectorwiseOp wrapper of *this providing additional partial reduction operations * * \sa rowwise(), class VectorwiseOp, \ref TutorialReductionsVisitorsBroadcasting */ template EIGEN_DEVICE_FUNC inline typename DenseBase::ColwiseReturnType DenseBase::colwise() { return ColwiseReturnType(derived()); } //const rowwise moved to DenseBase.h due to CUDA compiler bug /** \returns a writable VectorwiseOp wrapper of *this providing additional partial reduction operations * * \sa colwise(), class VectorwiseOp, \ref TutorialReductionsVisitorsBroadcasting */ template EIGEN_DEVICE_FUNC inline typename DenseBase::RowwiseReturnType DenseBase::rowwise() { return RowwiseReturnType(derived()); } } // end namespace Eigen #endif // EIGEN_PARTIAL_REDUX_H RcppEigen/inst/include/Eigen/src/Core/CoreEvaluators.h0000644000176200001440000017454114567757725022410 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2011 Benoit Jacob // Copyright (C) 2011-2014 Gael Guennebaud // Copyright (C) 2011-2012 Jitse Niesen // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_COREEVALUATORS_H #define EIGEN_COREEVALUATORS_H namespace Eigen { namespace internal { // This class returns the evaluator kind from the expression storage kind. // Default assumes index based accessors template struct storage_kind_to_evaluator_kind { typedef IndexBased Kind; }; // This class returns the evaluator shape from the expression storage kind. // It can be Dense, Sparse, Triangular, Diagonal, SelfAdjoint, Band, etc. template struct storage_kind_to_shape; template<> struct storage_kind_to_shape { typedef DenseShape Shape; }; template<> struct storage_kind_to_shape { typedef SolverShape Shape; }; template<> struct storage_kind_to_shape { typedef PermutationShape Shape; }; template<> struct storage_kind_to_shape { typedef TranspositionsShape Shape; }; // Evaluators have to be specialized with respect to various criteria such as: // - storage/structure/shape // - scalar type // - etc. // Therefore, we need specialization of evaluator providing additional template arguments for each kind of evaluators. // We currently distinguish the following kind of evaluators: // - unary_evaluator for expressions taking only one arguments (CwiseUnaryOp, CwiseUnaryView, Transpose, MatrixWrapper, ArrayWrapper, Reverse, Replicate) // - binary_evaluator for expression taking two arguments (CwiseBinaryOp) // - ternary_evaluator for expression taking three arguments (CwiseTernaryOp) // - product_evaluator for linear algebra products (Product); special case of binary_evaluator because it requires additional tags for dispatching. // - mapbase_evaluator for Map, Block, Ref // - block_evaluator for Block (special dispatching to a mapbase_evaluator or unary_evaluator) template< typename T, typename Arg1Kind = typename evaluator_traits::Kind, typename Arg2Kind = typename evaluator_traits::Kind, typename Arg3Kind = typename evaluator_traits::Kind, typename Arg1Scalar = typename traits::Scalar, typename Arg2Scalar = typename traits::Scalar, typename Arg3Scalar = typename traits::Scalar> struct ternary_evaluator; template< typename T, typename LhsKind = typename evaluator_traits::Kind, typename RhsKind = typename evaluator_traits::Kind, typename LhsScalar = typename traits::Scalar, typename RhsScalar = typename traits::Scalar> struct binary_evaluator; template< typename T, typename Kind = typename evaluator_traits::Kind, typename Scalar = typename T::Scalar> struct unary_evaluator; // evaluator_traits contains traits for evaluator template struct evaluator_traits_base { // by default, get evaluator kind and shape from storage typedef typename storage_kind_to_evaluator_kind::StorageKind>::Kind Kind; typedef typename storage_kind_to_shape::StorageKind>::Shape Shape; }; // Default evaluator traits template struct evaluator_traits : public evaluator_traits_base { }; template::Shape > struct evaluator_assume_aliasing { static const bool value = false; }; // By default, we assume a unary expression: template struct evaluator : public unary_evaluator { typedef unary_evaluator Base; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const T& xpr) : Base(xpr) {} }; // TODO: Think about const-correctness template struct evaluator : evaluator { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const T& xpr) : evaluator(xpr) {} }; // ---------- base class for all evaluators ---------- template struct evaluator_base { // TODO that's not very nice to have to propagate all these traits. They are currently only needed to handle outer,inner indices. typedef traits ExpressionTraits; enum { Alignment = 0 }; // noncopyable: // Don't make this class inherit noncopyable as this kills EBO (Empty Base Optimization) // and make complex evaluator much larger than then should do. EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE evaluator_base() {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ~evaluator_base() {} private: EIGEN_DEVICE_FUNC evaluator_base(const evaluator_base&); EIGEN_DEVICE_FUNC const evaluator_base& operator=(const evaluator_base&); }; // -------------------- Matrix and Array -------------------- // // evaluator is a common base class for the // Matrix and Array evaluators. // Here we directly specialize evaluator. This is not really a unary expression, and it is, by definition, dense, // so no need for more sophisticated dispatching. // this helper permits to completely eliminate m_outerStride if it is known at compiletime. template class plainobjectbase_evaluator_data { public: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE plainobjectbase_evaluator_data(const Scalar* ptr, Index outerStride) : data(ptr) { #ifndef EIGEN_INTERNAL_DEBUGGING EIGEN_UNUSED_VARIABLE(outerStride); #endif eigen_internal_assert(outerStride==OuterStride); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index outerStride() const EIGEN_NOEXCEPT { return OuterStride; } const Scalar *data; }; template class plainobjectbase_evaluator_data { public: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE plainobjectbase_evaluator_data(const Scalar* ptr, Index outerStride) : data(ptr), m_outerStride(outerStride) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index outerStride() const { return m_outerStride; } const Scalar *data; protected: Index m_outerStride; }; template struct evaluator > : evaluator_base { typedef PlainObjectBase PlainObjectType; typedef typename PlainObjectType::Scalar Scalar; typedef typename PlainObjectType::CoeffReturnType CoeffReturnType; enum { IsRowMajor = PlainObjectType::IsRowMajor, IsVectorAtCompileTime = PlainObjectType::IsVectorAtCompileTime, RowsAtCompileTime = PlainObjectType::RowsAtCompileTime, ColsAtCompileTime = PlainObjectType::ColsAtCompileTime, CoeffReadCost = NumTraits::ReadCost, Flags = traits::EvaluatorFlags, Alignment = traits::Alignment }; enum { // We do not need to know the outer stride for vectors OuterStrideAtCompileTime = IsVectorAtCompileTime ? 0 : int(IsRowMajor) ? ColsAtCompileTime : RowsAtCompileTime }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE evaluator() : m_d(0,OuterStrideAtCompileTime) { EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const PlainObjectType& m) : m_d(m.data(),IsVectorAtCompileTime ? 0 : m.outerStride()) { EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { if (IsRowMajor) return m_d.data[row * m_d.outerStride() + col]; else return m_d.data[row + col * m_d.outerStride()]; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_d.data[index]; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { if (IsRowMajor) return const_cast(m_d.data)[row * m_d.outerStride() + col]; else return const_cast(m_d.data)[row + col * m_d.outerStride()]; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return const_cast(m_d.data)[index]; } template EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { if (IsRowMajor) return ploadt(m_d.data + row * m_d.outerStride() + col); else return ploadt(m_d.data + row + col * m_d.outerStride()); } template EIGEN_STRONG_INLINE PacketType packet(Index index) const { return ploadt(m_d.data + index); } template EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { if (IsRowMajor) return pstoret (const_cast(m_d.data) + row * m_d.outerStride() + col, x); else return pstoret (const_cast(m_d.data) + row + col * m_d.outerStride(), x); } template EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { return pstoret(const_cast(m_d.data) + index, x); } protected: plainobjectbase_evaluator_data m_d; }; template struct evaluator > : evaluator > > { typedef Matrix XprType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE evaluator() {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& m) : evaluator >(m) { } }; template struct evaluator > : evaluator > > { typedef Array XprType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE evaluator() {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& m) : evaluator >(m) { } }; // -------------------- Transpose -------------------- template struct unary_evaluator, IndexBased> : evaluator_base > { typedef Transpose XprType; enum { CoeffReadCost = evaluator::CoeffReadCost, Flags = evaluator::Flags ^ RowMajorBit, Alignment = evaluator::Alignment }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit unary_evaluator(const XprType& t) : m_argImpl(t.nestedExpression()) {} typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { return m_argImpl.coeff(col, row); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_argImpl.coeff(index); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { return m_argImpl.coeffRef(col, row); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename XprType::Scalar& coeffRef(Index index) { return m_argImpl.coeffRef(index); } template EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_argImpl.template packet(col, row); } template EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_argImpl.template packet(index); } template EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { m_argImpl.template writePacket(col, row, x); } template EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { m_argImpl.template writePacket(index, x); } protected: evaluator m_argImpl; }; // -------------------- CwiseNullaryOp -------------------- // Like Matrix and Array, this is not really a unary expression, so we directly specialize evaluator. // Likewise, there is not need to more sophisticated dispatching here. template::value, bool has_unary = has_unary_operator::value, bool has_binary = has_binary_operator::value> struct nullary_wrapper { template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i, IndexType j) const { return op(i,j); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i) const { return op(i); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i, IndexType j) const { return op.template packetOp(i,j); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i) const { return op.template packetOp(i); } }; template struct nullary_wrapper { template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType=0, IndexType=0) const { return op(); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType=0, IndexType=0) const { return op.template packetOp(); } }; template struct nullary_wrapper { template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i, IndexType j=0) const { return op(i,j); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i, IndexType j=0) const { return op.template packetOp(i,j); } }; // We need the following specialization for vector-only functors assigned to a runtime vector, // for instance, using linspace and assigning a RowVectorXd to a MatrixXd or even a row of a MatrixXd. // In this case, i==0 and j is used for the actual iteration. template struct nullary_wrapper { template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i, IndexType j) const { eigen_assert(i==0 || j==0); return op(i+j); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i, IndexType j) const { eigen_assert(i==0 || j==0); return op.template packetOp(i+j); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i) const { return op(i); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i) const { return op.template packetOp(i); } }; template struct nullary_wrapper {}; #if 0 && EIGEN_COMP_MSVC>0 // Disable this ugly workaround. This is now handled in traits::match, // but this piece of code might still become handly if some other weird compilation // erros pop up again. // MSVC exhibits a weird compilation error when // compiling: // Eigen::MatrixXf A = MatrixXf::Random(3,3); // Ref R = 2.f*A; // and that has_*ary_operator> have not been instantiated yet. // The "problem" is that evaluator<2.f*A> is instantiated by traits::match<2.f*A> // and at that time has_*ary_operator returns true regardless of T. // Then nullary_wrapper is badly instantiated as nullary_wrapper<.,.,true,true,true>. // The trick is thus to defer the proper instantiation of nullary_wrapper when coeff(), // and packet() are really instantiated as implemented below: // This is a simple wrapper around Index to enforce the re-instantiation of // has_*ary_operator when needed. template struct nullary_wrapper_workaround_msvc { nullary_wrapper_workaround_msvc(const T&); operator T()const; }; template struct nullary_wrapper { template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i, IndexType j) const { return nullary_wrapper >::value, has_unary_operator >::value, has_binary_operator >::value>().operator()(op,i,j); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i) const { return nullary_wrapper >::value, has_unary_operator >::value, has_binary_operator >::value>().operator()(op,i); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i, IndexType j) const { return nullary_wrapper >::value, has_unary_operator >::value, has_binary_operator >::value>().template packetOp(op,i,j); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i) const { return nullary_wrapper >::value, has_unary_operator >::value, has_binary_operator >::value>().template packetOp(op,i); } }; #endif // MSVC workaround template struct evaluator > : evaluator_base > { typedef CwiseNullaryOp XprType; typedef typename internal::remove_all::type PlainObjectTypeCleaned; enum { CoeffReadCost = internal::functor_traits::Cost, Flags = (evaluator::Flags & ( HereditaryBits | (functor_has_linear_access::ret ? LinearAccessBit : 0) | (functor_traits::PacketAccess ? PacketAccessBit : 0))) | (functor_traits::IsRepeatable ? 0 : EvalBeforeNestingBit), Alignment = AlignedMax }; EIGEN_DEVICE_FUNC explicit evaluator(const XprType& n) : m_functor(n.functor()), m_wrapper() { EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } typedef typename XprType::CoeffReturnType CoeffReturnType; template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(IndexType row, IndexType col) const { return m_wrapper(m_functor, row, col); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(IndexType index) const { return m_wrapper(m_functor,index); } template EIGEN_STRONG_INLINE PacketType packet(IndexType row, IndexType col) const { return m_wrapper.template packetOp(m_functor, row, col); } template EIGEN_STRONG_INLINE PacketType packet(IndexType index) const { return m_wrapper.template packetOp(m_functor, index); } protected: const NullaryOp m_functor; const internal::nullary_wrapper m_wrapper; }; // -------------------- CwiseUnaryOp -------------------- template struct unary_evaluator, IndexBased > : evaluator_base > { typedef CwiseUnaryOp XprType; enum { CoeffReadCost = int(evaluator::CoeffReadCost) + int(functor_traits::Cost), Flags = evaluator::Flags & (HereditaryBits | LinearAccessBit | (functor_traits::PacketAccess ? PacketAccessBit : 0)), Alignment = evaluator::Alignment }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit unary_evaluator(const XprType& op) : m_d(op) { EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits::Cost); EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } typedef typename XprType::CoeffReturnType CoeffReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { return m_d.func()(m_d.argImpl.coeff(row, col)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_d.func()(m_d.argImpl.coeff(index)); } template EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_d.func().packetOp(m_d.argImpl.template packet(row, col)); } template EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_d.func().packetOp(m_d.argImpl.template packet(index)); } protected: // this helper permits to completely eliminate the functor if it is empty struct Data { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Data(const XprType& xpr) : op(xpr.functor()), argImpl(xpr.nestedExpression()) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const UnaryOp& func() const { return op; } UnaryOp op; evaluator argImpl; }; Data m_d; }; // -------------------- CwiseTernaryOp -------------------- // this is a ternary expression template struct evaluator > : public ternary_evaluator > { typedef CwiseTernaryOp XprType; typedef ternary_evaluator > Base; EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) : Base(xpr) {} }; template struct ternary_evaluator, IndexBased, IndexBased> : evaluator_base > { typedef CwiseTernaryOp XprType; enum { CoeffReadCost = int(evaluator::CoeffReadCost) + int(evaluator::CoeffReadCost) + int(evaluator::CoeffReadCost) + int(functor_traits::Cost), Arg1Flags = evaluator::Flags, Arg2Flags = evaluator::Flags, Arg3Flags = evaluator::Flags, SameType = is_same::value && is_same::value, StorageOrdersAgree = (int(Arg1Flags)&RowMajorBit)==(int(Arg2Flags)&RowMajorBit) && (int(Arg1Flags)&RowMajorBit)==(int(Arg3Flags)&RowMajorBit), Flags0 = (int(Arg1Flags) | int(Arg2Flags) | int(Arg3Flags)) & ( HereditaryBits | (int(Arg1Flags) & int(Arg2Flags) & int(Arg3Flags) & ( (StorageOrdersAgree ? LinearAccessBit : 0) | (functor_traits::PacketAccess && StorageOrdersAgree && SameType ? PacketAccessBit : 0) ) ) ), Flags = (Flags0 & ~RowMajorBit) | (Arg1Flags & RowMajorBit), Alignment = EIGEN_PLAIN_ENUM_MIN( EIGEN_PLAIN_ENUM_MIN(evaluator::Alignment, evaluator::Alignment), evaluator::Alignment) }; EIGEN_DEVICE_FUNC explicit ternary_evaluator(const XprType& xpr) : m_d(xpr) { EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits::Cost); EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } typedef typename XprType::CoeffReturnType CoeffReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { return m_d.func()(m_d.arg1Impl.coeff(row, col), m_d.arg2Impl.coeff(row, col), m_d.arg3Impl.coeff(row, col)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_d.func()(m_d.arg1Impl.coeff(index), m_d.arg2Impl.coeff(index), m_d.arg3Impl.coeff(index)); } template EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_d.func().packetOp(m_d.arg1Impl.template packet(row, col), m_d.arg2Impl.template packet(row, col), m_d.arg3Impl.template packet(row, col)); } template EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_d.func().packetOp(m_d.arg1Impl.template packet(index), m_d.arg2Impl.template packet(index), m_d.arg3Impl.template packet(index)); } protected: // this helper permits to completely eliminate the functor if it is empty struct Data { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Data(const XprType& xpr) : op(xpr.functor()), arg1Impl(xpr.arg1()), arg2Impl(xpr.arg2()), arg3Impl(xpr.arg3()) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TernaryOp& func() const { return op; } TernaryOp op; evaluator arg1Impl; evaluator arg2Impl; evaluator arg3Impl; }; Data m_d; }; // -------------------- CwiseBinaryOp -------------------- // this is a binary expression template struct evaluator > : public binary_evaluator > { typedef CwiseBinaryOp XprType; typedef binary_evaluator > Base; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& xpr) : Base(xpr) {} }; template struct binary_evaluator, IndexBased, IndexBased> : evaluator_base > { typedef CwiseBinaryOp XprType; enum { CoeffReadCost = int(evaluator::CoeffReadCost) + int(evaluator::CoeffReadCost) + int(functor_traits::Cost), LhsFlags = evaluator::Flags, RhsFlags = evaluator::Flags, SameType = is_same::value, StorageOrdersAgree = (int(LhsFlags)&RowMajorBit)==(int(RhsFlags)&RowMajorBit), Flags0 = (int(LhsFlags) | int(RhsFlags)) & ( HereditaryBits | (int(LhsFlags) & int(RhsFlags) & ( (StorageOrdersAgree ? LinearAccessBit : 0) | (functor_traits::PacketAccess && StorageOrdersAgree && SameType ? PacketAccessBit : 0) ) ) ), Flags = (Flags0 & ~RowMajorBit) | (LhsFlags & RowMajorBit), Alignment = EIGEN_PLAIN_ENUM_MIN(evaluator::Alignment,evaluator::Alignment) }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit binary_evaluator(const XprType& xpr) : m_d(xpr) { EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits::Cost); EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } typedef typename XprType::CoeffReturnType CoeffReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { return m_d.func()(m_d.lhsImpl.coeff(row, col), m_d.rhsImpl.coeff(row, col)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_d.func()(m_d.lhsImpl.coeff(index), m_d.rhsImpl.coeff(index)); } template EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_d.func().packetOp(m_d.lhsImpl.template packet(row, col), m_d.rhsImpl.template packet(row, col)); } template EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_d.func().packetOp(m_d.lhsImpl.template packet(index), m_d.rhsImpl.template packet(index)); } protected: // this helper permits to completely eliminate the functor if it is empty struct Data { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Data(const XprType& xpr) : op(xpr.functor()), lhsImpl(xpr.lhs()), rhsImpl(xpr.rhs()) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const BinaryOp& func() const { return op; } BinaryOp op; evaluator lhsImpl; evaluator rhsImpl; }; Data m_d; }; // -------------------- CwiseUnaryView -------------------- template struct unary_evaluator, IndexBased> : evaluator_base > { typedef CwiseUnaryView XprType; enum { CoeffReadCost = int(evaluator::CoeffReadCost) + int(functor_traits::Cost), Flags = (evaluator::Flags & (HereditaryBits | LinearAccessBit | DirectAccessBit)), Alignment = 0 // FIXME it is not very clear why alignment is necessarily lost... }; EIGEN_DEVICE_FUNC explicit unary_evaluator(const XprType& op) : m_d(op) { EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits::Cost); EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { return m_d.func()(m_d.argImpl.coeff(row, col)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_d.func()(m_d.argImpl.coeff(index)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { return m_d.func()(m_d.argImpl.coeffRef(row, col)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return m_d.func()(m_d.argImpl.coeffRef(index)); } protected: // this helper permits to completely eliminate the functor if it is empty struct Data { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Data(const XprType& xpr) : op(xpr.functor()), argImpl(xpr.nestedExpression()) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const UnaryOp& func() const { return op; } UnaryOp op; evaluator argImpl; }; Data m_d; }; // -------------------- Map -------------------- // FIXME perhaps the PlainObjectType could be provided by Derived::PlainObject ? // but that might complicate template specialization template struct mapbase_evaluator; template struct mapbase_evaluator : evaluator_base { typedef Derived XprType; typedef typename XprType::PointerType PointerType; typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; enum { IsRowMajor = XprType::RowsAtCompileTime, ColsAtCompileTime = XprType::ColsAtCompileTime, CoeffReadCost = NumTraits::ReadCost }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit mapbase_evaluator(const XprType& map) : m_data(const_cast(map.data())), m_innerStride(map.innerStride()), m_outerStride(map.outerStride()) { EIGEN_STATIC_ASSERT(EIGEN_IMPLIES(evaluator::Flags&PacketAccessBit, internal::inner_stride_at_compile_time::ret==1), PACKET_ACCESS_REQUIRES_TO_HAVE_INNER_STRIDE_FIXED_TO_1); EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { return m_data[col * colStride() + row * rowStride()]; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_data[index * m_innerStride.value()]; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { return m_data[col * colStride() + row * rowStride()]; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return m_data[index * m_innerStride.value()]; } template EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { PointerType ptr = m_data + row * rowStride() + col * colStride(); return internal::ploadt(ptr); } template EIGEN_STRONG_INLINE PacketType packet(Index index) const { return internal::ploadt(m_data + index * m_innerStride.value()); } template EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { PointerType ptr = m_data + row * rowStride() + col * colStride(); return internal::pstoret(ptr, x); } template EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { internal::pstoret(m_data + index * m_innerStride.value(), x); } protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rowStride() const EIGEN_NOEXCEPT { return XprType::IsRowMajor ? m_outerStride.value() : m_innerStride.value(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index colStride() const EIGEN_NOEXCEPT { return XprType::IsRowMajor ? m_innerStride.value() : m_outerStride.value(); } PointerType m_data; const internal::variable_if_dynamic m_innerStride; const internal::variable_if_dynamic m_outerStride; }; template struct evaluator > : public mapbase_evaluator, PlainObjectType> { typedef Map XprType; typedef typename XprType::Scalar Scalar; // TODO: should check for smaller packet types once we can handle multi-sized packet types typedef typename packet_traits::type PacketScalar; enum { InnerStrideAtCompileTime = StrideType::InnerStrideAtCompileTime == 0 ? int(PlainObjectType::InnerStrideAtCompileTime) : int(StrideType::InnerStrideAtCompileTime), OuterStrideAtCompileTime = StrideType::OuterStrideAtCompileTime == 0 ? int(PlainObjectType::OuterStrideAtCompileTime) : int(StrideType::OuterStrideAtCompileTime), HasNoInnerStride = InnerStrideAtCompileTime == 1, HasNoOuterStride = StrideType::OuterStrideAtCompileTime == 0, HasNoStride = HasNoInnerStride && HasNoOuterStride, IsDynamicSize = PlainObjectType::SizeAtCompileTime==Dynamic, PacketAccessMask = bool(HasNoInnerStride) ? ~int(0) : ~int(PacketAccessBit), LinearAccessMask = bool(HasNoStride) || bool(PlainObjectType::IsVectorAtCompileTime) ? ~int(0) : ~int(LinearAccessBit), Flags = int( evaluator::Flags) & (LinearAccessMask&PacketAccessMask), Alignment = int(MapOptions)&int(AlignedMask) }; EIGEN_DEVICE_FUNC explicit evaluator(const XprType& map) : mapbase_evaluator(map) { } }; // -------------------- Ref -------------------- template struct evaluator > : public mapbase_evaluator, PlainObjectType> { typedef Ref XprType; enum { Flags = evaluator >::Flags, Alignment = evaluator >::Alignment }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& ref) : mapbase_evaluator(ref) { } }; // -------------------- Block -------------------- template::ret> struct block_evaluator; template struct evaluator > : block_evaluator { typedef Block XprType; typedef typename XprType::Scalar Scalar; // TODO: should check for smaller packet types once we can handle multi-sized packet types typedef typename packet_traits::type PacketScalar; enum { CoeffReadCost = evaluator::CoeffReadCost, RowsAtCompileTime = traits::RowsAtCompileTime, ColsAtCompileTime = traits::ColsAtCompileTime, MaxRowsAtCompileTime = traits::MaxRowsAtCompileTime, MaxColsAtCompileTime = traits::MaxColsAtCompileTime, ArgTypeIsRowMajor = (int(evaluator::Flags)&RowMajorBit) != 0, IsRowMajor = (MaxRowsAtCompileTime==1 && MaxColsAtCompileTime!=1) ? 1 : (MaxColsAtCompileTime==1 && MaxRowsAtCompileTime!=1) ? 0 : ArgTypeIsRowMajor, HasSameStorageOrderAsArgType = (IsRowMajor == ArgTypeIsRowMajor), InnerSize = IsRowMajor ? int(ColsAtCompileTime) : int(RowsAtCompileTime), InnerStrideAtCompileTime = HasSameStorageOrderAsArgType ? int(inner_stride_at_compile_time::ret) : int(outer_stride_at_compile_time::ret), OuterStrideAtCompileTime = HasSameStorageOrderAsArgType ? int(outer_stride_at_compile_time::ret) : int(inner_stride_at_compile_time::ret), MaskPacketAccessBit = (InnerStrideAtCompileTime == 1 || HasSameStorageOrderAsArgType) ? PacketAccessBit : 0, FlagsLinearAccessBit = (RowsAtCompileTime == 1 || ColsAtCompileTime == 1 || (InnerPanel && (evaluator::Flags&LinearAccessBit))) ? LinearAccessBit : 0, FlagsRowMajorBit = XprType::Flags&RowMajorBit, Flags0 = evaluator::Flags & ( (HereditaryBits & ~RowMajorBit) | DirectAccessBit | MaskPacketAccessBit), Flags = Flags0 | FlagsLinearAccessBit | FlagsRowMajorBit, PacketAlignment = unpacket_traits::alignment, Alignment0 = (InnerPanel && (OuterStrideAtCompileTime!=Dynamic) && (OuterStrideAtCompileTime!=0) && (((OuterStrideAtCompileTime * int(sizeof(Scalar))) % int(PacketAlignment)) == 0)) ? int(PacketAlignment) : 0, Alignment = EIGEN_PLAIN_ENUM_MIN(evaluator::Alignment, Alignment0) }; typedef block_evaluator block_evaluator_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& block) : block_evaluator_type(block) { EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } }; // no direct-access => dispatch to a unary evaluator template struct block_evaluator : unary_evaluator > { typedef Block XprType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit block_evaluator(const XprType& block) : unary_evaluator(block) {} }; template struct unary_evaluator, IndexBased> : evaluator_base > { typedef Block XprType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit unary_evaluator(const XprType& block) : m_argImpl(block.nestedExpression()), m_startRow(block.startRow()), m_startCol(block.startCol()), m_linear_offset(ForwardLinearAccess?(ArgType::IsRowMajor ? block.startRow()*block.nestedExpression().cols() + block.startCol() : block.startCol()*block.nestedExpression().rows() + block.startRow()):0) { } typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; enum { RowsAtCompileTime = XprType::RowsAtCompileTime, ForwardLinearAccess = (InnerPanel || int(XprType::IsRowMajor)==int(ArgType::IsRowMajor)) && bool(evaluator::Flags&LinearAccessBit) }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { return m_argImpl.coeff(m_startRow.value() + row, m_startCol.value() + col); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return linear_coeff_impl(index, bool_constant()); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { return m_argImpl.coeffRef(m_startRow.value() + row, m_startCol.value() + col); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return linear_coeffRef_impl(index, bool_constant()); } template EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_argImpl.template packet(m_startRow.value() + row, m_startCol.value() + col); } template EIGEN_STRONG_INLINE PacketType packet(Index index) const { if (ForwardLinearAccess) return m_argImpl.template packet(m_linear_offset.value() + index); else return packet(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0); } template EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { return m_argImpl.template writePacket(m_startRow.value() + row, m_startCol.value() + col, x); } template EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { if (ForwardLinearAccess) return m_argImpl.template writePacket(m_linear_offset.value() + index, x); else return writePacket(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0, x); } protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType linear_coeff_impl(Index index, internal::true_type /* ForwardLinearAccess */) const { return m_argImpl.coeff(m_linear_offset.value() + index); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType linear_coeff_impl(Index index, internal::false_type /* not ForwardLinearAccess */) const { return coeff(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& linear_coeffRef_impl(Index index, internal::true_type /* ForwardLinearAccess */) { return m_argImpl.coeffRef(m_linear_offset.value() + index); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& linear_coeffRef_impl(Index index, internal::false_type /* not ForwardLinearAccess */) { return coeffRef(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0); } evaluator m_argImpl; const variable_if_dynamic m_startRow; const variable_if_dynamic m_startCol; const variable_if_dynamic m_linear_offset; }; // TODO: This evaluator does not actually use the child evaluator; // all action is via the data() as returned by the Block expression. template struct block_evaluator : mapbase_evaluator, typename Block::PlainObject> { typedef Block XprType; typedef typename XprType::Scalar Scalar; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit block_evaluator(const XprType& block) : mapbase_evaluator(block) { // TODO: for the 3.3 release, this should be turned to an internal assertion, but let's keep it as is for the beta lifetime eigen_assert(((internal::UIntPtr(block.data()) % EIGEN_PLAIN_ENUM_MAX(1,evaluator::Alignment)) == 0) && "data is not aligned"); } }; // -------------------- Select -------------------- // NOTE shall we introduce a ternary_evaluator? // TODO enable vectorization for Select template struct evaluator > : evaluator_base > { typedef Select XprType; enum { CoeffReadCost = evaluator::CoeffReadCost + EIGEN_PLAIN_ENUM_MAX(evaluator::CoeffReadCost, evaluator::CoeffReadCost), Flags = (unsigned int)evaluator::Flags & evaluator::Flags & HereditaryBits, Alignment = EIGEN_PLAIN_ENUM_MIN(evaluator::Alignment, evaluator::Alignment) }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& select) : m_conditionImpl(select.conditionMatrix()), m_thenImpl(select.thenMatrix()), m_elseImpl(select.elseMatrix()) { EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } typedef typename XprType::CoeffReturnType CoeffReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { if (m_conditionImpl.coeff(row, col)) return m_thenImpl.coeff(row, col); else return m_elseImpl.coeff(row, col); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { if (m_conditionImpl.coeff(index)) return m_thenImpl.coeff(index); else return m_elseImpl.coeff(index); } protected: evaluator m_conditionImpl; evaluator m_thenImpl; evaluator m_elseImpl; }; // -------------------- Replicate -------------------- template struct unary_evaluator > : evaluator_base > { typedef Replicate XprType; typedef typename XprType::CoeffReturnType CoeffReturnType; enum { Factor = (RowFactor==Dynamic || ColFactor==Dynamic) ? Dynamic : RowFactor*ColFactor }; typedef typename internal::nested_eval::type ArgTypeNested; typedef typename internal::remove_all::type ArgTypeNestedCleaned; enum { CoeffReadCost = evaluator::CoeffReadCost, LinearAccessMask = XprType::IsVectorAtCompileTime ? LinearAccessBit : 0, Flags = (evaluator::Flags & (HereditaryBits|LinearAccessMask) & ~RowMajorBit) | (traits::Flags & RowMajorBit), Alignment = evaluator::Alignment }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit unary_evaluator(const XprType& replicate) : m_arg(replicate.nestedExpression()), m_argImpl(m_arg), m_rows(replicate.nestedExpression().rows()), m_cols(replicate.nestedExpression().cols()) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { // try to avoid using modulo; this is a pure optimization strategy const Index actual_row = internal::traits::RowsAtCompileTime==1 ? 0 : RowFactor==1 ? row : row % m_rows.value(); const Index actual_col = internal::traits::ColsAtCompileTime==1 ? 0 : ColFactor==1 ? col : col % m_cols.value(); return m_argImpl.coeff(actual_row, actual_col); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { // try to avoid using modulo; this is a pure optimization strategy const Index actual_index = internal::traits::RowsAtCompileTime==1 ? (ColFactor==1 ? index : index%m_cols.value()) : (RowFactor==1 ? index : index%m_rows.value()); return m_argImpl.coeff(actual_index); } template EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { const Index actual_row = internal::traits::RowsAtCompileTime==1 ? 0 : RowFactor==1 ? row : row % m_rows.value(); const Index actual_col = internal::traits::ColsAtCompileTime==1 ? 0 : ColFactor==1 ? col : col % m_cols.value(); return m_argImpl.template packet(actual_row, actual_col); } template EIGEN_STRONG_INLINE PacketType packet(Index index) const { const Index actual_index = internal::traits::RowsAtCompileTime==1 ? (ColFactor==1 ? index : index%m_cols.value()) : (RowFactor==1 ? index : index%m_rows.value()); return m_argImpl.template packet(actual_index); } protected: const ArgTypeNested m_arg; evaluator m_argImpl; const variable_if_dynamic m_rows; const variable_if_dynamic m_cols; }; // -------------------- MatrixWrapper and ArrayWrapper -------------------- // // evaluator_wrapper_base is a common base class for the // MatrixWrapper and ArrayWrapper evaluators. template struct evaluator_wrapper_base : evaluator_base { typedef typename remove_all::type ArgType; enum { CoeffReadCost = evaluator::CoeffReadCost, Flags = evaluator::Flags, Alignment = evaluator::Alignment }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator_wrapper_base(const ArgType& arg) : m_argImpl(arg) {} typedef typename ArgType::Scalar Scalar; typedef typename ArgType::CoeffReturnType CoeffReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { return m_argImpl.coeff(row, col); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_argImpl.coeff(index); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { return m_argImpl.coeffRef(row, col); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return m_argImpl.coeffRef(index); } template EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_argImpl.template packet(row, col); } template EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_argImpl.template packet(index); } template EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { m_argImpl.template writePacket(row, col, x); } template EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { m_argImpl.template writePacket(index, x); } protected: evaluator m_argImpl; }; template struct unary_evaluator > : evaluator_wrapper_base > { typedef MatrixWrapper XprType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit unary_evaluator(const XprType& wrapper) : evaluator_wrapper_base >(wrapper.nestedExpression()) { } }; template struct unary_evaluator > : evaluator_wrapper_base > { typedef ArrayWrapper XprType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit unary_evaluator(const XprType& wrapper) : evaluator_wrapper_base >(wrapper.nestedExpression()) { } }; // -------------------- Reverse -------------------- // defined in Reverse.h: template struct reverse_packet_cond; template struct unary_evaluator > : evaluator_base > { typedef Reverse XprType; typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; enum { IsRowMajor = XprType::IsRowMajor, IsColMajor = !IsRowMajor, ReverseRow = (Direction == Vertical) || (Direction == BothDirections), ReverseCol = (Direction == Horizontal) || (Direction == BothDirections), ReversePacket = (Direction == BothDirections) || ((Direction == Vertical) && IsColMajor) || ((Direction == Horizontal) && IsRowMajor), CoeffReadCost = evaluator::CoeffReadCost, // let's enable LinearAccess only with vectorization because of the product overhead // FIXME enable DirectAccess with negative strides? Flags0 = evaluator::Flags, LinearAccess = ( (Direction==BothDirections) && (int(Flags0)&PacketAccessBit) ) || ((ReverseRow && XprType::ColsAtCompileTime==1) || (ReverseCol && XprType::RowsAtCompileTime==1)) ? LinearAccessBit : 0, Flags = int(Flags0) & (HereditaryBits | PacketAccessBit | LinearAccess), Alignment = 0 // FIXME in some rare cases, Alignment could be preserved, like a Vector4f. }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit unary_evaluator(const XprType& reverse) : m_argImpl(reverse.nestedExpression()), m_rows(ReverseRow ? reverse.nestedExpression().rows() : 1), m_cols(ReverseCol ? reverse.nestedExpression().cols() : 1) { } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { return m_argImpl.coeff(ReverseRow ? m_rows.value() - row - 1 : row, ReverseCol ? m_cols.value() - col - 1 : col); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_argImpl.coeff(m_rows.value() * m_cols.value() - index - 1); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { return m_argImpl.coeffRef(ReverseRow ? m_rows.value() - row - 1 : row, ReverseCol ? m_cols.value() - col - 1 : col); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return m_argImpl.coeffRef(m_rows.value() * m_cols.value() - index - 1); } template EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { enum { PacketSize = unpacket_traits::size, OffsetRow = ReverseRow && IsColMajor ? PacketSize : 1, OffsetCol = ReverseCol && IsRowMajor ? PacketSize : 1 }; typedef internal::reverse_packet_cond reverse_packet; return reverse_packet::run(m_argImpl.template packet( ReverseRow ? m_rows.value() - row - OffsetRow : row, ReverseCol ? m_cols.value() - col - OffsetCol : col)); } template EIGEN_STRONG_INLINE PacketType packet(Index index) const { enum { PacketSize = unpacket_traits::size }; return preverse(m_argImpl.template packet(m_rows.value() * m_cols.value() - index - PacketSize)); } template EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { // FIXME we could factorize some code with packet(i,j) enum { PacketSize = unpacket_traits::size, OffsetRow = ReverseRow && IsColMajor ? PacketSize : 1, OffsetCol = ReverseCol && IsRowMajor ? PacketSize : 1 }; typedef internal::reverse_packet_cond reverse_packet; m_argImpl.template writePacket( ReverseRow ? m_rows.value() - row - OffsetRow : row, ReverseCol ? m_cols.value() - col - OffsetCol : col, reverse_packet::run(x)); } template EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { enum { PacketSize = unpacket_traits::size }; m_argImpl.template writePacket (m_rows.value() * m_cols.value() - index - PacketSize, preverse(x)); } protected: evaluator m_argImpl; // If we do not reverse rows, then we do not need to know the number of rows; same for columns // Nonetheless, in this case it is important to set to 1 such that the coeff(index) method works fine for vectors. const variable_if_dynamic m_rows; const variable_if_dynamic m_cols; }; // -------------------- Diagonal -------------------- template struct evaluator > : evaluator_base > { typedef Diagonal XprType; enum { CoeffReadCost = evaluator::CoeffReadCost, Flags = (unsigned int)(evaluator::Flags & (HereditaryBits | DirectAccessBit) & ~RowMajorBit) | LinearAccessBit, Alignment = 0 }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& diagonal) : m_argImpl(diagonal.nestedExpression()), m_index(diagonal.index()) { } typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index) const { return m_argImpl.coeff(row + rowOffset(), row + colOffset()); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_argImpl.coeff(index + rowOffset(), index + colOffset()); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index) { return m_argImpl.coeffRef(row + rowOffset(), row + colOffset()); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return m_argImpl.coeffRef(index + rowOffset(), index + colOffset()); } protected: evaluator m_argImpl; const internal::variable_if_dynamicindex m_index; private: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rowOffset() const { return m_index.value() > 0 ? 0 : -m_index.value(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index colOffset() const { return m_index.value() > 0 ? m_index.value() : 0; } }; //---------------------------------------------------------------------- // deprecated code //---------------------------------------------------------------------- // -------------------- EvalToTemp -------------------- // expression class for evaluating nested expression to a temporary template class EvalToTemp; template struct traits > : public traits { }; template class EvalToTemp : public dense_xpr_base >::type { public: typedef typename dense_xpr_base::type Base; EIGEN_GENERIC_PUBLIC_INTERFACE(EvalToTemp) explicit EvalToTemp(const ArgType& arg) : m_arg(arg) { } const ArgType& arg() const { return m_arg; } EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_arg.rows(); } EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_arg.cols(); } private: const ArgType& m_arg; }; template struct evaluator > : public evaluator { typedef EvalToTemp XprType; typedef typename ArgType::PlainObject PlainObject; typedef evaluator Base; EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) : m_result(xpr.arg()) { ::new (static_cast(this)) Base(m_result); } // This constructor is used when nesting an EvalTo evaluator in another evaluator EIGEN_DEVICE_FUNC evaluator(const ArgType& arg) : m_result(arg) { ::new (static_cast(this)) Base(m_result); } protected: PlainObject m_result; }; } // namespace internal } // end namespace Eigen #endif // EIGEN_COREEVALUATORS_H RcppEigen/inst/include/Eigen/src/Core/GeneralProduct.h0000644000176200001440000005225714567757725022367 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2006-2008 Benoit Jacob // Copyright (C) 2008-2011 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_GENERAL_PRODUCT_H #define EIGEN_GENERAL_PRODUCT_H namespace Eigen { enum { Large = 2, Small = 3 }; // Define the threshold value to fallback from the generic matrix-matrix product // implementation (heavy) to the lightweight coeff-based product one. // See generic_product_impl // in products/GeneralMatrixMatrix.h for more details. // TODO This threshold should also be used in the compile-time selector below. #ifndef EIGEN_GEMM_TO_COEFFBASED_THRESHOLD // This default value has been obtained on a Haswell architecture. #define EIGEN_GEMM_TO_COEFFBASED_THRESHOLD 20 #endif namespace internal { template struct product_type_selector; template struct product_size_category { enum { #ifndef EIGEN_GPU_COMPILE_PHASE is_large = MaxSize == Dynamic || Size >= EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD || (Size==Dynamic && MaxSize>=EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD), #else is_large = 0, #endif value = is_large ? Large : Size == 1 ? 1 : Small }; }; template struct product_type { typedef typename remove_all::type _Lhs; typedef typename remove_all::type _Rhs; enum { MaxRows = traits<_Lhs>::MaxRowsAtCompileTime, Rows = traits<_Lhs>::RowsAtCompileTime, MaxCols = traits<_Rhs>::MaxColsAtCompileTime, Cols = traits<_Rhs>::ColsAtCompileTime, MaxDepth = EIGEN_SIZE_MIN_PREFER_FIXED(traits<_Lhs>::MaxColsAtCompileTime, traits<_Rhs>::MaxRowsAtCompileTime), Depth = EIGEN_SIZE_MIN_PREFER_FIXED(traits<_Lhs>::ColsAtCompileTime, traits<_Rhs>::RowsAtCompileTime) }; // the splitting into different lines of code here, introducing the _select enums and the typedef below, // is to work around an internal compiler error with gcc 4.1 and 4.2. private: enum { rows_select = product_size_category::value, cols_select = product_size_category::value, depth_select = product_size_category::value }; typedef product_type_selector selector; public: enum { value = selector::ret, ret = selector::ret }; #ifdef EIGEN_DEBUG_PRODUCT static void debug() { EIGEN_DEBUG_VAR(Rows); EIGEN_DEBUG_VAR(Cols); EIGEN_DEBUG_VAR(Depth); EIGEN_DEBUG_VAR(rows_select); EIGEN_DEBUG_VAR(cols_select); EIGEN_DEBUG_VAR(depth_select); EIGEN_DEBUG_VAR(value); } #endif }; /* The following allows to select the kind of product at compile time * based on the three dimensions of the product. * This is a compile time mapping from {1,Small,Large}^3 -> {product types} */ // FIXME I'm not sure the current mapping is the ideal one. template struct product_type_selector { enum { ret = OuterProduct }; }; template struct product_type_selector { enum { ret = LazyCoeffBasedProductMode }; }; template struct product_type_selector<1, N, 1> { enum { ret = LazyCoeffBasedProductMode }; }; template struct product_type_selector<1, 1, Depth> { enum { ret = InnerProduct }; }; template<> struct product_type_selector<1, 1, 1> { enum { ret = InnerProduct }; }; template<> struct product_type_selector { enum { ret = CoeffBasedProductMode }; }; template<> struct product_type_selector<1, Small,Small> { enum { ret = CoeffBasedProductMode }; }; template<> struct product_type_selector { enum { ret = CoeffBasedProductMode }; }; template<> struct product_type_selector { enum { ret = LazyCoeffBasedProductMode }; }; template<> struct product_type_selector { enum { ret = LazyCoeffBasedProductMode }; }; template<> struct product_type_selector { enum { ret = LazyCoeffBasedProductMode }; }; template<> struct product_type_selector<1, Large,Small> { enum { ret = CoeffBasedProductMode }; }; template<> struct product_type_selector<1, Large,Large> { enum { ret = GemvProduct }; }; template<> struct product_type_selector<1, Small,Large> { enum { ret = CoeffBasedProductMode }; }; template<> struct product_type_selector { enum { ret = CoeffBasedProductMode }; }; template<> struct product_type_selector { enum { ret = GemvProduct }; }; template<> struct product_type_selector { enum { ret = CoeffBasedProductMode }; }; template<> struct product_type_selector { enum { ret = GemmProduct }; }; template<> struct product_type_selector { enum { ret = GemmProduct }; }; template<> struct product_type_selector { enum { ret = GemmProduct }; }; template<> struct product_type_selector { enum { ret = GemmProduct }; }; template<> struct product_type_selector { enum { ret = CoeffBasedProductMode }; }; template<> struct product_type_selector { enum { ret = CoeffBasedProductMode }; }; template<> struct product_type_selector { enum { ret = GemmProduct }; }; } // end namespace internal /*********************************************************************** * Implementation of Inner Vector Vector Product ***********************************************************************/ // FIXME : maybe the "inner product" could return a Scalar // instead of a 1x1 matrix ?? // Pro: more natural for the user // Cons: this could be a problem if in a meta unrolled algorithm a matrix-matrix // product ends up to a row-vector times col-vector product... To tackle this use // case, we could have a specialization for Block with: operator=(Scalar x); /*********************************************************************** * Implementation of Outer Vector Vector Product ***********************************************************************/ /*********************************************************************** * Implementation of General Matrix Vector Product ***********************************************************************/ /* According to the shape/flags of the matrix we have to distinghish 3 different cases: * 1 - the matrix is col-major, BLAS compatible and M is large => call fast BLAS-like colmajor routine * 2 - the matrix is row-major, BLAS compatible and N is large => call fast BLAS-like rowmajor routine * 3 - all other cases are handled using a simple loop along the outer-storage direction. * Therefore we need a lower level meta selector. * Furthermore, if the matrix is the rhs, then the product has to be transposed. */ namespace internal { template struct gemv_dense_selector; } // end namespace internal namespace internal { template struct gemv_static_vector_if; template struct gemv_static_vector_if { EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Scalar* data() { eigen_internal_assert(false && "should never be called"); return 0; } }; template struct gemv_static_vector_if { EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Scalar* data() { return 0; } }; template struct gemv_static_vector_if { enum { ForceAlignment = internal::packet_traits::Vectorizable, PacketSize = internal::packet_traits::size }; #if EIGEN_MAX_STATIC_ALIGN_BYTES!=0 internal::plain_array m_data; EIGEN_STRONG_INLINE Scalar* data() { return m_data.array; } #else // Some architectures cannot align on the stack, // => let's manually enforce alignment by allocating more data and return the address of the first aligned element. internal::plain_array m_data; EIGEN_STRONG_INLINE Scalar* data() { return ForceAlignment ? reinterpret_cast((internal::UIntPtr(m_data.array) & ~(std::size_t(EIGEN_MAX_ALIGN_BYTES-1))) + EIGEN_MAX_ALIGN_BYTES) : m_data.array; } #endif }; // The vector is on the left => transposition template struct gemv_dense_selector { template static void run(const Lhs &lhs, const Rhs &rhs, Dest& dest, const typename Dest::Scalar& alpha) { Transpose destT(dest); enum { OtherStorageOrder = StorageOrder == RowMajor ? ColMajor : RowMajor }; gemv_dense_selector ::run(rhs.transpose(), lhs.transpose(), destT, alpha); } }; template<> struct gemv_dense_selector { template static inline void run(const Lhs &lhs, const Rhs &rhs, Dest& dest, const typename Dest::Scalar& alpha) { typedef typename Lhs::Scalar LhsScalar; typedef typename Rhs::Scalar RhsScalar; typedef typename Dest::Scalar ResScalar; typedef typename Dest::RealScalar RealScalar; typedef internal::blas_traits LhsBlasTraits; typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; typedef internal::blas_traits RhsBlasTraits; typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; typedef Map, EIGEN_PLAIN_ENUM_MIN(AlignedMax,internal::packet_traits::size)> MappedDest; ActualLhsType actualLhs = LhsBlasTraits::extract(lhs); ActualRhsType actualRhs = RhsBlasTraits::extract(rhs); ResScalar actualAlpha = combine_scalar_factors(alpha, lhs, rhs); // make sure Dest is a compile-time vector type (bug 1166) typedef typename conditional::type ActualDest; enum { // FIXME find a way to allow an inner stride on the result if packet_traits::size==1 // on, the other hand it is good for the cache to pack the vector anyways... EvalToDestAtCompileTime = (ActualDest::InnerStrideAtCompileTime==1), ComplexByReal = (NumTraits::IsComplex) && (!NumTraits::IsComplex), MightCannotUseDest = ((!EvalToDestAtCompileTime) || ComplexByReal) && (ActualDest::MaxSizeAtCompileTime!=0) }; typedef const_blas_data_mapper LhsMapper; typedef const_blas_data_mapper RhsMapper; RhsScalar compatibleAlpha = get_factor::run(actualAlpha); if(!MightCannotUseDest) { // shortcut if we are sure to be able to use dest directly, // this ease the compiler to generate cleaner and more optimzized code for most common cases general_matrix_vector_product ::run( actualLhs.rows(), actualLhs.cols(), LhsMapper(actualLhs.data(), actualLhs.outerStride()), RhsMapper(actualRhs.data(), actualRhs.innerStride()), dest.data(), 1, compatibleAlpha); } else { gemv_static_vector_if static_dest; const bool alphaIsCompatible = (!ComplexByReal) || (numext::imag(actualAlpha)==RealScalar(0)); const bool evalToDest = EvalToDestAtCompileTime && alphaIsCompatible; ei_declare_aligned_stack_constructed_variable(ResScalar,actualDestPtr,dest.size(), evalToDest ? dest.data() : static_dest.data()); if(!evalToDest) { #ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN Index size = dest.size(); EIGEN_DENSE_STORAGE_CTOR_PLUGIN #endif if(!alphaIsCompatible) { MappedDest(actualDestPtr, dest.size()).setZero(); compatibleAlpha = RhsScalar(1); } else MappedDest(actualDestPtr, dest.size()) = dest; } general_matrix_vector_product ::run( actualLhs.rows(), actualLhs.cols(), LhsMapper(actualLhs.data(), actualLhs.outerStride()), RhsMapper(actualRhs.data(), actualRhs.innerStride()), actualDestPtr, 1, compatibleAlpha); if (!evalToDest) { if(!alphaIsCompatible) dest.matrix() += actualAlpha * MappedDest(actualDestPtr, dest.size()); else dest = MappedDest(actualDestPtr, dest.size()); } } } }; template<> struct gemv_dense_selector { template static void run(const Lhs &lhs, const Rhs &rhs, Dest& dest, const typename Dest::Scalar& alpha) { typedef typename Lhs::Scalar LhsScalar; typedef typename Rhs::Scalar RhsScalar; typedef typename Dest::Scalar ResScalar; typedef internal::blas_traits LhsBlasTraits; typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; typedef internal::blas_traits RhsBlasTraits; typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; typedef typename internal::remove_all::type ActualRhsTypeCleaned; typename add_const::type actualLhs = LhsBlasTraits::extract(lhs); typename add_const::type actualRhs = RhsBlasTraits::extract(rhs); ResScalar actualAlpha = combine_scalar_factors(alpha, lhs, rhs); enum { // FIXME find a way to allow an inner stride on the result if packet_traits::size==1 // on, the other hand it is good for the cache to pack the vector anyways... DirectlyUseRhs = ActualRhsTypeCleaned::InnerStrideAtCompileTime==1 || ActualRhsTypeCleaned::MaxSizeAtCompileTime==0 }; gemv_static_vector_if static_rhs; ei_declare_aligned_stack_constructed_variable(RhsScalar,actualRhsPtr,actualRhs.size(), DirectlyUseRhs ? const_cast(actualRhs.data()) : static_rhs.data()); if(!DirectlyUseRhs) { #ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN Index size = actualRhs.size(); EIGEN_DENSE_STORAGE_CTOR_PLUGIN #endif Map(actualRhsPtr, actualRhs.size()) = actualRhs; } typedef const_blas_data_mapper LhsMapper; typedef const_blas_data_mapper RhsMapper; general_matrix_vector_product ::run( actualLhs.rows(), actualLhs.cols(), LhsMapper(actualLhs.data(), actualLhs.outerStride()), RhsMapper(actualRhsPtr, 1), dest.data(), dest.col(0).innerStride(), //NOTE if dest is not a vector at compile-time, then dest.innerStride() might be wrong. (bug 1166) actualAlpha); } }; template<> struct gemv_dense_selector { template static void run(const Lhs &lhs, const Rhs &rhs, Dest& dest, const typename Dest::Scalar& alpha) { EIGEN_STATIC_ASSERT((!nested_eval::Evaluate),EIGEN_INTERNAL_COMPILATION_ERROR_OR_YOU_MADE_A_PROGRAMMING_MISTAKE); // TODO if rhs is large enough it might be beneficial to make sure that dest is sequentially stored in memory, otherwise use a temp typename nested_eval::type actual_rhs(rhs); const Index size = rhs.rows(); for(Index k=0; k struct gemv_dense_selector { template static void run(const Lhs &lhs, const Rhs &rhs, Dest& dest, const typename Dest::Scalar& alpha) { EIGEN_STATIC_ASSERT((!nested_eval::Evaluate),EIGEN_INTERNAL_COMPILATION_ERROR_OR_YOU_MADE_A_PROGRAMMING_MISTAKE); typename nested_eval::type actual_rhs(rhs); const Index rows = dest.rows(); for(Index i=0; i template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Product MatrixBase::operator*(const MatrixBase &other) const { // A note regarding the function declaration: In MSVC, this function will sometimes // not be inlined since DenseStorage is an unwindable object for dynamic // matrices and product types are holding a member to store the result. // Thus it does not help tagging this function with EIGEN_STRONG_INLINE. enum { ProductIsValid = Derived::ColsAtCompileTime==Dynamic || OtherDerived::RowsAtCompileTime==Dynamic || int(Derived::ColsAtCompileTime)==int(OtherDerived::RowsAtCompileTime), AreVectors = Derived::IsVectorAtCompileTime && OtherDerived::IsVectorAtCompileTime, SameSizes = EIGEN_PREDICATE_SAME_MATRIX_SIZE(Derived,OtherDerived) }; // note to the lost user: // * for a dot product use: v1.dot(v2) // * for a coeff-wise product use: v1.cwiseProduct(v2) EIGEN_STATIC_ASSERT(ProductIsValid || !(AreVectors && SameSizes), INVALID_VECTOR_VECTOR_PRODUCT__IF_YOU_WANTED_A_DOT_OR_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTIONS) EIGEN_STATIC_ASSERT(ProductIsValid || !(SameSizes && !AreVectors), INVALID_MATRIX_PRODUCT__IF_YOU_WANTED_A_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTION) EIGEN_STATIC_ASSERT(ProductIsValid || SameSizes, INVALID_MATRIX_PRODUCT) #ifdef EIGEN_DEBUG_PRODUCT internal::product_type::debug(); #endif return Product(derived(), other.derived()); } /** \returns an expression of the matrix product of \c *this and \a other without implicit evaluation. * * The returned product will behave like any other expressions: the coefficients of the product will be * computed once at a time as requested. This might be useful in some extremely rare cases when only * a small and no coherent fraction of the result's coefficients have to be computed. * * \warning This version of the matrix product can be much much slower. So use it only if you know * what you are doing and that you measured a true speed improvement. * * \sa operator*(const MatrixBase&) */ template template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Product MatrixBase::lazyProduct(const MatrixBase &other) const { enum { ProductIsValid = Derived::ColsAtCompileTime==Dynamic || OtherDerived::RowsAtCompileTime==Dynamic || int(Derived::ColsAtCompileTime)==int(OtherDerived::RowsAtCompileTime), AreVectors = Derived::IsVectorAtCompileTime && OtherDerived::IsVectorAtCompileTime, SameSizes = EIGEN_PREDICATE_SAME_MATRIX_SIZE(Derived,OtherDerived) }; // note to the lost user: // * for a dot product use: v1.dot(v2) // * for a coeff-wise product use: v1.cwiseProduct(v2) EIGEN_STATIC_ASSERT(ProductIsValid || !(AreVectors && SameSizes), INVALID_VECTOR_VECTOR_PRODUCT__IF_YOU_WANTED_A_DOT_OR_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTIONS) EIGEN_STATIC_ASSERT(ProductIsValid || !(SameSizes && !AreVectors), INVALID_MATRIX_PRODUCT__IF_YOU_WANTED_A_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTION) EIGEN_STATIC_ASSERT(ProductIsValid || SameSizes, INVALID_MATRIX_PRODUCT) return Product(derived(), other.derived()); } } // end namespace Eigen #endif // EIGEN_PRODUCT_H RcppEigen/inst/include/Eigen/src/Core/SelfCwiseBinaryOp.h0000644000176200001440000000324114107270226022736 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2009-2010 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_SELFCWISEBINARYOP_H #define EIGEN_SELFCWISEBINARYOP_H namespace Eigen { // TODO generalize the scalar type of 'other' template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase::operator*=(const Scalar& other) { internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::mul_assign_op()); return derived(); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& ArrayBase::operator+=(const Scalar& other) { internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::add_assign_op()); return derived(); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& ArrayBase::operator-=(const Scalar& other) { internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::sub_assign_op()); return derived(); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase::operator/=(const Scalar& other) { internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::div_assign_op()); return derived(); } } // end namespace Eigen #endif // EIGEN_SELFCWISEBINARYOP_H RcppEigen/inst/include/Eigen/src/Core/Replicate.h0000644000176200001440000001303014567757725021343 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2009-2010 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_REPLICATE_H #define EIGEN_REPLICATE_H namespace Eigen { namespace internal { template struct traits > : traits { typedef typename MatrixType::Scalar Scalar; typedef typename traits::StorageKind StorageKind; typedef typename traits::XprKind XprKind; typedef typename ref_selector::type MatrixTypeNested; typedef typename remove_reference::type _MatrixTypeNested; enum { RowsAtCompileTime = RowFactor==Dynamic || int(MatrixType::RowsAtCompileTime)==Dynamic ? Dynamic : RowFactor * MatrixType::RowsAtCompileTime, ColsAtCompileTime = ColFactor==Dynamic || int(MatrixType::ColsAtCompileTime)==Dynamic ? Dynamic : ColFactor * MatrixType::ColsAtCompileTime, //FIXME we don't propagate the max sizes !!! MaxRowsAtCompileTime = RowsAtCompileTime, MaxColsAtCompileTime = ColsAtCompileTime, IsRowMajor = MaxRowsAtCompileTime==1 && MaxColsAtCompileTime!=1 ? 1 : MaxColsAtCompileTime==1 && MaxRowsAtCompileTime!=1 ? 0 : (MatrixType::Flags & RowMajorBit) ? 1 : 0, // FIXME enable DirectAccess with negative strides? Flags = IsRowMajor ? RowMajorBit : 0 }; }; } /** * \class Replicate * \ingroup Core_Module * * \brief Expression of the multiple replication of a matrix or vector * * \tparam MatrixType the type of the object we are replicating * \tparam RowFactor number of repetitions at compile time along the vertical direction, can be Dynamic. * \tparam ColFactor number of repetitions at compile time along the horizontal direction, can be Dynamic. * * This class represents an expression of the multiple replication of a matrix or vector. * It is the return type of DenseBase::replicate() and most of the time * this is the only way it is used. * * \sa DenseBase::replicate() */ template class Replicate : public internal::dense_xpr_base< Replicate >::type { typedef typename internal::traits::MatrixTypeNested MatrixTypeNested; typedef typename internal::traits::_MatrixTypeNested _MatrixTypeNested; public: typedef typename internal::dense_xpr_base::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(Replicate) typedef typename internal::remove_all::type NestedExpression; template EIGEN_DEVICE_FUNC inline explicit Replicate(const OriginalMatrixType& matrix) : m_matrix(matrix), m_rowFactor(RowFactor), m_colFactor(ColFactor) { EIGEN_STATIC_ASSERT((internal::is_same::type,OriginalMatrixType>::value), THE_MATRIX_OR_EXPRESSION_THAT_YOU_PASSED_DOES_NOT_HAVE_THE_EXPECTED_TYPE) eigen_assert(RowFactor!=Dynamic && ColFactor!=Dynamic); } template EIGEN_DEVICE_FUNC inline Replicate(const OriginalMatrixType& matrix, Index rowFactor, Index colFactor) : m_matrix(matrix), m_rowFactor(rowFactor), m_colFactor(colFactor) { EIGEN_STATIC_ASSERT((internal::is_same::type,OriginalMatrixType>::value), THE_MATRIX_OR_EXPRESSION_THAT_YOU_PASSED_DOES_NOT_HAVE_THE_EXPECTED_TYPE) } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const { return m_matrix.rows() * m_rowFactor.value(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const { return m_matrix.cols() * m_colFactor.value(); } EIGEN_DEVICE_FUNC const _MatrixTypeNested& nestedExpression() const { return m_matrix; } protected: MatrixTypeNested m_matrix; const internal::variable_if_dynamic m_rowFactor; const internal::variable_if_dynamic m_colFactor; }; /** * \return an expression of the replication of \c *this * * Example: \include MatrixBase_replicate.cpp * Output: \verbinclude MatrixBase_replicate.out * * \sa VectorwiseOp::replicate(), DenseBase::replicate(Index,Index), class Replicate */ template template EIGEN_DEVICE_FUNC const Replicate DenseBase::replicate() const { return Replicate(derived()); } /** * \return an expression of the replication of each column (or row) of \c *this * * Example: \include DirectionWise_replicate_int.cpp * Output: \verbinclude DirectionWise_replicate_int.out * * \sa VectorwiseOp::replicate(), DenseBase::replicate(), class Replicate */ template EIGEN_DEVICE_FUNC const typename VectorwiseOp::ReplicateReturnType VectorwiseOp::replicate(Index factor) const { return typename VectorwiseOp::ReplicateReturnType (_expression(),Direction==Vertical?factor:1,Direction==Horizontal?factor:1); } } // end namespace Eigen #endif // EIGEN_REPLICATE_H RcppEigen/inst/include/Eigen/src/Core/GlobalFunctions.h0000644000176200001440000002642714567757725022542 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2010-2016 Gael Guennebaud // Copyright (C) 2010 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_GLOBAL_FUNCTIONS_H #define EIGEN_GLOBAL_FUNCTIONS_H #ifdef EIGEN_PARSED_BY_DOXYGEN #define EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(NAME,FUNCTOR,DOC_OP,DOC_DETAILS) \ /** \returns an expression of the coefficient-wise DOC_OP of \a x DOC_DETAILS \sa Math functions, class CwiseUnaryOp */ \ template \ inline const Eigen::CwiseUnaryOp, const Derived> \ NAME(const Eigen::ArrayBase& x); #else #define EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(NAME,FUNCTOR,DOC_OP,DOC_DETAILS) \ template \ inline const Eigen::CwiseUnaryOp, const Derived> \ (NAME)(const Eigen::ArrayBase& x) { \ return Eigen::CwiseUnaryOp, const Derived>(x.derived()); \ } #endif // EIGEN_PARSED_BY_DOXYGEN #define EIGEN_ARRAY_DECLARE_GLOBAL_EIGEN_UNARY(NAME,FUNCTOR) \ \ template \ struct NAME##_retval > \ { \ typedef const Eigen::CwiseUnaryOp, const Derived> type; \ }; \ template \ struct NAME##_impl > \ { \ static inline typename NAME##_retval >::type run(const Eigen::ArrayBase& x) \ { \ return typename NAME##_retval >::type(x.derived()); \ } \ }; namespace Eigen { EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(real,scalar_real_op,real part,\sa ArrayBase::real) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(imag,scalar_imag_op,imaginary part,\sa ArrayBase::imag) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(conj,scalar_conjugate_op,complex conjugate,\sa ArrayBase::conjugate) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(inverse,scalar_inverse_op,inverse,\sa ArrayBase::inverse) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sin,scalar_sin_op,sine,\sa ArrayBase::sin) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(cos,scalar_cos_op,cosine,\sa ArrayBase::cos) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(tan,scalar_tan_op,tangent,\sa ArrayBase::tan) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(atan,scalar_atan_op,arc-tangent,\sa ArrayBase::atan) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(asin,scalar_asin_op,arc-sine,\sa ArrayBase::asin) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(acos,scalar_acos_op,arc-consine,\sa ArrayBase::acos) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sinh,scalar_sinh_op,hyperbolic sine,\sa ArrayBase::sinh) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(cosh,scalar_cosh_op,hyperbolic cosine,\sa ArrayBase::cosh) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(tanh,scalar_tanh_op,hyperbolic tangent,\sa ArrayBase::tanh) #if EIGEN_HAS_CXX11_MATH EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(asinh,scalar_asinh_op,inverse hyperbolic sine,\sa ArrayBase::asinh) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(acosh,scalar_acosh_op,inverse hyperbolic cosine,\sa ArrayBase::acosh) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(atanh,scalar_atanh_op,inverse hyperbolic tangent,\sa ArrayBase::atanh) #endif EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(logistic,scalar_logistic_op,logistic function,\sa ArrayBase::logistic) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(lgamma,scalar_lgamma_op,natural logarithm of the gamma function,\sa ArrayBase::lgamma) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(digamma,scalar_digamma_op,derivative of lgamma,\sa ArrayBase::digamma) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(erf,scalar_erf_op,error function,\sa ArrayBase::erf) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(erfc,scalar_erfc_op,complement error function,\sa ArrayBase::erfc) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(ndtri,scalar_ndtri_op,inverse normal distribution function,\sa ArrayBase::ndtri) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(exp,scalar_exp_op,exponential,\sa ArrayBase::exp) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(expm1,scalar_expm1_op,exponential of a value minus 1,\sa ArrayBase::expm1) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(log,scalar_log_op,natural logarithm,\sa Eigen::log10 DOXCOMMA ArrayBase::log) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(log1p,scalar_log1p_op,natural logarithm of 1 plus the value,\sa ArrayBase::log1p) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(log10,scalar_log10_op,base 10 logarithm,\sa Eigen::log DOXCOMMA ArrayBase::log10) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(log2,scalar_log2_op,base 2 logarithm,\sa Eigen::log DOXCOMMA ArrayBase::log2) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(abs,scalar_abs_op,absolute value,\sa ArrayBase::abs DOXCOMMA MatrixBase::cwiseAbs) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(abs2,scalar_abs2_op,squared absolute value,\sa ArrayBase::abs2 DOXCOMMA MatrixBase::cwiseAbs2) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(arg,scalar_arg_op,complex argument,\sa ArrayBase::arg DOXCOMMA MatrixBase::cwiseArg) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sqrt,scalar_sqrt_op,square root,\sa ArrayBase::sqrt DOXCOMMA MatrixBase::cwiseSqrt) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(rsqrt,scalar_rsqrt_op,reciprocal square root,\sa ArrayBase::rsqrt) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(square,scalar_square_op,square (power 2),\sa Eigen::abs2 DOXCOMMA Eigen::pow DOXCOMMA ArrayBase::square) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(cube,scalar_cube_op,cube (power 3),\sa Eigen::pow DOXCOMMA ArrayBase::cube) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(rint,scalar_rint_op,nearest integer,\sa Eigen::floor DOXCOMMA Eigen::ceil DOXCOMMA ArrayBase::round) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(round,scalar_round_op,nearest integer,\sa Eigen::floor DOXCOMMA Eigen::ceil DOXCOMMA ArrayBase::round) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(floor,scalar_floor_op,nearest integer not greater than the giben value,\sa Eigen::ceil DOXCOMMA ArrayBase::floor) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(ceil,scalar_ceil_op,nearest integer not less than the giben value,\sa Eigen::floor DOXCOMMA ArrayBase::ceil) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(isnan,scalar_isnan_op,not-a-number test,\sa Eigen::isinf DOXCOMMA Eigen::isfinite DOXCOMMA ArrayBase::isnan) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(isinf,scalar_isinf_op,infinite value test,\sa Eigen::isnan DOXCOMMA Eigen::isfinite DOXCOMMA ArrayBase::isinf) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(isfinite,scalar_isfinite_op,finite value test,\sa Eigen::isinf DOXCOMMA Eigen::isnan DOXCOMMA ArrayBase::isfinite) EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sign,scalar_sign_op,sign (or 0),\sa ArrayBase::sign) /** \returns an expression of the coefficient-wise power of \a x to the given constant \a exponent. * * \tparam ScalarExponent is the scalar type of \a exponent. It must be compatible with the scalar type of the given expression (\c Derived::Scalar). * * \sa ArrayBase::pow() * * \relates ArrayBase */ #ifdef EIGEN_PARSED_BY_DOXYGEN template inline const CwiseBinaryOp,Derived,Constant > pow(const Eigen::ArrayBase& x, const ScalarExponent& exponent); #else template EIGEN_DEVICE_FUNC inline EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE( const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(Derived,typename internal::promote_scalar_arg::type,pow)) pow(const Eigen::ArrayBase& x, const ScalarExponent& exponent) { typedef typename internal::promote_scalar_arg::type PromotedExponent; return EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(Derived,PromotedExponent,pow)(x.derived(), typename internal::plain_constant_type::type(x.derived().rows(), x.derived().cols(), internal::scalar_constant_op(exponent))); } #endif /** \returns an expression of the coefficient-wise power of \a x to the given array of \a exponents. * * This function computes the coefficient-wise power. * * Example: \include Cwise_array_power_array.cpp * Output: \verbinclude Cwise_array_power_array.out * * \sa ArrayBase::pow() * * \relates ArrayBase */ template inline const Eigen::CwiseBinaryOp, const Derived, const ExponentDerived> pow(const Eigen::ArrayBase& x, const Eigen::ArrayBase& exponents) { return Eigen::CwiseBinaryOp, const Derived, const ExponentDerived>( x.derived(), exponents.derived() ); } /** \returns an expression of the coefficient-wise power of the scalar \a x to the given array of \a exponents. * * This function computes the coefficient-wise power between a scalar and an array of exponents. * * \tparam Scalar is the scalar type of \a x. It must be compatible with the scalar type of the given array expression (\c Derived::Scalar). * * Example: \include Cwise_scalar_power_array.cpp * Output: \verbinclude Cwise_scalar_power_array.out * * \sa ArrayBase::pow() * * \relates ArrayBase */ #ifdef EIGEN_PARSED_BY_DOXYGEN template inline const CwiseBinaryOp,Constant,Derived> pow(const Scalar& x,const Eigen::ArrayBase& x); #else template EIGEN_DEVICE_FUNC inline EIGEN_MSVC10_WORKAROUND_BINARYOP_RETURN_TYPE( const EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(typename internal::promote_scalar_arg::type,Derived,pow)) pow(const Scalar& x, const Eigen::ArrayBase& exponents) { typedef typename internal::promote_scalar_arg::type PromotedScalar; return EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(PromotedScalar,Derived,pow)( typename internal::plain_constant_type::type(exponents.derived().rows(), exponents.derived().cols(), internal::scalar_constant_op(x)), exponents.derived()); } #endif namespace internal { EIGEN_ARRAY_DECLARE_GLOBAL_EIGEN_UNARY(real,scalar_real_op) EIGEN_ARRAY_DECLARE_GLOBAL_EIGEN_UNARY(imag,scalar_imag_op) EIGEN_ARRAY_DECLARE_GLOBAL_EIGEN_UNARY(abs2,scalar_abs2_op) } } // TODO: cleanly disable those functions that are not supported on Array (numext::real_ref, internal::random, internal::isApprox...) #endif // EIGEN_GLOBAL_FUNCTIONS_H RcppEigen/inst/include/Eigen/src/Core/Diagonal.h0000644000176200001440000002321614567757725021160 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2007-2009 Benoit Jacob // Copyright (C) 2009-2010 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_DIAGONAL_H #define EIGEN_DIAGONAL_H namespace Eigen { /** \class Diagonal * \ingroup Core_Module * * \brief Expression of a diagonal/subdiagonal/superdiagonal in a matrix * * \param MatrixType the type of the object in which we are taking a sub/main/super diagonal * \param DiagIndex the index of the sub/super diagonal. The default is 0 and it means the main diagonal. * A positive value means a superdiagonal, a negative value means a subdiagonal. * You can also use DynamicIndex so the index can be set at runtime. * * The matrix is not required to be square. * * This class represents an expression of the main diagonal, or any sub/super diagonal * of a square matrix. It is the return type of MatrixBase::diagonal() and MatrixBase::diagonal(Index) and most of the * time this is the only way it is used. * * \sa MatrixBase::diagonal(), MatrixBase::diagonal(Index) */ namespace internal { template struct traits > : traits { typedef typename ref_selector::type MatrixTypeNested; typedef typename remove_reference::type _MatrixTypeNested; typedef typename MatrixType::StorageKind StorageKind; enum { RowsAtCompileTime = (int(DiagIndex) == DynamicIndex || int(MatrixType::SizeAtCompileTime) == Dynamic) ? Dynamic : (EIGEN_PLAIN_ENUM_MIN(MatrixType::RowsAtCompileTime - EIGEN_PLAIN_ENUM_MAX(-DiagIndex, 0), MatrixType::ColsAtCompileTime - EIGEN_PLAIN_ENUM_MAX( DiagIndex, 0))), ColsAtCompileTime = 1, MaxRowsAtCompileTime = int(MatrixType::MaxSizeAtCompileTime) == Dynamic ? Dynamic : DiagIndex == DynamicIndex ? EIGEN_SIZE_MIN_PREFER_FIXED(MatrixType::MaxRowsAtCompileTime, MatrixType::MaxColsAtCompileTime) : (EIGEN_PLAIN_ENUM_MIN(MatrixType::MaxRowsAtCompileTime - EIGEN_PLAIN_ENUM_MAX(-DiagIndex, 0), MatrixType::MaxColsAtCompileTime - EIGEN_PLAIN_ENUM_MAX( DiagIndex, 0))), MaxColsAtCompileTime = 1, MaskLvalueBit = is_lvalue::value ? LvalueBit : 0, Flags = (unsigned int)_MatrixTypeNested::Flags & (RowMajorBit | MaskLvalueBit | DirectAccessBit) & ~RowMajorBit, // FIXME DirectAccessBit should not be handled by expressions MatrixTypeOuterStride = outer_stride_at_compile_time::ret, InnerStrideAtCompileTime = MatrixTypeOuterStride == Dynamic ? Dynamic : MatrixTypeOuterStride+1, OuterStrideAtCompileTime = 0 }; }; } template class Diagonal : public internal::dense_xpr_base< Diagonal >::type { public: enum { DiagIndex = _DiagIndex }; typedef typename internal::dense_xpr_base::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(Diagonal) EIGEN_DEVICE_FUNC explicit inline Diagonal(MatrixType& matrix, Index a_index = DiagIndex) : m_matrix(matrix), m_index(a_index) { eigen_assert( a_index <= m_matrix.cols() && -a_index <= m_matrix.rows() ); } EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Diagonal) EIGEN_DEVICE_FUNC inline Index rows() const { return m_index.value()<0 ? numext::mini(m_matrix.cols(),m_matrix.rows()+m_index.value()) : numext::mini(m_matrix.rows(),m_matrix.cols()-m_index.value()); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return 1; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT { return m_matrix.outerStride() + 1; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT { return 0; } typedef typename internal::conditional< internal::is_lvalue::value, Scalar, const Scalar >::type ScalarWithConstIfNotLvalue; EIGEN_DEVICE_FUNC inline ScalarWithConstIfNotLvalue* data() { return &(m_matrix.coeffRef(rowOffset(), colOffset())); } EIGEN_DEVICE_FUNC inline const Scalar* data() const { return &(m_matrix.coeffRef(rowOffset(), colOffset())); } EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index row, Index) { EIGEN_STATIC_ASSERT_LVALUE(MatrixType) return m_matrix.coeffRef(row+rowOffset(), row+colOffset()); } EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index row, Index) const { return m_matrix.coeffRef(row+rowOffset(), row+colOffset()); } EIGEN_DEVICE_FUNC inline CoeffReturnType coeff(Index row, Index) const { return m_matrix.coeff(row+rowOffset(), row+colOffset()); } EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index idx) { EIGEN_STATIC_ASSERT_LVALUE(MatrixType) return m_matrix.coeffRef(idx+rowOffset(), idx+colOffset()); } EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index idx) const { return m_matrix.coeffRef(idx+rowOffset(), idx+colOffset()); } EIGEN_DEVICE_FUNC inline CoeffReturnType coeff(Index idx) const { return m_matrix.coeff(idx+rowOffset(), idx+colOffset()); } EIGEN_DEVICE_FUNC inline const typename internal::remove_all::type& nestedExpression() const { return m_matrix; } EIGEN_DEVICE_FUNC inline Index index() const { return m_index.value(); } protected: typename internal::ref_selector::non_const_type m_matrix; const internal::variable_if_dynamicindex m_index; private: // some compilers may fail to optimize std::max etc in case of compile-time constants... EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index absDiagIndex() const EIGEN_NOEXCEPT { return m_index.value()>0 ? m_index.value() : -m_index.value(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rowOffset() const EIGEN_NOEXCEPT { return m_index.value()>0 ? 0 : -m_index.value(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index colOffset() const EIGEN_NOEXCEPT { return m_index.value()>0 ? m_index.value() : 0; } // trigger a compile-time error if someone try to call packet template typename MatrixType::PacketReturnType packet(Index) const; template typename MatrixType::PacketReturnType packet(Index,Index) const; }; /** \returns an expression of the main diagonal of the matrix \c *this * * \c *this is not required to be square. * * Example: \include MatrixBase_diagonal.cpp * Output: \verbinclude MatrixBase_diagonal.out * * \sa class Diagonal */ template EIGEN_DEVICE_FUNC inline typename MatrixBase::DiagonalReturnType MatrixBase::diagonal() { return DiagonalReturnType(derived()); } /** This is the const version of diagonal(). */ template EIGEN_DEVICE_FUNC inline typename MatrixBase::ConstDiagonalReturnType MatrixBase::diagonal() const { return ConstDiagonalReturnType(derived()); } /** \returns an expression of the \a DiagIndex-th sub or super diagonal of the matrix \c *this * * \c *this is not required to be square. * * The template parameter \a DiagIndex represent a super diagonal if \a DiagIndex > 0 * and a sub diagonal otherwise. \a DiagIndex == 0 is equivalent to the main diagonal. * * Example: \include MatrixBase_diagonal_int.cpp * Output: \verbinclude MatrixBase_diagonal_int.out * * \sa MatrixBase::diagonal(), class Diagonal */ template EIGEN_DEVICE_FUNC inline typename MatrixBase::DiagonalDynamicIndexReturnType MatrixBase::diagonal(Index index) { return DiagonalDynamicIndexReturnType(derived(), index); } /** This is the const version of diagonal(Index). */ template EIGEN_DEVICE_FUNC inline typename MatrixBase::ConstDiagonalDynamicIndexReturnType MatrixBase::diagonal(Index index) const { return ConstDiagonalDynamicIndexReturnType(derived(), index); } /** \returns an expression of the \a DiagIndex-th sub or super diagonal of the matrix \c *this * * \c *this is not required to be square. * * The template parameter \a DiagIndex represent a super diagonal if \a DiagIndex > 0 * and a sub diagonal otherwise. \a DiagIndex == 0 is equivalent to the main diagonal. * * Example: \include MatrixBase_diagonal_template_int.cpp * Output: \verbinclude MatrixBase_diagonal_template_int.out * * \sa MatrixBase::diagonal(), class Diagonal */ template template EIGEN_DEVICE_FUNC inline typename MatrixBase::template DiagonalIndexReturnType::Type MatrixBase::diagonal() { return typename DiagonalIndexReturnType::Type(derived()); } /** This is the const version of diagonal(). */ template template EIGEN_DEVICE_FUNC inline typename MatrixBase::template ConstDiagonalIndexReturnType::Type MatrixBase::diagonal() const { return typename ConstDiagonalIndexReturnType::Type(derived()); } } // end namespace Eigen #endif // EIGEN_DIAGONAL_H RcppEigen/inst/include/Eigen/src/Core/Reshaped.h0000644000176200001440000004121114567757725021170 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2017 Gael Guennebaud // Copyright (C) 2014 yoco // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_RESHAPED_H #define EIGEN_RESHAPED_H namespace Eigen { /** \class Reshaped * \ingroup Core_Module * * \brief Expression of a fixed-size or dynamic-size reshape * * \tparam XprType the type of the expression in which we are taking a reshape * \tparam Rows the number of rows of the reshape we are taking at compile time (optional) * \tparam Cols the number of columns of the reshape we are taking at compile time (optional) * \tparam Order can be ColMajor or RowMajor, default is ColMajor. * * This class represents an expression of either a fixed-size or dynamic-size reshape. * It is the return type of DenseBase::reshaped(NRowsType,NColsType) and * most of the time this is the only way it is used. * * However, in C++98, if you want to directly maniputate reshaped expressions, * for instance if you want to write a function returning such an expression, you * will need to use this class. In C++11, it is advised to use the \em auto * keyword for such use cases. * * Here is an example illustrating the dynamic case: * \include class_Reshaped.cpp * Output: \verbinclude class_Reshaped.out * * Here is an example illustrating the fixed-size case: * \include class_FixedReshaped.cpp * Output: \verbinclude class_FixedReshaped.out * * \sa DenseBase::reshaped(NRowsType,NColsType) */ namespace internal { template struct traits > : traits { typedef typename traits::Scalar Scalar; typedef typename traits::StorageKind StorageKind; typedef typename traits::XprKind XprKind; enum{ MatrixRows = traits::RowsAtCompileTime, MatrixCols = traits::ColsAtCompileTime, RowsAtCompileTime = Rows, ColsAtCompileTime = Cols, MaxRowsAtCompileTime = Rows, MaxColsAtCompileTime = Cols, XpxStorageOrder = ((int(traits::Flags) & RowMajorBit) == RowMajorBit) ? RowMajor : ColMajor, ReshapedStorageOrder = (RowsAtCompileTime == 1 && ColsAtCompileTime != 1) ? RowMajor : (ColsAtCompileTime == 1 && RowsAtCompileTime != 1) ? ColMajor : XpxStorageOrder, HasSameStorageOrderAsXprType = (ReshapedStorageOrder == XpxStorageOrder), InnerSize = (ReshapedStorageOrder==int(RowMajor)) ? int(ColsAtCompileTime) : int(RowsAtCompileTime), InnerStrideAtCompileTime = HasSameStorageOrderAsXprType ? int(inner_stride_at_compile_time::ret) : Dynamic, OuterStrideAtCompileTime = Dynamic, HasDirectAccess = internal::has_direct_access::ret && (Order==int(XpxStorageOrder)) && ((evaluator::Flags&LinearAccessBit)==LinearAccessBit), MaskPacketAccessBit = (InnerSize == Dynamic || (InnerSize % packet_traits::size) == 0) && (InnerStrideAtCompileTime == 1) ? PacketAccessBit : 0, //MaskAlignedBit = ((OuterStrideAtCompileTime!=Dynamic) && (((OuterStrideAtCompileTime * int(sizeof(Scalar))) % 16) == 0)) ? AlignedBit : 0, FlagsLinearAccessBit = (RowsAtCompileTime == 1 || ColsAtCompileTime == 1) ? LinearAccessBit : 0, FlagsLvalueBit = is_lvalue::value ? LvalueBit : 0, FlagsRowMajorBit = (ReshapedStorageOrder==int(RowMajor)) ? RowMajorBit : 0, FlagsDirectAccessBit = HasDirectAccess ? DirectAccessBit : 0, Flags0 = traits::Flags & ( (HereditaryBits & ~RowMajorBit) | MaskPacketAccessBit), Flags = (Flags0 | FlagsLinearAccessBit | FlagsLvalueBit | FlagsRowMajorBit | FlagsDirectAccessBit) }; }; template class ReshapedImpl_dense; } // end namespace internal template class ReshapedImpl; template class Reshaped : public ReshapedImpl::StorageKind> { typedef ReshapedImpl::StorageKind> Impl; public: //typedef typename Impl::Base Base; typedef Impl Base; EIGEN_GENERIC_PUBLIC_INTERFACE(Reshaped) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Reshaped) /** Fixed-size constructor */ EIGEN_DEVICE_FUNC inline Reshaped(XprType& xpr) : Impl(xpr) { EIGEN_STATIC_ASSERT(RowsAtCompileTime!=Dynamic && ColsAtCompileTime!=Dynamic,THIS_METHOD_IS_ONLY_FOR_FIXED_SIZE) eigen_assert(Rows * Cols == xpr.rows() * xpr.cols()); } /** Dynamic-size constructor */ EIGEN_DEVICE_FUNC inline Reshaped(XprType& xpr, Index reshapeRows, Index reshapeCols) : Impl(xpr, reshapeRows, reshapeCols) { eigen_assert((RowsAtCompileTime==Dynamic || RowsAtCompileTime==reshapeRows) && (ColsAtCompileTime==Dynamic || ColsAtCompileTime==reshapeCols)); eigen_assert(reshapeRows * reshapeCols == xpr.rows() * xpr.cols()); } }; // The generic default implementation for dense reshape simply forward to the internal::ReshapedImpl_dense // that must be specialized for direct and non-direct access... template class ReshapedImpl : public internal::ReshapedImpl_dense >::HasDirectAccess> { typedef internal::ReshapedImpl_dense >::HasDirectAccess> Impl; public: typedef Impl Base; EIGEN_INHERIT_ASSIGNMENT_OPERATORS(ReshapedImpl) EIGEN_DEVICE_FUNC inline ReshapedImpl(XprType& xpr) : Impl(xpr) {} EIGEN_DEVICE_FUNC inline ReshapedImpl(XprType& xpr, Index reshapeRows, Index reshapeCols) : Impl(xpr, reshapeRows, reshapeCols) {} }; namespace internal { /** \internal Internal implementation of dense Reshaped in the general case. */ template class ReshapedImpl_dense : public internal::dense_xpr_base >::type { typedef Reshaped ReshapedType; public: typedef typename internal::dense_xpr_base::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(ReshapedType) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(ReshapedImpl_dense) typedef typename internal::ref_selector::non_const_type MatrixTypeNested; typedef typename internal::remove_all::type NestedExpression; class InnerIterator; /** Fixed-size constructor */ EIGEN_DEVICE_FUNC inline ReshapedImpl_dense(XprType& xpr) : m_xpr(xpr), m_rows(Rows), m_cols(Cols) {} /** Dynamic-size constructor */ EIGEN_DEVICE_FUNC inline ReshapedImpl_dense(XprType& xpr, Index nRows, Index nCols) : m_xpr(xpr), m_rows(nRows), m_cols(nCols) {} EIGEN_DEVICE_FUNC Index rows() const { return m_rows; } EIGEN_DEVICE_FUNC Index cols() const { return m_cols; } #ifdef EIGEN_PARSED_BY_DOXYGEN /** \sa MapBase::data() */ EIGEN_DEVICE_FUNC inline const Scalar* data() const; EIGEN_DEVICE_FUNC inline Index innerStride() const; EIGEN_DEVICE_FUNC inline Index outerStride() const; #endif /** \returns the nested expression */ EIGEN_DEVICE_FUNC const typename internal::remove_all::type& nestedExpression() const { return m_xpr; } /** \returns the nested expression */ EIGEN_DEVICE_FUNC typename internal::remove_reference::type& nestedExpression() { return m_xpr; } protected: MatrixTypeNested m_xpr; const internal::variable_if_dynamic m_rows; const internal::variable_if_dynamic m_cols; }; /** \internal Internal implementation of dense Reshaped in the direct access case. */ template class ReshapedImpl_dense : public MapBase > { typedef Reshaped ReshapedType; typedef typename internal::ref_selector::non_const_type XprTypeNested; public: typedef MapBase Base; EIGEN_DENSE_PUBLIC_INTERFACE(ReshapedType) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(ReshapedImpl_dense) /** Fixed-size constructor */ EIGEN_DEVICE_FUNC inline ReshapedImpl_dense(XprType& xpr) : Base(xpr.data()), m_xpr(xpr) {} /** Dynamic-size constructor */ EIGEN_DEVICE_FUNC inline ReshapedImpl_dense(XprType& xpr, Index nRows, Index nCols) : Base(xpr.data(), nRows, nCols), m_xpr(xpr) {} EIGEN_DEVICE_FUNC const typename internal::remove_all::type& nestedExpression() const { return m_xpr; } EIGEN_DEVICE_FUNC XprType& nestedExpression() { return m_xpr; } /** \sa MapBase::innerStride() */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const { return m_xpr.innerStride(); } /** \sa MapBase::outerStride() */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const { return ((Flags&RowMajorBit)==RowMajorBit) ? this->cols() : this->rows(); } protected: XprTypeNested m_xpr; }; // Evaluators template struct reshaped_evaluator; template struct evaluator > : reshaped_evaluator >::HasDirectAccess> { typedef Reshaped XprType; typedef typename XprType::Scalar Scalar; // TODO: should check for smaller packet types typedef typename packet_traits::type PacketScalar; enum { CoeffReadCost = evaluator::CoeffReadCost, HasDirectAccess = traits::HasDirectAccess, // RowsAtCompileTime = traits::RowsAtCompileTime, // ColsAtCompileTime = traits::ColsAtCompileTime, // MaxRowsAtCompileTime = traits::MaxRowsAtCompileTime, // MaxColsAtCompileTime = traits::MaxColsAtCompileTime, // // InnerStrideAtCompileTime = traits::HasSameStorageOrderAsXprType // ? int(inner_stride_at_compile_time::ret) // : Dynamic, // OuterStrideAtCompileTime = Dynamic, FlagsLinearAccessBit = (traits::RowsAtCompileTime == 1 || traits::ColsAtCompileTime == 1 || HasDirectAccess) ? LinearAccessBit : 0, FlagsRowMajorBit = (traits::ReshapedStorageOrder==int(RowMajor)) ? RowMajorBit : 0, FlagsDirectAccessBit = HasDirectAccess ? DirectAccessBit : 0, Flags0 = evaluator::Flags & (HereditaryBits & ~RowMajorBit), Flags = Flags0 | FlagsLinearAccessBit | FlagsRowMajorBit | FlagsDirectAccessBit, PacketAlignment = unpacket_traits::alignment, Alignment = evaluator::Alignment }; typedef reshaped_evaluator reshaped_evaluator_type; EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) : reshaped_evaluator_type(xpr) { EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } }; template struct reshaped_evaluator : evaluator_base > { typedef Reshaped XprType; enum { CoeffReadCost = evaluator::CoeffReadCost /* TODO + cost of index computations */, Flags = (evaluator::Flags & (HereditaryBits /*| LinearAccessBit | DirectAccessBit*/)), Alignment = 0 }; EIGEN_DEVICE_FUNC explicit reshaped_evaluator(const XprType& xpr) : m_argImpl(xpr.nestedExpression()), m_xpr(xpr) { EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); } typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef std::pair RowCol; inline RowCol index_remap(Index rowId, Index colId) const { if(Order==ColMajor) { const Index nth_elem_idx = colId * m_xpr.rows() + rowId; return RowCol(nth_elem_idx % m_xpr.nestedExpression().rows(), nth_elem_idx / m_xpr.nestedExpression().rows()); } else { const Index nth_elem_idx = colId + rowId * m_xpr.cols(); return RowCol(nth_elem_idx / m_xpr.nestedExpression().cols(), nth_elem_idx % m_xpr.nestedExpression().cols()); } } EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index rowId, Index colId) { EIGEN_STATIC_ASSERT_LVALUE(XprType) const RowCol row_col = index_remap(rowId, colId); return m_argImpl.coeffRef(row_col.first, row_col.second); } EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index rowId, Index colId) const { const RowCol row_col = index_remap(rowId, colId); return m_argImpl.coeffRef(row_col.first, row_col.second); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CoeffReturnType coeff(Index rowId, Index colId) const { const RowCol row_col = index_remap(rowId, colId); return m_argImpl.coeff(row_col.first, row_col.second); } EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index index) { EIGEN_STATIC_ASSERT_LVALUE(XprType) const RowCol row_col = index_remap(Rows == 1 ? 0 : index, Rows == 1 ? index : 0); return m_argImpl.coeffRef(row_col.first, row_col.second); } EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index index) const { const RowCol row_col = index_remap(Rows == 1 ? 0 : index, Rows == 1 ? index : 0); return m_argImpl.coeffRef(row_col.first, row_col.second); } EIGEN_DEVICE_FUNC inline const CoeffReturnType coeff(Index index) const { const RowCol row_col = index_remap(Rows == 1 ? 0 : index, Rows == 1 ? index : 0); return m_argImpl.coeff(row_col.first, row_col.second); } #if 0 EIGEN_DEVICE_FUNC template inline PacketScalar packet(Index rowId, Index colId) const { const RowCol row_col = index_remap(rowId, colId); return m_argImpl.template packet(row_col.first, row_col.second); } template EIGEN_DEVICE_FUNC inline void writePacket(Index rowId, Index colId, const PacketScalar& val) { const RowCol row_col = index_remap(rowId, colId); m_argImpl.const_cast_derived().template writePacket (row_col.first, row_col.second, val); } template EIGEN_DEVICE_FUNC inline PacketScalar packet(Index index) const { const RowCol row_col = index_remap(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0); return m_argImpl.template packet(row_col.first, row_col.second); } template EIGEN_DEVICE_FUNC inline void writePacket(Index index, const PacketScalar& val) { const RowCol row_col = index_remap(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0); return m_argImpl.template packet(row_col.first, row_col.second, val); } #endif protected: evaluator m_argImpl; const XprType& m_xpr; }; template struct reshaped_evaluator : mapbase_evaluator, typename Reshaped::PlainObject> { typedef Reshaped XprType; typedef typename XprType::Scalar Scalar; EIGEN_DEVICE_FUNC explicit reshaped_evaluator(const XprType& xpr) : mapbase_evaluator(xpr) { // TODO: for the 3.4 release, this should be turned to an internal assertion, but let's keep it as is for the beta lifetime eigen_assert(((internal::UIntPtr(xpr.data()) % EIGEN_PLAIN_ENUM_MAX(1,evaluator::Alignment)) == 0) && "data is not aligned"); } }; } // end namespace internal } // end namespace Eigen #endif // EIGEN_RESHAPED_H RcppEigen/inst/include/Eigen/src/Core/Redux.h0000644000176200001440000004537314567757725020541 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008 Gael Guennebaud // Copyright (C) 2006-2008 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_REDUX_H #define EIGEN_REDUX_H namespace Eigen { namespace internal { // TODO // * implement other kind of vectorization // * factorize code /*************************************************************************** * Part 1 : the logic deciding a strategy for vectorization and unrolling ***************************************************************************/ template struct redux_traits { public: typedef typename find_best_packet::type PacketType; enum { PacketSize = unpacket_traits::size, InnerMaxSize = int(Evaluator::IsRowMajor) ? Evaluator::MaxColsAtCompileTime : Evaluator::MaxRowsAtCompileTime, OuterMaxSize = int(Evaluator::IsRowMajor) ? Evaluator::MaxRowsAtCompileTime : Evaluator::MaxColsAtCompileTime, SliceVectorizedWork = int(InnerMaxSize)==Dynamic ? Dynamic : int(OuterMaxSize)==Dynamic ? (int(InnerMaxSize)>=int(PacketSize) ? Dynamic : 0) : (int(InnerMaxSize)/int(PacketSize)) * int(OuterMaxSize) }; enum { MightVectorize = (int(Evaluator::Flags)&ActualPacketAccessBit) && (functor_traits::PacketAccess), MayLinearVectorize = bool(MightVectorize) && (int(Evaluator::Flags)&LinearAccessBit), MaySliceVectorize = bool(MightVectorize) && (int(SliceVectorizedWork)==Dynamic || int(SliceVectorizedWork)>=3) }; public: enum { Traversal = int(MayLinearVectorize) ? int(LinearVectorizedTraversal) : int(MaySliceVectorize) ? int(SliceVectorizedTraversal) : int(DefaultTraversal) }; public: enum { Cost = Evaluator::SizeAtCompileTime == Dynamic ? HugeCost : int(Evaluator::SizeAtCompileTime) * int(Evaluator::CoeffReadCost) + (Evaluator::SizeAtCompileTime-1) * functor_traits::Cost, UnrollingLimit = EIGEN_UNROLLING_LIMIT * (int(Traversal) == int(DefaultTraversal) ? 1 : int(PacketSize)) }; public: enum { Unrolling = Cost <= UnrollingLimit ? CompleteUnrolling : NoUnrolling }; #ifdef EIGEN_DEBUG_ASSIGN static void debug() { std::cerr << "Xpr: " << typeid(typename Evaluator::XprType).name() << std::endl; std::cerr.setf(std::ios::hex, std::ios::basefield); EIGEN_DEBUG_VAR(Evaluator::Flags) std::cerr.unsetf(std::ios::hex); EIGEN_DEBUG_VAR(InnerMaxSize) EIGEN_DEBUG_VAR(OuterMaxSize) EIGEN_DEBUG_VAR(SliceVectorizedWork) EIGEN_DEBUG_VAR(PacketSize) EIGEN_DEBUG_VAR(MightVectorize) EIGEN_DEBUG_VAR(MayLinearVectorize) EIGEN_DEBUG_VAR(MaySliceVectorize) std::cerr << "Traversal" << " = " << Traversal << " (" << demangle_traversal(Traversal) << ")" << std::endl; EIGEN_DEBUG_VAR(UnrollingLimit) std::cerr << "Unrolling" << " = " << Unrolling << " (" << demangle_unrolling(Unrolling) << ")" << std::endl; std::cerr << std::endl; } #endif }; /*************************************************************************** * Part 2 : unrollers ***************************************************************************/ /*** no vectorization ***/ template struct redux_novec_unroller { enum { HalfLength = Length/2 }; typedef typename Evaluator::Scalar Scalar; EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator &eval, const Func& func) { return func(redux_novec_unroller::run(eval,func), redux_novec_unroller::run(eval,func)); } }; template struct redux_novec_unroller { enum { outer = Start / Evaluator::InnerSizeAtCompileTime, inner = Start % Evaluator::InnerSizeAtCompileTime }; typedef typename Evaluator::Scalar Scalar; EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator &eval, const Func&) { return eval.coeffByOuterInner(outer, inner); } }; // This is actually dead code and will never be called. It is required // to prevent false warnings regarding failed inlining though // for 0 length run() will never be called at all. template struct redux_novec_unroller { typedef typename Evaluator::Scalar Scalar; EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator&, const Func&) { return Scalar(); } }; /*** vectorization ***/ template struct redux_vec_unroller { template EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE PacketType run(const Evaluator &eval, const Func& func) { enum { PacketSize = unpacket_traits::size, HalfLength = Length/2 }; return func.packetOp( redux_vec_unroller::template run(eval,func), redux_vec_unroller::template run(eval,func) ); } }; template struct redux_vec_unroller { template EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE PacketType run(const Evaluator &eval, const Func&) { enum { PacketSize = unpacket_traits::size, index = Start * PacketSize, outer = index / int(Evaluator::InnerSizeAtCompileTime), inner = index % int(Evaluator::InnerSizeAtCompileTime), alignment = Evaluator::Alignment }; return eval.template packetByOuterInner(outer, inner); } }; /*************************************************************************** * Part 3 : implementation of all cases ***************************************************************************/ template::Traversal, int Unrolling = redux_traits::Unrolling > struct redux_impl; template struct redux_impl { typedef typename Evaluator::Scalar Scalar; template EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator &eval, const Func& func, const XprType& xpr) { eigen_assert(xpr.rows()>0 && xpr.cols()>0 && "you are using an empty matrix"); Scalar res; res = eval.coeffByOuterInner(0, 0); for(Index i = 1; i < xpr.innerSize(); ++i) res = func(res, eval.coeffByOuterInner(0, i)); for(Index i = 1; i < xpr.outerSize(); ++i) for(Index j = 0; j < xpr.innerSize(); ++j) res = func(res, eval.coeffByOuterInner(i, j)); return res; } }; template struct redux_impl : redux_novec_unroller { typedef redux_novec_unroller Base; typedef typename Evaluator::Scalar Scalar; template EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator &eval, const Func& func, const XprType& /*xpr*/) { return Base::run(eval,func); } }; template struct redux_impl { typedef typename Evaluator::Scalar Scalar; typedef typename redux_traits::PacketType PacketScalar; template static Scalar run(const Evaluator &eval, const Func& func, const XprType& xpr) { const Index size = xpr.size(); const Index packetSize = redux_traits::PacketSize; const int packetAlignment = unpacket_traits::alignment; enum { alignment0 = (bool(Evaluator::Flags & DirectAccessBit) && bool(packet_traits::AlignedOnScalar)) ? int(packetAlignment) : int(Unaligned), alignment = EIGEN_PLAIN_ENUM_MAX(alignment0, Evaluator::Alignment) }; const Index alignedStart = internal::first_default_aligned(xpr); const Index alignedSize2 = ((size-alignedStart)/(2*packetSize))*(2*packetSize); const Index alignedSize = ((size-alignedStart)/(packetSize))*(packetSize); const Index alignedEnd2 = alignedStart + alignedSize2; const Index alignedEnd = alignedStart + alignedSize; Scalar res; if(alignedSize) { PacketScalar packet_res0 = eval.template packet(alignedStart); if(alignedSize>packetSize) // we have at least two packets to partly unroll the loop { PacketScalar packet_res1 = eval.template packet(alignedStart+packetSize); for(Index index = alignedStart + 2*packetSize; index < alignedEnd2; index += 2*packetSize) { packet_res0 = func.packetOp(packet_res0, eval.template packet(index)); packet_res1 = func.packetOp(packet_res1, eval.template packet(index+packetSize)); } packet_res0 = func.packetOp(packet_res0,packet_res1); if(alignedEnd>alignedEnd2) packet_res0 = func.packetOp(packet_res0, eval.template packet(alignedEnd2)); } res = func.predux(packet_res0); for(Index index = 0; index < alignedStart; ++index) res = func(res,eval.coeff(index)); for(Index index = alignedEnd; index < size; ++index) res = func(res,eval.coeff(index)); } else // too small to vectorize anything. // since this is dynamic-size hence inefficient anyway for such small sizes, don't try to optimize. { res = eval.coeff(0); for(Index index = 1; index < size; ++index) res = func(res,eval.coeff(index)); } return res; } }; // NOTE: for SliceVectorizedTraversal we simply bypass unrolling template struct redux_impl { typedef typename Evaluator::Scalar Scalar; typedef typename redux_traits::PacketType PacketType; template EIGEN_DEVICE_FUNC static Scalar run(const Evaluator &eval, const Func& func, const XprType& xpr) { eigen_assert(xpr.rows()>0 && xpr.cols()>0 && "you are using an empty matrix"); const Index innerSize = xpr.innerSize(); const Index outerSize = xpr.outerSize(); enum { packetSize = redux_traits::PacketSize }; const Index packetedInnerSize = ((innerSize)/packetSize)*packetSize; Scalar res; if(packetedInnerSize) { PacketType packet_res = eval.template packet(0,0); for(Index j=0; j(j,i)); res = func.predux(packet_res); for(Index j=0; j::run(eval, func, xpr); } return res; } }; template struct redux_impl { typedef typename Evaluator::Scalar Scalar; typedef typename redux_traits::PacketType PacketType; enum { PacketSize = redux_traits::PacketSize, Size = Evaluator::SizeAtCompileTime, VectorizedSize = (int(Size) / int(PacketSize)) * int(PacketSize) }; template EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator &eval, const Func& func, const XprType &xpr) { EIGEN_ONLY_USED_FOR_DEBUG(xpr) eigen_assert(xpr.rows()>0 && xpr.cols()>0 && "you are using an empty matrix"); if (VectorizedSize > 0) { Scalar res = func.predux(redux_vec_unroller::template run(eval,func)); if (VectorizedSize != Size) res = func(res,redux_novec_unroller::run(eval,func)); return res; } else { return redux_novec_unroller::run(eval,func); } } }; // evaluator adaptor template class redux_evaluator : public internal::evaluator<_XprType> { typedef internal::evaluator<_XprType> Base; public: typedef _XprType XprType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit redux_evaluator(const XprType &xpr) : Base(xpr) {} typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename XprType::PacketScalar PacketScalar; enum { MaxRowsAtCompileTime = XprType::MaxRowsAtCompileTime, MaxColsAtCompileTime = XprType::MaxColsAtCompileTime, // TODO we should not remove DirectAccessBit and rather find an elegant way to query the alignment offset at runtime from the evaluator Flags = Base::Flags & ~DirectAccessBit, IsRowMajor = XprType::IsRowMajor, SizeAtCompileTime = XprType::SizeAtCompileTime, InnerSizeAtCompileTime = XprType::InnerSizeAtCompileTime }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeffByOuterInner(Index outer, Index inner) const { return Base::coeff(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetByOuterInner(Index outer, Index inner) const { return Base::template packet(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer); } }; } // end namespace internal /*************************************************************************** * Part 4 : public API ***************************************************************************/ /** \returns the result of a full redux operation on the whole matrix or vector using \a func * * The template parameter \a BinaryOp is the type of the functor \a func which must be * an associative operator. Both current C++98 and C++11 functor styles are handled. * * \warning the matrix must be not empty, otherwise an assertion is triggered. * * \sa DenseBase::sum(), DenseBase::minCoeff(), DenseBase::maxCoeff(), MatrixBase::colwise(), MatrixBase::rowwise() */ template template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits::Scalar DenseBase::redux(const Func& func) const { eigen_assert(this->rows()>0 && this->cols()>0 && "you are using an empty matrix"); typedef typename internal::redux_evaluator ThisEvaluator; ThisEvaluator thisEval(derived()); // The initial expression is passed to the reducer as an additional argument instead of // passing it as a member of redux_evaluator to help return internal::redux_impl::run(thisEval, func, derived()); } /** \returns the minimum of all coefficients of \c *this. * In case \c *this contains NaN, NaNPropagation determines the behavior: * NaNPropagation == PropagateFast : undefined * NaNPropagation == PropagateNaN : result is NaN * NaNPropagation == PropagateNumbers : result is minimum of elements that are not NaN * \warning the matrix must be not empty, otherwise an assertion is triggered. */ template template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits::Scalar DenseBase::minCoeff() const { return derived().redux(Eigen::internal::scalar_min_op()); } /** \returns the maximum of all coefficients of \c *this. * In case \c *this contains NaN, NaNPropagation determines the behavior: * NaNPropagation == PropagateFast : undefined * NaNPropagation == PropagateNaN : result is NaN * NaNPropagation == PropagateNumbers : result is maximum of elements that are not NaN * \warning the matrix must be not empty, otherwise an assertion is triggered. */ template template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits::Scalar DenseBase::maxCoeff() const { return derived().redux(Eigen::internal::scalar_max_op()); } /** \returns the sum of all coefficients of \c *this * * If \c *this is empty, then the value 0 is returned. * * \sa trace(), prod(), mean() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits::Scalar DenseBase::sum() const { if(SizeAtCompileTime==0 || (SizeAtCompileTime==Dynamic && size()==0)) return Scalar(0); return derived().redux(Eigen::internal::scalar_sum_op()); } /** \returns the mean of all coefficients of *this * * \sa trace(), prod(), sum() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits::Scalar DenseBase::mean() const { #ifdef __INTEL_COMPILER #pragma warning push #pragma warning ( disable : 2259 ) #endif return Scalar(derived().redux(Eigen::internal::scalar_sum_op())) / Scalar(this->size()); #ifdef __INTEL_COMPILER #pragma warning pop #endif } /** \returns the product of all coefficients of *this * * Example: \include MatrixBase_prod.cpp * Output: \verbinclude MatrixBase_prod.out * * \sa sum(), mean(), trace() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits::Scalar DenseBase::prod() const { if(SizeAtCompileTime==0 || (SizeAtCompileTime==Dynamic && size()==0)) return Scalar(1); return derived().redux(Eigen::internal::scalar_product_op()); } /** \returns the trace of \c *this, i.e. the sum of the coefficients on the main diagonal. * * \c *this can be any matrix, not necessarily square. * * \sa diagonal(), sum() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits::Scalar MatrixBase::trace() const { return derived().diagonal().sum(); } } // end namespace Eigen #endif // EIGEN_REDUX_H RcppEigen/inst/include/Eigen/src/Core/SolverBase.h0000644000176200001440000001403214567757725021503 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2015 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_SOLVERBASE_H #define EIGEN_SOLVERBASE_H namespace Eigen { namespace internal { template struct solve_assertion { template static void run(const Derived& solver, const Rhs& b) { solver.template _check_solve_assertion(b); } }; template struct solve_assertion > { typedef Transpose type; template static void run(const type& transpose, const Rhs& b) { internal::solve_assertion::type>::template run(transpose.nestedExpression(), b); } }; template struct solve_assertion, const Transpose > > { typedef CwiseUnaryOp, const Transpose > type; template static void run(const type& adjoint, const Rhs& b) { internal::solve_assertion >::type>::template run(adjoint.nestedExpression(), b); } }; } // end namespace internal /** \class SolverBase * \brief A base class for matrix decomposition and solvers * * \tparam Derived the actual type of the decomposition/solver. * * Any matrix decomposition inheriting this base class provide the following API: * * \code * MatrixType A, b, x; * DecompositionType dec(A); * x = dec.solve(b); // solve A * x = b * x = dec.transpose().solve(b); // solve A^T * x = b * x = dec.adjoint().solve(b); // solve A' * x = b * \endcode * * \warning Currently, any other usage of transpose() and adjoint() are not supported and will produce compilation errors. * * \sa class PartialPivLU, class FullPivLU, class HouseholderQR, class ColPivHouseholderQR, class FullPivHouseholderQR, class CompleteOrthogonalDecomposition, class LLT, class LDLT, class SVDBase */ template class SolverBase : public EigenBase { public: typedef EigenBase Base; typedef typename internal::traits::Scalar Scalar; typedef Scalar CoeffReturnType; template friend struct internal::solve_assertion; enum { RowsAtCompileTime = internal::traits::RowsAtCompileTime, ColsAtCompileTime = internal::traits::ColsAtCompileTime, SizeAtCompileTime = (internal::size_at_compile_time::RowsAtCompileTime, internal::traits::ColsAtCompileTime>::ret), MaxRowsAtCompileTime = internal::traits::MaxRowsAtCompileTime, MaxColsAtCompileTime = internal::traits::MaxColsAtCompileTime, MaxSizeAtCompileTime = (internal::size_at_compile_time::MaxRowsAtCompileTime, internal::traits::MaxColsAtCompileTime>::ret), IsVectorAtCompileTime = internal::traits::MaxRowsAtCompileTime == 1 || internal::traits::MaxColsAtCompileTime == 1, NumDimensions = int(MaxSizeAtCompileTime) == 1 ? 0 : bool(IsVectorAtCompileTime) ? 1 : 2 }; /** Default constructor */ SolverBase() {} ~SolverBase() {} using Base::derived; /** \returns an expression of the solution x of \f$ A x = b \f$ using the current decomposition of A. */ template inline const Solve solve(const MatrixBase& b) const { internal::solve_assertion::type>::template run(derived(), b); return Solve(derived(), b.derived()); } /** \internal the return type of transpose() */ typedef typename internal::add_const >::type ConstTransposeReturnType; /** \returns an expression of the transposed of the factored matrix. * * A typical usage is to solve for the transposed problem A^T x = b: * \code x = dec.transpose().solve(b); \endcode * * \sa adjoint(), solve() */ inline ConstTransposeReturnType transpose() const { return ConstTransposeReturnType(derived()); } /** \internal the return type of adjoint() */ typedef typename internal::conditional::IsComplex, CwiseUnaryOp, ConstTransposeReturnType>, ConstTransposeReturnType >::type AdjointReturnType; /** \returns an expression of the adjoint of the factored matrix * * A typical usage is to solve for the adjoint problem A' x = b: * \code x = dec.adjoint().solve(b); \endcode * * For real scalar types, this function is equivalent to transpose(). * * \sa transpose(), solve() */ inline AdjointReturnType adjoint() const { return AdjointReturnType(derived().transpose()); } protected: template void _check_solve_assertion(const Rhs& b) const { EIGEN_ONLY_USED_FOR_DEBUG(b); eigen_assert(derived().m_isInitialized && "Solver is not initialized."); eigen_assert((Transpose_?derived().cols():derived().rows())==b.rows() && "SolverBase::solve(): invalid number of rows of the right hand side matrix b"); } }; namespace internal { template struct generic_xpr_base { typedef SolverBase type; }; } // end namespace internal } // end namespace Eigen #endif // EIGEN_SOLVERBASE_H RcppEigen/inst/include/Eigen/src/Core/Array.h0000644000176200001440000004061614567757725020523 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2009 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_ARRAY_H #define EIGEN_ARRAY_H namespace Eigen { namespace internal { template struct traits > : traits > { typedef ArrayXpr XprKind; typedef ArrayBase > XprBase; }; } /** \class Array * \ingroup Core_Module * * \brief General-purpose arrays with easy API for coefficient-wise operations * * The %Array class is very similar to the Matrix class. It provides * general-purpose one- and two-dimensional arrays. The difference between the * %Array and the %Matrix class is primarily in the API: the API for the * %Array class provides easy access to coefficient-wise operations, while the * API for the %Matrix class provides easy access to linear-algebra * operations. * * See documentation of class Matrix for detailed information on the template parameters * storage layout. * * This class can be extended with the help of the plugin mechanism described on the page * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_ARRAY_PLUGIN. * * \sa \blank \ref TutorialArrayClass, \ref TopicClassHierarchy */ template class Array : public PlainObjectBase > { public: typedef PlainObjectBase Base; EIGEN_DENSE_PUBLIC_INTERFACE(Array) enum { Options = _Options }; typedef typename Base::PlainObject PlainObject; protected: template friend struct internal::conservative_resize_like_impl; using Base::m_storage; public: using Base::base; using Base::coeff; using Base::coeffRef; /** * The usage of * using Base::operator=; * fails on MSVC. Since the code below is working with GCC and MSVC, we skipped * the usage of 'using'. This should be done only for operator=. */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array& operator=(const EigenBase &other) { return Base::operator=(other); } /** Set all the entries to \a value. * \sa DenseBase::setConstant(), DenseBase::fill() */ /* This overload is needed because the usage of * using Base::operator=; * fails on MSVC. Since the code below is working with GCC and MSVC, we skipped * the usage of 'using'. This should be done only for operator=. */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array& operator=(const Scalar &value) { Base::setConstant(value); return *this; } /** Copies the value of the expression \a other into \c *this with automatic resizing. * * *this might be resized to match the dimensions of \a other. If *this was a null matrix (not already initialized), * it will be initialized. * * Note that copying a row-vector into a vector (and conversely) is allowed. * The resizing, if any, is then done in the appropriate way so that row-vectors * remain row-vectors and vectors remain vectors. */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array& operator=(const DenseBase& other) { return Base::_set(other); } /** This is a special case of the templated operator=. Its purpose is to * prevent a default operator= from hiding the templated operator=. */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array& operator=(const Array& other) { return Base::_set(other); } /** Default constructor. * * For fixed-size matrices, does nothing. * * For dynamic-size matrices, creates an empty matrix of size 0. Does not allocate any array. Such a matrix * is called a null matrix. This constructor is the unique way to create null matrices: resizing * a matrix to 0 is not supported. * * \sa resize(Index,Index) */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array() : Base() { Base::_check_template_params(); EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } #ifndef EIGEN_PARSED_BY_DOXYGEN // FIXME is it still needed ?? /** \internal */ EIGEN_DEVICE_FUNC Array(internal::constructor_without_unaligned_array_assert) : Base(internal::constructor_without_unaligned_array_assert()) { Base::_check_template_params(); EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } #endif #if EIGEN_HAS_RVALUE_REFERENCES EIGEN_DEVICE_FUNC Array(Array&& other) EIGEN_NOEXCEPT_IF(std::is_nothrow_move_constructible::value) : Base(std::move(other)) { Base::_check_template_params(); } EIGEN_DEVICE_FUNC Array& operator=(Array&& other) EIGEN_NOEXCEPT_IF(std::is_nothrow_move_assignable::value) { Base::operator=(std::move(other)); return *this; } #endif #if EIGEN_HAS_CXX11 /** \copydoc PlainObjectBase(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) * * Example: \include Array_variadic_ctor_cxx11.cpp * Output: \verbinclude Array_variadic_ctor_cxx11.out * * \sa Array(const std::initializer_list>&) * \sa Array(const Scalar&), Array(const Scalar&,const Scalar&) */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) : Base(a0, a1, a2, a3, args...) {} /** \brief Constructs an array and initializes it from the coefficients given as initializer-lists grouped by row. \cpp11 * * In the general case, the constructor takes a list of rows, each row being represented as a list of coefficients: * * Example: \include Array_initializer_list_23_cxx11.cpp * Output: \verbinclude Array_initializer_list_23_cxx11.out * * Each of the inner initializer lists must contain the exact same number of elements, otherwise an assertion is triggered. * * In the case of a compile-time column 1D array, implicit transposition from a single row is allowed. * Therefore Array{{1,2,3,4,5}} is legal and the more verbose syntax * Array{{1},{2},{3},{4},{5}} can be avoided: * * Example: \include Array_initializer_list_vector_cxx11.cpp * Output: \verbinclude Array_initializer_list_vector_cxx11.out * * In the case of fixed-sized arrays, the initializer list sizes must exactly match the array sizes, * and implicit transposition is allowed for compile-time 1D arrays only. * * \sa Array(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const std::initializer_list>& list) : Base(list) {} #endif // end EIGEN_HAS_CXX11 #ifndef EIGEN_PARSED_BY_DOXYGEN template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Array(const T& x) { Base::_check_template_params(); Base::template _init1(x); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const T0& val0, const T1& val1) { Base::_check_template_params(); this->template _init2(val0, val1); } #else /** \brief Constructs a fixed-sized array initialized with coefficients starting at \a data */ EIGEN_DEVICE_FUNC explicit Array(const Scalar *data); /** Constructs a vector or row-vector with given dimension. \only_for_vectors * * Note that this is only useful for dynamic-size vectors. For fixed-size vectors, * it is redundant to pass the dimension here, so it makes more sense to use the default * constructor Array() instead. */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Array(Index dim); /** constructs an initialized 1x1 Array with the given coefficient * \sa const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args */ Array(const Scalar& value); /** constructs an uninitialized array with \a rows rows and \a cols columns. * * This is useful for dynamic-size arrays. For fixed-size arrays, * it is redundant to pass these parameters, so one should use the default constructor * Array() instead. */ Array(Index rows, Index cols); /** constructs an initialized 2D vector with given coefficients * \sa Array(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) */ Array(const Scalar& val0, const Scalar& val1); #endif // end EIGEN_PARSED_BY_DOXYGEN /** constructs an initialized 3D vector with given coefficients * \sa Array(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const Scalar& val0, const Scalar& val1, const Scalar& val2) { Base::_check_template_params(); EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Array, 3) m_storage.data()[0] = val0; m_storage.data()[1] = val1; m_storage.data()[2] = val2; } /** constructs an initialized 4D vector with given coefficients * \sa Array(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const Scalar& val0, const Scalar& val1, const Scalar& val2, const Scalar& val3) { Base::_check_template_params(); EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Array, 4) m_storage.data()[0] = val0; m_storage.data()[1] = val1; m_storage.data()[2] = val2; m_storage.data()[3] = val3; } /** Copy constructor */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const Array& other) : Base(other) { } private: struct PrivateType {}; public: /** \sa MatrixBase::operator=(const EigenBase&) */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const EigenBase &other, typename internal::enable_if::value, PrivateType>::type = PrivateType()) : Base(other.derived()) { } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT{ return 1; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT { return this->innerSize(); } #ifdef EIGEN_ARRAY_PLUGIN #include EIGEN_ARRAY_PLUGIN #endif private: template friend struct internal::matrix_swap_impl; }; /** \defgroup arraytypedefs Global array typedefs * \ingroup Core_Module * * %Eigen defines several typedef shortcuts for most common 1D and 2D array types. * * The general patterns are the following: * * \c ArrayRowsColsType where \c Rows and \c Cols can be \c 2,\c 3,\c 4 for fixed size square matrices or \c X for dynamic size, * and where \c Type can be \c i for integer, \c f for float, \c d for double, \c cf for complex float, \c cd * for complex double. * * For example, \c Array33d is a fixed-size 3x3 array type of doubles, and \c ArrayXXf is a dynamic-size matrix of floats. * * There are also \c ArraySizeType which are self-explanatory. For example, \c Array4cf is * a fixed-size 1D array of 4 complex floats. * * With \cpp11, template alias are also defined for common sizes. * They follow the same pattern as above except that the scalar type suffix is replaced by a * template parameter, i.e.: * - `ArrayRowsCols` where `Rows` and `Cols` can be \c 2,\c 3,\c 4, or \c X for fixed or dynamic size. * - `ArraySize` where `Size` can be \c 2,\c 3,\c 4 or \c X for fixed or dynamic size 1D arrays. * * \sa class Array */ #define EIGEN_MAKE_ARRAY_TYPEDEFS(Type, TypeSuffix, Size, SizeSuffix) \ /** \ingroup arraytypedefs */ \ typedef Array Array##SizeSuffix##SizeSuffix##TypeSuffix; \ /** \ingroup arraytypedefs */ \ typedef Array Array##SizeSuffix##TypeSuffix; #define EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(Type, TypeSuffix, Size) \ /** \ingroup arraytypedefs */ \ typedef Array Array##Size##X##TypeSuffix; \ /** \ingroup arraytypedefs */ \ typedef Array Array##X##Size##TypeSuffix; #define EIGEN_MAKE_ARRAY_TYPEDEFS_ALL_SIZES(Type, TypeSuffix) \ EIGEN_MAKE_ARRAY_TYPEDEFS(Type, TypeSuffix, 2, 2) \ EIGEN_MAKE_ARRAY_TYPEDEFS(Type, TypeSuffix, 3, 3) \ EIGEN_MAKE_ARRAY_TYPEDEFS(Type, TypeSuffix, 4, 4) \ EIGEN_MAKE_ARRAY_TYPEDEFS(Type, TypeSuffix, Dynamic, X) \ EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(Type, TypeSuffix, 2) \ EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(Type, TypeSuffix, 3) \ EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(Type, TypeSuffix, 4) EIGEN_MAKE_ARRAY_TYPEDEFS_ALL_SIZES(int, i) EIGEN_MAKE_ARRAY_TYPEDEFS_ALL_SIZES(float, f) EIGEN_MAKE_ARRAY_TYPEDEFS_ALL_SIZES(double, d) EIGEN_MAKE_ARRAY_TYPEDEFS_ALL_SIZES(std::complex, cf) EIGEN_MAKE_ARRAY_TYPEDEFS_ALL_SIZES(std::complex, cd) #undef EIGEN_MAKE_ARRAY_TYPEDEFS_ALL_SIZES #undef EIGEN_MAKE_ARRAY_TYPEDEFS #undef EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS #if EIGEN_HAS_CXX11 #define EIGEN_MAKE_ARRAY_TYPEDEFS(Size, SizeSuffix) \ /** \ingroup arraytypedefs */ \ /** \brief \cpp11 */ \ template \ using Array##SizeSuffix##SizeSuffix = Array; \ /** \ingroup arraytypedefs */ \ /** \brief \cpp11 */ \ template \ using Array##SizeSuffix = Array; #define EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(Size) \ /** \ingroup arraytypedefs */ \ /** \brief \cpp11 */ \ template \ using Array##Size##X = Array; \ /** \ingroup arraytypedefs */ \ /** \brief \cpp11 */ \ template \ using Array##X##Size = Array; EIGEN_MAKE_ARRAY_TYPEDEFS(2, 2) EIGEN_MAKE_ARRAY_TYPEDEFS(3, 3) EIGEN_MAKE_ARRAY_TYPEDEFS(4, 4) EIGEN_MAKE_ARRAY_TYPEDEFS(Dynamic, X) EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(2) EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(3) EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(4) #undef EIGEN_MAKE_ARRAY_TYPEDEFS #undef EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS #endif // EIGEN_HAS_CXX11 #define EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE_AND_SIZE(TypeSuffix, SizeSuffix) \ using Eigen::Matrix##SizeSuffix##TypeSuffix; \ using Eigen::Vector##SizeSuffix##TypeSuffix; \ using Eigen::RowVector##SizeSuffix##TypeSuffix; #define EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE(TypeSuffix) \ EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE_AND_SIZE(TypeSuffix, 2) \ EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE_AND_SIZE(TypeSuffix, 3) \ EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE_AND_SIZE(TypeSuffix, 4) \ EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE_AND_SIZE(TypeSuffix, X) \ #define EIGEN_USING_ARRAY_TYPEDEFS \ EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE(i) \ EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE(f) \ EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE(d) \ EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE(cf) \ EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE(cd) } // end namespace Eigen #endif // EIGEN_ARRAY_H RcppEigen/inst/include/Eigen/src/Core/ReturnByValue.h0000644000176200001440000001027414567757725022211 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2009-2010 Gael Guennebaud // Copyright (C) 2009-2010 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_RETURNBYVALUE_H #define EIGEN_RETURNBYVALUE_H namespace Eigen { namespace internal { template struct traits > : public traits::ReturnType> { enum { // We're disabling the DirectAccess because e.g. the constructor of // the Block-with-DirectAccess expression requires to have a coeffRef method. // Also, we don't want to have to implement the stride stuff. Flags = (traits::ReturnType>::Flags | EvalBeforeNestingBit) & ~DirectAccessBit }; }; /* The ReturnByValue object doesn't even have a coeff() method. * So the only way that nesting it in an expression can work, is by evaluating it into a plain matrix. * So internal::nested always gives the plain return matrix type. * * FIXME: I don't understand why we need this specialization: isn't this taken care of by the EvalBeforeNestingBit ?? * Answer: EvalBeforeNestingBit should be deprecated since we have the evaluators */ template struct nested_eval, n, PlainObject> { typedef typename traits::ReturnType type; }; } // end namespace internal /** \class ReturnByValue * \ingroup Core_Module * */ template class ReturnByValue : public internal::dense_xpr_base< ReturnByValue >::type, internal::no_assignment_operator { public: typedef typename internal::traits::ReturnType ReturnType; typedef typename internal::dense_xpr_base::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(ReturnByValue) template EIGEN_DEVICE_FUNC inline void evalTo(Dest& dst) const { static_cast(this)->evalTo(dst); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return static_cast(this)->rows(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return static_cast(this)->cols(); } #ifndef EIGEN_PARSED_BY_DOXYGEN #define Unusable YOU_ARE_TRYING_TO_ACCESS_A_SINGLE_COEFFICIENT_IN_A_SPECIAL_EXPRESSION_WHERE_THAT_IS_NOT_ALLOWED_BECAUSE_THAT_WOULD_BE_INEFFICIENT class Unusable{ Unusable(const Unusable&) {} Unusable& operator=(const Unusable&) {return *this;} }; const Unusable& coeff(Index) const { return *reinterpret_cast(this); } const Unusable& coeff(Index,Index) const { return *reinterpret_cast(this); } Unusable& coeffRef(Index) { return *reinterpret_cast(this); } Unusable& coeffRef(Index,Index) { return *reinterpret_cast(this); } #undef Unusable #endif }; template template EIGEN_DEVICE_FUNC Derived& DenseBase::operator=(const ReturnByValue& other) { other.evalTo(derived()); return derived(); } namespace internal { // Expression is evaluated in a temporary; default implementation of Assignment is bypassed so that // when a ReturnByValue expression is assigned, the evaluator is not constructed. // TODO: Finalize port to new regime; ReturnByValue should not exist in the expression world template struct evaluator > : public evaluator::ReturnType> { typedef ReturnByValue XprType; typedef typename internal::traits::ReturnType PlainObject; typedef evaluator Base; EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) : m_result(xpr.rows(), xpr.cols()) { ::new (static_cast(this)) Base(m_result); xpr.evalTo(m_result); } protected: PlainObject m_result; }; } // end namespace internal } // end namespace Eigen #endif // EIGEN_RETURNBYVALUE_H RcppEigen/inst/include/Eigen/src/Core/StableNorm.h0000644000176200001440000002077414567757725021516 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2009 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_STABLENORM_H #define EIGEN_STABLENORM_H namespace Eigen { namespace internal { template inline void stable_norm_kernel(const ExpressionType& bl, Scalar& ssq, Scalar& scale, Scalar& invScale) { Scalar maxCoeff = bl.cwiseAbs().maxCoeff(); if(maxCoeff>scale) { ssq = ssq * numext::abs2(scale/maxCoeff); Scalar tmp = Scalar(1)/maxCoeff; if(tmp > NumTraits::highest()) { invScale = NumTraits::highest(); scale = Scalar(1)/invScale; } else if(maxCoeff>NumTraits::highest()) // we got a INF { invScale = Scalar(1); scale = maxCoeff; } else { scale = maxCoeff; invScale = tmp; } } else if(maxCoeff!=maxCoeff) // we got a NaN { scale = maxCoeff; } // TODO if the maxCoeff is much much smaller than the current scale, // then we can neglect this sub vector if(scale>Scalar(0)) // if scale==0, then bl is 0 ssq += (bl*invScale).squaredNorm(); } template void stable_norm_impl_inner_step(const VectorType &vec, RealScalar& ssq, RealScalar& scale, RealScalar& invScale) { typedef typename VectorType::Scalar Scalar; const Index blockSize = 4096; typedef typename internal::nested_eval::type VectorTypeCopy; typedef typename internal::remove_all::type VectorTypeCopyClean; const VectorTypeCopy copy(vec); enum { CanAlign = ( (int(VectorTypeCopyClean::Flags)&DirectAccessBit) || (int(internal::evaluator::Alignment)>0) // FIXME Alignment)>0 might not be enough ) && (blockSize*sizeof(Scalar)*20) // if we cannot allocate on the stack, then let's not bother about this optimization }; typedef typename internal::conditional, internal::evaluator::Alignment>, typename VectorTypeCopyClean::ConstSegmentReturnType>::type SegmentWrapper; Index n = vec.size(); Index bi = internal::first_default_aligned(copy); if (bi>0) internal::stable_norm_kernel(copy.head(bi), ssq, scale, invScale); for (; bi typename VectorType::RealScalar stable_norm_impl(const VectorType &vec, typename enable_if::type* = 0 ) { using std::sqrt; using std::abs; Index n = vec.size(); if(n==1) return abs(vec.coeff(0)); typedef typename VectorType::RealScalar RealScalar; RealScalar scale(0); RealScalar invScale(1); RealScalar ssq(0); // sum of squares stable_norm_impl_inner_step(vec, ssq, scale, invScale); return scale * sqrt(ssq); } template typename MatrixType::RealScalar stable_norm_impl(const MatrixType &mat, typename enable_if::type* = 0 ) { using std::sqrt; typedef typename MatrixType::RealScalar RealScalar; RealScalar scale(0); RealScalar invScale(1); RealScalar ssq(0); // sum of squares for(Index j=0; j inline typename NumTraits::Scalar>::Real blueNorm_impl(const EigenBase& _vec) { typedef typename Derived::RealScalar RealScalar; using std::pow; using std::sqrt; using std::abs; // This program calculates the machine-dependent constants // bl, b2, slm, s2m, relerr overfl // from the "basic" machine-dependent numbers // nbig, ibeta, it, iemin, iemax, rbig. // The following define the basic machine-dependent constants. // For portability, the PORT subprograms "ilmaeh" and "rlmach" // are used. For any specific computer, each of the assignment // statements can be replaced static const int ibeta = std::numeric_limits::radix; // base for floating-point numbers static const int it = NumTraits::digits(); // number of base-beta digits in mantissa static const int iemin = NumTraits::min_exponent(); // minimum exponent static const int iemax = NumTraits::max_exponent(); // maximum exponent static const RealScalar rbig = NumTraits::highest(); // largest floating-point number static const RealScalar b1 = RealScalar(pow(RealScalar(ibeta),RealScalar(-((1-iemin)/2)))); // lower boundary of midrange static const RealScalar b2 = RealScalar(pow(RealScalar(ibeta),RealScalar((iemax + 1 - it)/2))); // upper boundary of midrange static const RealScalar s1m = RealScalar(pow(RealScalar(ibeta),RealScalar((2-iemin)/2))); // scaling factor for lower range static const RealScalar s2m = RealScalar(pow(RealScalar(ibeta),RealScalar(- ((iemax+it)/2)))); // scaling factor for upper range static const RealScalar eps = RealScalar(pow(double(ibeta), 1-it)); static const RealScalar relerr = sqrt(eps); // tolerance for neglecting asml const Derived& vec(_vec.derived()); Index n = vec.size(); RealScalar ab2 = b2 / RealScalar(n); RealScalar asml = RealScalar(0); RealScalar amed = RealScalar(0); RealScalar abig = RealScalar(0); for(Index j=0; j ab2) abig += numext::abs2(ax*s2m); else if(ax < b1) asml += numext::abs2(ax*s1m); else amed += numext::abs2(ax); } } if(amed!=amed) return amed; // we got a NaN if(abig > RealScalar(0)) { abig = sqrt(abig); if(abig > rbig) // overflow, or *this contains INF values return abig; // return INF if(amed > RealScalar(0)) { abig = abig/s2m; amed = sqrt(amed); } else return abig/s2m; } else if(asml > RealScalar(0)) { if (amed > RealScalar(0)) { abig = sqrt(amed); amed = sqrt(asml) / s1m; } else return sqrt(asml)/s1m; } else return sqrt(amed); asml = numext::mini(abig, amed); abig = numext::maxi(abig, amed); if(asml <= abig*relerr) return abig; else return abig * sqrt(RealScalar(1) + numext::abs2(asml/abig)); } } // end namespace internal /** \returns the \em l2 norm of \c *this avoiding underflow and overflow. * This version use a blockwise two passes algorithm: * 1 - find the absolute largest coefficient \c s * 2 - compute \f$ s \Vert \frac{*this}{s} \Vert \f$ in a standard way * * For architecture/scalar types supporting vectorization, this version * is faster than blueNorm(). Otherwise the blueNorm() is much faster. * * \sa norm(), blueNorm(), hypotNorm() */ template inline typename NumTraits::Scalar>::Real MatrixBase::stableNorm() const { return internal::stable_norm_impl(derived()); } /** \returns the \em l2 norm of \c *this using the Blue's algorithm. * A Portable Fortran Program to Find the Euclidean Norm of a Vector, * ACM TOMS, Vol 4, Issue 1, 1978. * * For architecture/scalar types without vectorization, this version * is much faster than stableNorm(). Otherwise the stableNorm() is faster. * * \sa norm(), stableNorm(), hypotNorm() */ template inline typename NumTraits::Scalar>::Real MatrixBase::blueNorm() const { return internal::blueNorm_impl(*this); } /** \returns the \em l2 norm of \c *this avoiding undeflow and overflow. * This version use a concatenation of hypot() calls, and it is very slow. * * \sa norm(), stableNorm() */ template inline typename NumTraits::Scalar>::Real MatrixBase::hypotNorm() const { if(size()==1) return numext::abs(coeff(0,0)); else return this->cwiseAbs().redux(internal::scalar_hypot_op()); } } // end namespace Eigen #endif // EIGEN_STABLENORM_H RcppEigen/inst/include/Eigen/src/Core/TriangularMatrix.h0000644000176200001440000011260514567757725022740 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008 Benoit Jacob // Copyright (C) 2008-2009 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_TRIANGULARMATRIX_H #define EIGEN_TRIANGULARMATRIX_H namespace Eigen { namespace internal { template struct triangular_solve_retval; } /** \class TriangularBase * \ingroup Core_Module * * \brief Base class for triangular part in a matrix */ template class TriangularBase : public EigenBase { public: enum { Mode = internal::traits::Mode, RowsAtCompileTime = internal::traits::RowsAtCompileTime, ColsAtCompileTime = internal::traits::ColsAtCompileTime, MaxRowsAtCompileTime = internal::traits::MaxRowsAtCompileTime, MaxColsAtCompileTime = internal::traits::MaxColsAtCompileTime, SizeAtCompileTime = (internal::size_at_compile_time::RowsAtCompileTime, internal::traits::ColsAtCompileTime>::ret), /**< This is equal to the number of coefficients, i.e. the number of * rows times the number of columns, or to \a Dynamic if this is not * known at compile-time. \sa RowsAtCompileTime, ColsAtCompileTime */ MaxSizeAtCompileTime = (internal::size_at_compile_time::MaxRowsAtCompileTime, internal::traits::MaxColsAtCompileTime>::ret) }; typedef typename internal::traits::Scalar Scalar; typedef typename internal::traits::StorageKind StorageKind; typedef typename internal::traits::StorageIndex StorageIndex; typedef typename internal::traits::FullMatrixType DenseMatrixType; typedef DenseMatrixType DenseType; typedef Derived const& Nested; EIGEN_DEVICE_FUNC inline TriangularBase() { eigen_assert(!((int(Mode) & int(UnitDiag)) && (int(Mode) & int(ZeroDiag)))); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return derived().rows(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return derived().cols(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT { return derived().outerStride(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT { return derived().innerStride(); } // dummy resize function EIGEN_DEVICE_FUNC void resize(Index rows, Index cols) { EIGEN_UNUSED_VARIABLE(rows); EIGEN_UNUSED_VARIABLE(cols); eigen_assert(rows==this->rows() && cols==this->cols()); } EIGEN_DEVICE_FUNC inline Scalar coeff(Index row, Index col) const { return derived().coeff(row,col); } EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index row, Index col) { return derived().coeffRef(row,col); } /** \see MatrixBase::copyCoeff(row,col) */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void copyCoeff(Index row, Index col, Other& other) { derived().coeffRef(row, col) = other.coeff(row, col); } EIGEN_DEVICE_FUNC inline Scalar operator()(Index row, Index col) const { check_coordinates(row, col); return coeff(row,col); } EIGEN_DEVICE_FUNC inline Scalar& operator()(Index row, Index col) { check_coordinates(row, col); return coeffRef(row,col); } #ifndef EIGEN_PARSED_BY_DOXYGEN EIGEN_DEVICE_FUNC inline const Derived& derived() const { return *static_cast(this); } EIGEN_DEVICE_FUNC inline Derived& derived() { return *static_cast(this); } #endif // not EIGEN_PARSED_BY_DOXYGEN template EIGEN_DEVICE_FUNC void evalTo(MatrixBase &other) const; template EIGEN_DEVICE_FUNC void evalToLazy(MatrixBase &other) const; EIGEN_DEVICE_FUNC DenseMatrixType toDenseMatrix() const { DenseMatrixType res(rows(), cols()); evalToLazy(res); return res; } protected: void check_coordinates(Index row, Index col) const { EIGEN_ONLY_USED_FOR_DEBUG(row); EIGEN_ONLY_USED_FOR_DEBUG(col); eigen_assert(col>=0 && col=0 && row=row) || (mode==Lower && col<=row) || ((mode==StrictlyUpper || mode==UnitUpper) && col>row) || ((mode==StrictlyLower || mode==UnitLower) && col struct traits > : traits { typedef typename ref_selector::non_const_type MatrixTypeNested; typedef typename remove_reference::type MatrixTypeNestedNonRef; typedef typename remove_all::type MatrixTypeNestedCleaned; typedef typename MatrixType::PlainObject FullMatrixType; typedef MatrixType ExpressionType; enum { Mode = _Mode, FlagsLvalueBit = is_lvalue::value ? LvalueBit : 0, Flags = (MatrixTypeNestedCleaned::Flags & (HereditaryBits | FlagsLvalueBit) & (~(PacketAccessBit | DirectAccessBit | LinearAccessBit))) }; }; } template class TriangularViewImpl; template class TriangularView : public TriangularViewImpl<_MatrixType, _Mode, typename internal::traits<_MatrixType>::StorageKind > { public: typedef TriangularViewImpl<_MatrixType, _Mode, typename internal::traits<_MatrixType>::StorageKind > Base; typedef typename internal::traits::Scalar Scalar; typedef _MatrixType MatrixType; protected: typedef typename internal::traits::MatrixTypeNested MatrixTypeNested; typedef typename internal::traits::MatrixTypeNestedNonRef MatrixTypeNestedNonRef; typedef typename internal::remove_all::type MatrixConjugateReturnType; typedef TriangularView::type, _Mode> ConstTriangularView; public: typedef typename internal::traits::StorageKind StorageKind; typedef typename internal::traits::MatrixTypeNestedCleaned NestedExpression; enum { Mode = _Mode, Flags = internal::traits::Flags, TransposeMode = (Mode & Upper ? Lower : 0) | (Mode & Lower ? Upper : 0) | (Mode & (UnitDiag)) | (Mode & (ZeroDiag)), IsVectorAtCompileTime = false }; EIGEN_DEVICE_FUNC explicit inline TriangularView(MatrixType& matrix) : m_matrix(matrix) {} EIGEN_INHERIT_ASSIGNMENT_OPERATORS(TriangularView) /** \copydoc EigenBase::rows() */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return m_matrix.rows(); } /** \copydoc EigenBase::cols() */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return m_matrix.cols(); } /** \returns a const reference to the nested expression */ EIGEN_DEVICE_FUNC const NestedExpression& nestedExpression() const { return m_matrix; } /** \returns a reference to the nested expression */ EIGEN_DEVICE_FUNC NestedExpression& nestedExpression() { return m_matrix; } typedef TriangularView ConjugateReturnType; /** \sa MatrixBase::conjugate() const */ EIGEN_DEVICE_FUNC inline const ConjugateReturnType conjugate() const { return ConjugateReturnType(m_matrix.conjugate()); } /** \returns an expression of the complex conjugate of \c *this if Cond==true, * returns \c *this otherwise. */ template EIGEN_DEVICE_FUNC inline typename internal::conditional::type conjugateIf() const { typedef typename internal::conditional::type ReturnType; return ReturnType(m_matrix.template conjugateIf()); } typedef TriangularView AdjointReturnType; /** \sa MatrixBase::adjoint() const */ EIGEN_DEVICE_FUNC inline const AdjointReturnType adjoint() const { return AdjointReturnType(m_matrix.adjoint()); } typedef TriangularView TransposeReturnType; /** \sa MatrixBase::transpose() */ EIGEN_DEVICE_FUNC inline TransposeReturnType transpose() { EIGEN_STATIC_ASSERT_LVALUE(MatrixType) typename MatrixType::TransposeReturnType tmp(m_matrix); return TransposeReturnType(tmp); } typedef TriangularView ConstTransposeReturnType; /** \sa MatrixBase::transpose() const */ EIGEN_DEVICE_FUNC inline const ConstTransposeReturnType transpose() const { return ConstTransposeReturnType(m_matrix.transpose()); } template EIGEN_DEVICE_FUNC inline const Solve solve(const MatrixBase& other) const { return Solve(*this, other.derived()); } // workaround MSVC ICE #if EIGEN_COMP_MSVC template EIGEN_DEVICE_FUNC inline const internal::triangular_solve_retval solve(const MatrixBase& other) const { return Base::template solve(other); } #else using Base::solve; #endif /** \returns a selfadjoint view of the referenced triangular part which must be either \c #Upper or \c #Lower. * * This is a shortcut for \code this->nestedExpression().selfadjointView<(*this)::Mode>() \endcode * \sa MatrixBase::selfadjointView() */ EIGEN_DEVICE_FUNC SelfAdjointView selfadjointView() { EIGEN_STATIC_ASSERT((Mode&(UnitDiag|ZeroDiag))==0,PROGRAMMING_ERROR); return SelfAdjointView(m_matrix); } /** This is the const version of selfadjointView() */ EIGEN_DEVICE_FUNC const SelfAdjointView selfadjointView() const { EIGEN_STATIC_ASSERT((Mode&(UnitDiag|ZeroDiag))==0,PROGRAMMING_ERROR); return SelfAdjointView(m_matrix); } /** \returns the determinant of the triangular matrix * \sa MatrixBase::determinant() */ EIGEN_DEVICE_FUNC Scalar determinant() const { if (Mode & UnitDiag) return 1; else if (Mode & ZeroDiag) return 0; else return m_matrix.diagonal().prod(); } protected: MatrixTypeNested m_matrix; }; /** \ingroup Core_Module * * \brief Base class for a triangular part in a \b dense matrix * * This class is an abstract base class of class TriangularView, and objects of type TriangularViewImpl cannot be instantiated. * It extends class TriangularView with additional methods which available for dense expressions only. * * \sa class TriangularView, MatrixBase::triangularView() */ template class TriangularViewImpl<_MatrixType,_Mode,Dense> : public TriangularBase > { public: typedef TriangularView<_MatrixType, _Mode> TriangularViewType; typedef TriangularBase Base; typedef typename internal::traits::Scalar Scalar; typedef _MatrixType MatrixType; typedef typename MatrixType::PlainObject DenseMatrixType; typedef DenseMatrixType PlainObject; public: using Base::evalToLazy; using Base::derived; typedef typename internal::traits::StorageKind StorageKind; enum { Mode = _Mode, Flags = internal::traits::Flags }; /** \returns the outer-stride of the underlying dense matrix * \sa DenseCoeffsBase::outerStride() */ EIGEN_DEVICE_FUNC inline Index outerStride() const { return derived().nestedExpression().outerStride(); } /** \returns the inner-stride of the underlying dense matrix * \sa DenseCoeffsBase::innerStride() */ EIGEN_DEVICE_FUNC inline Index innerStride() const { return derived().nestedExpression().innerStride(); } /** \sa MatrixBase::operator+=() */ template EIGEN_DEVICE_FUNC TriangularViewType& operator+=(const DenseBase& other) { internal::call_assignment_no_alias(derived(), other.derived(), internal::add_assign_op()); return derived(); } /** \sa MatrixBase::operator-=() */ template EIGEN_DEVICE_FUNC TriangularViewType& operator-=(const DenseBase& other) { internal::call_assignment_no_alias(derived(), other.derived(), internal::sub_assign_op()); return derived(); } /** \sa MatrixBase::operator*=() */ EIGEN_DEVICE_FUNC TriangularViewType& operator*=(const typename internal::traits::Scalar& other) { return *this = derived().nestedExpression() * other; } /** \sa DenseBase::operator/=() */ EIGEN_DEVICE_FUNC TriangularViewType& operator/=(const typename internal::traits::Scalar& other) { return *this = derived().nestedExpression() / other; } /** \sa MatrixBase::fill() */ EIGEN_DEVICE_FUNC void fill(const Scalar& value) { setConstant(value); } /** \sa MatrixBase::setConstant() */ EIGEN_DEVICE_FUNC TriangularViewType& setConstant(const Scalar& value) { return *this = MatrixType::Constant(derived().rows(), derived().cols(), value); } /** \sa MatrixBase::setZero() */ EIGEN_DEVICE_FUNC TriangularViewType& setZero() { return setConstant(Scalar(0)); } /** \sa MatrixBase::setOnes() */ EIGEN_DEVICE_FUNC TriangularViewType& setOnes() { return setConstant(Scalar(1)); } /** \sa MatrixBase::coeff() * \warning the coordinates must fit into the referenced triangular part */ EIGEN_DEVICE_FUNC inline Scalar coeff(Index row, Index col) const { Base::check_coordinates_internal(row, col); return derived().nestedExpression().coeff(row, col); } /** \sa MatrixBase::coeffRef() * \warning the coordinates must fit into the referenced triangular part */ EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index row, Index col) { EIGEN_STATIC_ASSERT_LVALUE(TriangularViewType); Base::check_coordinates_internal(row, col); return derived().nestedExpression().coeffRef(row, col); } /** Assigns a triangular matrix to a triangular part of a dense matrix */ template EIGEN_DEVICE_FUNC TriangularViewType& operator=(const TriangularBase& other); /** Shortcut for\code *this = other.other.triangularView<(*this)::Mode>() \endcode */ template EIGEN_DEVICE_FUNC TriangularViewType& operator=(const MatrixBase& other); #ifndef EIGEN_PARSED_BY_DOXYGEN EIGEN_DEVICE_FUNC TriangularViewType& operator=(const TriangularViewImpl& other) { return *this = other.derived().nestedExpression(); } template /** \deprecated */ EIGEN_DEPRECATED EIGEN_DEVICE_FUNC void lazyAssign(const TriangularBase& other); template /** \deprecated */ EIGEN_DEPRECATED EIGEN_DEVICE_FUNC void lazyAssign(const MatrixBase& other); #endif /** Efficient triangular matrix times vector/matrix product */ template EIGEN_DEVICE_FUNC const Product operator*(const MatrixBase& rhs) const { return Product(derived(), rhs.derived()); } /** Efficient vector/matrix times triangular matrix product */ template friend EIGEN_DEVICE_FUNC const Product operator*(const MatrixBase& lhs, const TriangularViewImpl& rhs) { return Product(lhs.derived(),rhs.derived()); } /** \returns the product of the inverse of \c *this with \a other, \a *this being triangular. * * This function computes the inverse-matrix matrix product inverse(\c *this) * \a other if * \a Side==OnTheLeft (the default), or the right-inverse-multiply \a other * inverse(\c *this) if * \a Side==OnTheRight. * * Note that the template parameter \c Side can be omitted, in which case \c Side==OnTheLeft * * The matrix \c *this must be triangular and invertible (i.e., all the coefficients of the * diagonal must be non zero). It works as a forward (resp. backward) substitution if \c *this * is an upper (resp. lower) triangular matrix. * * Example: \include Triangular_solve.cpp * Output: \verbinclude Triangular_solve.out * * This function returns an expression of the inverse-multiply and can works in-place if it is assigned * to the same matrix or vector \a other. * * For users coming from BLAS, this function (and more specifically solveInPlace()) offer * all the operations supported by the \c *TRSV and \c *TRSM BLAS routines. * * \sa TriangularView::solveInPlace() */ template inline const internal::triangular_solve_retval solve(const MatrixBase& other) const; /** "in-place" version of TriangularView::solve() where the result is written in \a other * * \warning The parameter is only marked 'const' to make the C++ compiler accept a temporary expression here. * This function will const_cast it, so constness isn't honored here. * * Note that the template parameter \c Side can be omitted, in which case \c Side==OnTheLeft * * See TriangularView:solve() for the details. */ template EIGEN_DEVICE_FUNC void solveInPlace(const MatrixBase& other) const; template EIGEN_DEVICE_FUNC void solveInPlace(const MatrixBase& other) const { return solveInPlace(other); } /** Swaps the coefficients of the common triangular parts of two matrices */ template EIGEN_DEVICE_FUNC #ifdef EIGEN_PARSED_BY_DOXYGEN void swap(TriangularBase &other) #else void swap(TriangularBase const & other) #endif { EIGEN_STATIC_ASSERT_LVALUE(OtherDerived); call_assignment(derived(), other.const_cast_derived(), internal::swap_assign_op()); } /** Shortcut for \code (*this).swap(other.triangularView<(*this)::Mode>()) \endcode */ template /** \deprecated */ EIGEN_DEPRECATED EIGEN_DEVICE_FUNC void swap(MatrixBase const & other) { EIGEN_STATIC_ASSERT_LVALUE(OtherDerived); call_assignment(derived(), other.const_cast_derived(), internal::swap_assign_op()); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void _solve_impl(const RhsType &rhs, DstType &dst) const { if(!internal::is_same_dense(dst,rhs)) dst = rhs; this->solveInPlace(dst); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TriangularViewType& _assignProduct(const ProductType& prod, const Scalar& alpha, bool beta); protected: EIGEN_DEFAULT_COPY_CONSTRUCTOR(TriangularViewImpl) EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(TriangularViewImpl) }; /*************************************************************************** * Implementation of triangular evaluation/assignment ***************************************************************************/ #ifndef EIGEN_PARSED_BY_DOXYGEN // FIXME should we keep that possibility template template EIGEN_DEVICE_FUNC inline TriangularView& TriangularViewImpl::operator=(const MatrixBase& other) { internal::call_assignment_no_alias(derived(), other.derived(), internal::assign_op()); return derived(); } // FIXME should we keep that possibility template template EIGEN_DEVICE_FUNC void TriangularViewImpl::lazyAssign(const MatrixBase& other) { internal::call_assignment_no_alias(derived(), other.template triangularView()); } template template EIGEN_DEVICE_FUNC inline TriangularView& TriangularViewImpl::operator=(const TriangularBase& other) { eigen_assert(Mode == int(OtherDerived::Mode)); internal::call_assignment(derived(), other.derived()); return derived(); } template template EIGEN_DEVICE_FUNC void TriangularViewImpl::lazyAssign(const TriangularBase& other) { eigen_assert(Mode == int(OtherDerived::Mode)); internal::call_assignment_no_alias(derived(), other.derived()); } #endif /*************************************************************************** * Implementation of TriangularBase methods ***************************************************************************/ /** Assigns a triangular or selfadjoint matrix to a dense matrix. * If the matrix is triangular, the opposite part is set to zero. */ template template EIGEN_DEVICE_FUNC void TriangularBase::evalTo(MatrixBase &other) const { evalToLazy(other.derived()); } /*************************************************************************** * Implementation of TriangularView methods ***************************************************************************/ /*************************************************************************** * Implementation of MatrixBase methods ***************************************************************************/ /** * \returns an expression of a triangular view extracted from the current matrix * * The parameter \a Mode can have the following values: \c #Upper, \c #StrictlyUpper, \c #UnitUpper, * \c #Lower, \c #StrictlyLower, \c #UnitLower. * * Example: \include MatrixBase_triangularView.cpp * Output: \verbinclude MatrixBase_triangularView.out * * \sa class TriangularView */ template template EIGEN_DEVICE_FUNC typename MatrixBase::template TriangularViewReturnType::Type MatrixBase::triangularView() { return typename TriangularViewReturnType::Type(derived()); } /** This is the const version of MatrixBase::triangularView() */ template template EIGEN_DEVICE_FUNC typename MatrixBase::template ConstTriangularViewReturnType::Type MatrixBase::triangularView() const { return typename ConstTriangularViewReturnType::Type(derived()); } /** \returns true if *this is approximately equal to an upper triangular matrix, * within the precision given by \a prec. * * \sa isLowerTriangular() */ template bool MatrixBase::isUpperTriangular(const RealScalar& prec) const { RealScalar maxAbsOnUpperPart = static_cast(-1); for(Index j = 0; j < cols(); ++j) { Index maxi = numext::mini(j, rows()-1); for(Index i = 0; i <= maxi; ++i) { RealScalar absValue = numext::abs(coeff(i,j)); if(absValue > maxAbsOnUpperPart) maxAbsOnUpperPart = absValue; } } RealScalar threshold = maxAbsOnUpperPart * prec; for(Index j = 0; j < cols(); ++j) for(Index i = j+1; i < rows(); ++i) if(numext::abs(coeff(i, j)) > threshold) return false; return true; } /** \returns true if *this is approximately equal to a lower triangular matrix, * within the precision given by \a prec. * * \sa isUpperTriangular() */ template bool MatrixBase::isLowerTriangular(const RealScalar& prec) const { RealScalar maxAbsOnLowerPart = static_cast(-1); for(Index j = 0; j < cols(); ++j) for(Index i = j; i < rows(); ++i) { RealScalar absValue = numext::abs(coeff(i,j)); if(absValue > maxAbsOnLowerPart) maxAbsOnLowerPart = absValue; } RealScalar threshold = maxAbsOnLowerPart * prec; for(Index j = 1; j < cols(); ++j) { Index maxi = numext::mini(j, rows()-1); for(Index i = 0; i < maxi; ++i) if(numext::abs(coeff(i, j)) > threshold) return false; } return true; } /*************************************************************************** **************************************************************************** * Evaluators and Assignment of triangular expressions *************************************************************************** ***************************************************************************/ namespace internal { // TODO currently a triangular expression has the form TriangularView<.,.> // in the future triangular-ness should be defined by the expression traits // such that Transpose > is valid. (currently TriangularBase::transpose() is overloaded to make it work) template struct evaluator_traits > { typedef typename storage_kind_to_evaluator_kind::Kind Kind; typedef typename glue_shapes::Shape, TriangularShape>::type Shape; }; template struct unary_evaluator, IndexBased> : evaluator::type> { typedef TriangularView XprType; typedef evaluator::type> Base; EIGEN_DEVICE_FUNC unary_evaluator(const XprType &xpr) : Base(xpr.nestedExpression()) {} }; // Additional assignment kinds: struct Triangular2Triangular {}; struct Triangular2Dense {}; struct Dense2Triangular {}; template struct triangular_assignment_loop; /** \internal Specialization of the dense assignment kernel for triangular matrices. * The main difference is that the triangular, diagonal, and opposite parts are processed through three different functions. * \tparam UpLo must be either Lower or Upper * \tparam Mode must be either 0, UnitDiag, ZeroDiag, or SelfAdjoint */ template class triangular_dense_assignment_kernel : public generic_dense_assignment_kernel { protected: typedef generic_dense_assignment_kernel Base; typedef typename Base::DstXprType DstXprType; typedef typename Base::SrcXprType SrcXprType; using Base::m_dst; using Base::m_src; using Base::m_functor; public: typedef typename Base::DstEvaluatorType DstEvaluatorType; typedef typename Base::SrcEvaluatorType SrcEvaluatorType; typedef typename Base::Scalar Scalar; typedef typename Base::AssignmentTraits AssignmentTraits; EIGEN_DEVICE_FUNC triangular_dense_assignment_kernel(DstEvaluatorType &dst, const SrcEvaluatorType &src, const Functor &func, DstXprType& dstExpr) : Base(dst, src, func, dstExpr) {} #ifdef EIGEN_INTERNAL_DEBUGGING EIGEN_DEVICE_FUNC void assignCoeff(Index row, Index col) { eigen_internal_assert(row!=col); Base::assignCoeff(row,col); } #else using Base::assignCoeff; #endif EIGEN_DEVICE_FUNC void assignDiagonalCoeff(Index id) { if(Mode==UnitDiag && SetOpposite) m_functor.assignCoeff(m_dst.coeffRef(id,id), Scalar(1)); else if(Mode==ZeroDiag && SetOpposite) m_functor.assignCoeff(m_dst.coeffRef(id,id), Scalar(0)); else if(Mode==0) Base::assignCoeff(id,id); } EIGEN_DEVICE_FUNC void assignOppositeCoeff(Index row, Index col) { eigen_internal_assert(row!=col); if(SetOpposite) m_functor.assignCoeff(m_dst.coeffRef(row,col), Scalar(0)); } }; template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_triangular_assignment_loop(DstXprType& dst, const SrcXprType& src, const Functor &func) { typedef evaluator DstEvaluatorType; typedef evaluator SrcEvaluatorType; SrcEvaluatorType srcEvaluator(src); Index dstRows = src.rows(); Index dstCols = src.cols(); if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) dst.resize(dstRows, dstCols); DstEvaluatorType dstEvaluator(dst); typedef triangular_dense_assignment_kernel< Mode&(Lower|Upper),Mode&(UnitDiag|ZeroDiag|SelfAdjoint),SetOpposite, DstEvaluatorType,SrcEvaluatorType,Functor> Kernel; Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived()); enum { unroll = DstXprType::SizeAtCompileTime != Dynamic && SrcEvaluatorType::CoeffReadCost < HugeCost && DstXprType::SizeAtCompileTime * (int(DstEvaluatorType::CoeffReadCost) + int(SrcEvaluatorType::CoeffReadCost)) / 2 <= EIGEN_UNROLLING_LIMIT }; triangular_assignment_loop::run(kernel); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_triangular_assignment_loop(DstXprType& dst, const SrcXprType& src) { call_triangular_assignment_loop(dst, src, internal::assign_op()); } template<> struct AssignmentKind { typedef Triangular2Triangular Kind; }; template<> struct AssignmentKind { typedef Triangular2Dense Kind; }; template<> struct AssignmentKind { typedef Dense2Triangular Kind; }; template< typename DstXprType, typename SrcXprType, typename Functor> struct Assignment { EIGEN_DEVICE_FUNC static void run(DstXprType &dst, const SrcXprType &src, const Functor &func) { eigen_assert(int(DstXprType::Mode) == int(SrcXprType::Mode)); call_triangular_assignment_loop(dst, src, func); } }; template< typename DstXprType, typename SrcXprType, typename Functor> struct Assignment { EIGEN_DEVICE_FUNC static void run(DstXprType &dst, const SrcXprType &src, const Functor &func) { call_triangular_assignment_loop(dst, src, func); } }; template< typename DstXprType, typename SrcXprType, typename Functor> struct Assignment { EIGEN_DEVICE_FUNC static void run(DstXprType &dst, const SrcXprType &src, const Functor &func) { call_triangular_assignment_loop(dst, src, func); } }; template struct triangular_assignment_loop { // FIXME: this is not very clean, perhaps this information should be provided by the kernel? typedef typename Kernel::DstEvaluatorType DstEvaluatorType; typedef typename DstEvaluatorType::XprType DstXprType; enum { col = (UnrollCount-1) / DstXprType::RowsAtCompileTime, row = (UnrollCount-1) % DstXprType::RowsAtCompileTime }; typedef typename Kernel::Scalar Scalar; EIGEN_DEVICE_FUNC static inline void run(Kernel &kernel) { triangular_assignment_loop::run(kernel); if(row==col) kernel.assignDiagonalCoeff(row); else if( ((Mode&Lower) && row>col) || ((Mode&Upper) && row struct triangular_assignment_loop { EIGEN_DEVICE_FUNC static inline void run(Kernel &) {} }; // TODO: experiment with a recursive assignment procedure splitting the current // triangular part into one rectangular and two triangular parts. template struct triangular_assignment_loop { typedef typename Kernel::Scalar Scalar; EIGEN_DEVICE_FUNC static inline void run(Kernel &kernel) { for(Index j = 0; j < kernel.cols(); ++j) { Index maxi = numext::mini(j, kernel.rows()); Index i = 0; if (((Mode&Lower) && SetOpposite) || (Mode&Upper)) { for(; i < maxi; ++i) if(Mode&Upper) kernel.assignCoeff(i, j); else kernel.assignOppositeCoeff(i, j); } else i = maxi; if(i template EIGEN_DEVICE_FUNC void TriangularBase::evalToLazy(MatrixBase &other) const { other.derived().resize(this->rows(), this->cols()); internal::call_triangular_assignment_loop(other.derived(), derived().nestedExpression()); } namespace internal { // Triangular = Product template< typename DstXprType, typename Lhs, typename Rhs, typename Scalar> struct Assignment, internal::assign_op::Scalar>, Dense2Triangular> { typedef Product SrcXprType; static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op &) { Index dstRows = src.rows(); Index dstCols = src.cols(); if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) dst.resize(dstRows, dstCols); dst._assignProduct(src, Scalar(1), false); } }; // Triangular += Product template< typename DstXprType, typename Lhs, typename Rhs, typename Scalar> struct Assignment, internal::add_assign_op::Scalar>, Dense2Triangular> { typedef Product SrcXprType; static void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op &) { dst._assignProduct(src, Scalar(1), true); } }; // Triangular -= Product template< typename DstXprType, typename Lhs, typename Rhs, typename Scalar> struct Assignment, internal::sub_assign_op::Scalar>, Dense2Triangular> { typedef Product SrcXprType; static void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op &) { dst._assignProduct(src, Scalar(-1), true); } }; } // end namespace internal } // end namespace Eigen #endif // EIGEN_TRIANGULARMATRIX_H RcppEigen/inst/include/Eigen/src/Core/Ref.h0000644000176200001440000004263514567757725020164 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2012 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_REF_H #define EIGEN_REF_H namespace Eigen { namespace internal { template struct traits > : public traits > { typedef _PlainObjectType PlainObjectType; typedef _StrideType StrideType; enum { Options = _Options, Flags = traits >::Flags | NestByRefBit, Alignment = traits >::Alignment }; template struct match { enum { IsVectorAtCompileTime = PlainObjectType::IsVectorAtCompileTime || Derived::IsVectorAtCompileTime, HasDirectAccess = internal::has_direct_access::ret, StorageOrderMatch = IsVectorAtCompileTime || ((PlainObjectType::Flags&RowMajorBit)==(Derived::Flags&RowMajorBit)), InnerStrideMatch = int(StrideType::InnerStrideAtCompileTime)==int(Dynamic) || int(StrideType::InnerStrideAtCompileTime)==int(Derived::InnerStrideAtCompileTime) || (int(StrideType::InnerStrideAtCompileTime)==0 && int(Derived::InnerStrideAtCompileTime)==1), OuterStrideMatch = IsVectorAtCompileTime || int(StrideType::OuterStrideAtCompileTime)==int(Dynamic) || int(StrideType::OuterStrideAtCompileTime)==int(Derived::OuterStrideAtCompileTime), // NOTE, this indirection of evaluator::Alignment is needed // to workaround a very strange bug in MSVC related to the instantiation // of has_*ary_operator in evaluator. // This line is surprisingly very sensitive. For instance, simply adding parenthesis // as "DerivedAlignment = (int(evaluator::Alignment))," will make MSVC fail... DerivedAlignment = int(evaluator::Alignment), AlignmentMatch = (int(traits::Alignment)==int(Unaligned)) || (DerivedAlignment >= int(Alignment)), // FIXME the first condition is not very clear, it should be replaced by the required alignment ScalarTypeMatch = internal::is_same::value, MatchAtCompileTime = HasDirectAccess && StorageOrderMatch && InnerStrideMatch && OuterStrideMatch && AlignmentMatch && ScalarTypeMatch }; typedef typename internal::conditional::type type; }; }; template struct traits > : public traits {}; } template class RefBase : public MapBase { typedef typename internal::traits::PlainObjectType PlainObjectType; typedef typename internal::traits::StrideType StrideType; public: typedef MapBase Base; EIGEN_DENSE_PUBLIC_INTERFACE(RefBase) EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const { return StrideType::InnerStrideAtCompileTime != 0 ? m_stride.inner() : 1; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const { return StrideType::OuterStrideAtCompileTime != 0 ? m_stride.outer() : IsVectorAtCompileTime ? this->size() : int(Flags)&RowMajorBit ? this->cols() : this->rows(); } EIGEN_DEVICE_FUNC RefBase() : Base(0,RowsAtCompileTime==Dynamic?0:RowsAtCompileTime,ColsAtCompileTime==Dynamic?0:ColsAtCompileTime), // Stride<> does not allow default ctor for Dynamic strides, so let' initialize it with dummy values: m_stride(StrideType::OuterStrideAtCompileTime==Dynamic?0:StrideType::OuterStrideAtCompileTime, StrideType::InnerStrideAtCompileTime==Dynamic?0:StrideType::InnerStrideAtCompileTime) {} EIGEN_INHERIT_ASSIGNMENT_OPERATORS(RefBase) protected: typedef Stride StrideBase; // Resolves inner stride if default 0. static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index resolveInnerStride(Index inner) { return inner == 0 ? 1 : inner; } // Resolves outer stride if default 0. static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index resolveOuterStride(Index inner, Index outer, Index rows, Index cols, bool isVectorAtCompileTime, bool isRowMajor) { return outer == 0 ? isVectorAtCompileTime ? inner * rows * cols : isRowMajor ? inner * cols : inner * rows : outer; } // Returns true if construction is valid, false if there is a stride mismatch, // and fails if there is a size mismatch. template EIGEN_DEVICE_FUNC bool construct(Expression& expr) { // Check matrix sizes. If this is a compile-time vector, we do allow // implicitly transposing. EIGEN_STATIC_ASSERT( EIGEN_PREDICATE_SAME_MATRIX_SIZE(PlainObjectType, Expression) // If it is a vector, the transpose sizes might match. || ( PlainObjectType::IsVectorAtCompileTime && ((int(PlainObjectType::RowsAtCompileTime)==Eigen::Dynamic || int(Expression::ColsAtCompileTime)==Eigen::Dynamic || int(PlainObjectType::RowsAtCompileTime)==int(Expression::ColsAtCompileTime)) && (int(PlainObjectType::ColsAtCompileTime)==Eigen::Dynamic || int(Expression::RowsAtCompileTime)==Eigen::Dynamic || int(PlainObjectType::ColsAtCompileTime)==int(Expression::RowsAtCompileTime)))), YOU_MIXED_MATRICES_OF_DIFFERENT_SIZES ) // Determine runtime rows and columns. Index rows = expr.rows(); Index cols = expr.cols(); if(PlainObjectType::RowsAtCompileTime==1) { eigen_assert(expr.rows()==1 || expr.cols()==1); rows = 1; cols = expr.size(); } else if(PlainObjectType::ColsAtCompileTime==1) { eigen_assert(expr.rows()==1 || expr.cols()==1); rows = expr.size(); cols = 1; } // Verify that the sizes are valid. eigen_assert( (PlainObjectType::RowsAtCompileTime == Dynamic) || (PlainObjectType::RowsAtCompileTime == rows)); eigen_assert( (PlainObjectType::ColsAtCompileTime == Dynamic) || (PlainObjectType::ColsAtCompileTime == cols)); // If this is a vector, we might be transposing, which means that stride should swap. const bool transpose = PlainObjectType::IsVectorAtCompileTime && (rows != expr.rows()); // If the storage format differs, we also need to swap the stride. const bool row_major = ((PlainObjectType::Flags)&RowMajorBit) != 0; const bool expr_row_major = (Expression::Flags&RowMajorBit) != 0; const bool storage_differs = (row_major != expr_row_major); const bool swap_stride = (transpose != storage_differs); // Determine expr's actual strides, resolving any defaults if zero. const Index expr_inner_actual = resolveInnerStride(expr.innerStride()); const Index expr_outer_actual = resolveOuterStride(expr_inner_actual, expr.outerStride(), expr.rows(), expr.cols(), Expression::IsVectorAtCompileTime != 0, expr_row_major); // If this is a column-major row vector or row-major column vector, the inner-stride // is arbitrary, so set it to either the compile-time inner stride or 1. const bool row_vector = (rows == 1); const bool col_vector = (cols == 1); const Index inner_stride = ( (!row_major && row_vector) || (row_major && col_vector) ) ? ( StrideType::InnerStrideAtCompileTime > 0 ? Index(StrideType::InnerStrideAtCompileTime) : 1) : swap_stride ? expr_outer_actual : expr_inner_actual; // If this is a column-major column vector or row-major row vector, the outer-stride // is arbitrary, so set it to either the compile-time outer stride or vector size. const Index outer_stride = ( (!row_major && col_vector) || (row_major && row_vector) ) ? ( StrideType::OuterStrideAtCompileTime > 0 ? Index(StrideType::OuterStrideAtCompileTime) : rows * cols * inner_stride) : swap_stride ? expr_inner_actual : expr_outer_actual; // Check if given inner/outer strides are compatible with compile-time strides. const bool inner_valid = (StrideType::InnerStrideAtCompileTime == Dynamic) || (resolveInnerStride(Index(StrideType::InnerStrideAtCompileTime)) == inner_stride); if (!inner_valid) { return false; } const bool outer_valid = (StrideType::OuterStrideAtCompileTime == Dynamic) || (resolveOuterStride( inner_stride, Index(StrideType::OuterStrideAtCompileTime), rows, cols, PlainObjectType::IsVectorAtCompileTime != 0, row_major) == outer_stride); if (!outer_valid) { return false; } ::new (static_cast(this)) Base(expr.data(), rows, cols); ::new (&m_stride) StrideBase( (StrideType::OuterStrideAtCompileTime == 0) ? 0 : outer_stride, (StrideType::InnerStrideAtCompileTime == 0) ? 0 : inner_stride ); return true; } StrideBase m_stride; }; /** \class Ref * \ingroup Core_Module * * \brief A matrix or vector expression mapping an existing expression * * \tparam PlainObjectType the equivalent matrix type of the mapped data * \tparam Options specifies the pointer alignment in bytes. It can be: \c #Aligned128, , \c #Aligned64, \c #Aligned32, \c #Aligned16, \c #Aligned8 or \c #Unaligned. * The default is \c #Unaligned. * \tparam StrideType optionally specifies strides. By default, Ref implies a contiguous storage along the inner dimension (inner stride==1), * but accepts a variable outer stride (leading dimension). * This can be overridden by specifying strides. * The type passed here must be a specialization of the Stride template, see examples below. * * This class provides a way to write non-template functions taking Eigen objects as parameters while limiting the number of copies. * A Ref<> object can represent either a const expression or a l-value: * \code * // in-out argument: * void foo1(Ref x); * * // read-only const argument: * void foo2(const Ref& x); * \endcode * * In the in-out case, the input argument must satisfy the constraints of the actual Ref<> type, otherwise a compilation issue will be triggered. * By default, a Ref can reference any dense vector expression of float having a contiguous memory layout. * Likewise, a Ref can reference any column-major dense matrix expression of float whose column's elements are contiguously stored with * the possibility to have a constant space in-between each column, i.e. the inner stride must be equal to 1, but the outer stride (or leading dimension) * can be greater than the number of rows. * * In the const case, if the input expression does not match the above requirement, then it is evaluated into a temporary before being passed to the function. * Here are some examples: * \code * MatrixXf A; * VectorXf a; * foo1(a.head()); // OK * foo1(A.col()); // OK * foo1(A.row()); // Compilation error because here innerstride!=1 * foo2(A.row()); // Compilation error because A.row() is a 1xN object while foo2 is expecting a Nx1 object * foo2(A.row().transpose()); // The row is copied into a contiguous temporary * foo2(2*a); // The expression is evaluated into a temporary * foo2(A.col().segment(2,4)); // No temporary * \endcode * * The range of inputs that can be referenced without temporary can be enlarged using the last two template parameters. * Here is an example accepting an innerstride!=1: * \code * // in-out argument: * void foo3(Ref > x); * foo3(A.row()); // OK * \endcode * The downside here is that the function foo3 might be significantly slower than foo1 because it won't be able to exploit vectorization, and will involve more * expensive address computations even if the input is contiguously stored in memory. To overcome this issue, one might propose to overload internally calling a * template function, e.g.: * \code * // in the .h: * void foo(const Ref& A); * void foo(const Ref >& A); * * // in the .cpp: * template void foo_impl(const TypeOfA& A) { * ... // crazy code goes here * } * void foo(const Ref& A) { foo_impl(A); } * void foo(const Ref >& A) { foo_impl(A); } * \endcode * * See also the following stackoverflow questions for further references: * - Correct usage of the Eigen::Ref<> class * * \sa PlainObjectBase::Map(), \ref TopicStorageOrders */ template class Ref : public RefBase > { private: typedef internal::traits Traits; template EIGEN_DEVICE_FUNC inline Ref(const PlainObjectBase& expr, typename internal::enable_if::MatchAtCompileTime),Derived>::type* = 0); public: typedef RefBase Base; EIGEN_DENSE_PUBLIC_INTERFACE(Ref) #ifndef EIGEN_PARSED_BY_DOXYGEN template EIGEN_DEVICE_FUNC inline Ref(PlainObjectBase& expr, typename internal::enable_if::MatchAtCompileTime),Derived>::type* = 0) { EIGEN_STATIC_ASSERT(bool(Traits::template match::MatchAtCompileTime), STORAGE_LAYOUT_DOES_NOT_MATCH); // Construction must pass since we will not create temprary storage in the non-const case. const bool success = Base::construct(expr.derived()); EIGEN_UNUSED_VARIABLE(success) eigen_assert(success); } template EIGEN_DEVICE_FUNC inline Ref(const DenseBase& expr, typename internal::enable_if::MatchAtCompileTime),Derived>::type* = 0) #else /** Implicit constructor from any dense expression */ template inline Ref(DenseBase& expr) #endif { EIGEN_STATIC_ASSERT(bool(internal::is_lvalue::value), THIS_EXPRESSION_IS_NOT_A_LVALUE__IT_IS_READ_ONLY); EIGEN_STATIC_ASSERT(bool(Traits::template match::MatchAtCompileTime), STORAGE_LAYOUT_DOES_NOT_MATCH); EIGEN_STATIC_ASSERT(!Derived::IsPlainObjectBase,THIS_EXPRESSION_IS_NOT_A_LVALUE__IT_IS_READ_ONLY); // Construction must pass since we will not create temporary storage in the non-const case. const bool success = Base::construct(expr.const_cast_derived()); EIGEN_UNUSED_VARIABLE(success) eigen_assert(success); } EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Ref) }; // this is the const ref version template class Ref : public RefBase > { typedef internal::traits Traits; public: typedef RefBase Base; EIGEN_DENSE_PUBLIC_INTERFACE(Ref) template EIGEN_DEVICE_FUNC inline Ref(const DenseBase& expr, typename internal::enable_if::ScalarTypeMatch),Derived>::type* = 0) { // std::cout << match_helper::HasDirectAccess << "," << match_helper::OuterStrideMatch << "," << match_helper::InnerStrideMatch << "\n"; // std::cout << int(StrideType::OuterStrideAtCompileTime) << " - " << int(Derived::OuterStrideAtCompileTime) << "\n"; // std::cout << int(StrideType::InnerStrideAtCompileTime) << " - " << int(Derived::InnerStrideAtCompileTime) << "\n"; construct(expr.derived(), typename Traits::template match::type()); } EIGEN_DEVICE_FUNC inline Ref(const Ref& other) : Base(other) { // copy constructor shall not copy the m_object, to avoid unnecessary malloc and copy } template EIGEN_DEVICE_FUNC inline Ref(const RefBase& other) { construct(other.derived(), typename Traits::template match::type()); } protected: template EIGEN_DEVICE_FUNC void construct(const Expression& expr,internal::true_type) { // Check if we can use the underlying expr's storage directly, otherwise call the copy version. if (!Base::construct(expr)) { construct(expr, internal::false_type()); } } template EIGEN_DEVICE_FUNC void construct(const Expression& expr, internal::false_type) { internal::call_assignment_no_alias(m_object,expr,internal::assign_op()); Base::construct(m_object); } protected: TPlainObjectType m_object; }; } // end namespace Eigen #endif // EIGEN_REF_H RcppEigen/inst/include/Eigen/src/Core/NumTraits.h0000644000176200001440000003112414567757725021365 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2006-2010 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_NUMTRAITS_H #define EIGEN_NUMTRAITS_H namespace Eigen { namespace internal { // default implementation of digits10(), based on numeric_limits if specialized, // 0 for integer types, and log10(epsilon()) otherwise. template< typename T, bool use_numeric_limits = std::numeric_limits::is_specialized, bool is_integer = NumTraits::IsInteger> struct default_digits10_impl { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() { return std::numeric_limits::digits10; } }; template struct default_digits10_impl // Floating point { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() { using std::log10; using std::ceil; typedef typename NumTraits::Real Real; return int(ceil(-log10(NumTraits::epsilon()))); } }; template struct default_digits10_impl // Integer { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() { return 0; } }; // default implementation of digits(), based on numeric_limits if specialized, // 0 for integer types, and log2(epsilon()) otherwise. template< typename T, bool use_numeric_limits = std::numeric_limits::is_specialized, bool is_integer = NumTraits::IsInteger> struct default_digits_impl { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() { return std::numeric_limits::digits; } }; template struct default_digits_impl // Floating point { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() { using std::log; using std::ceil; typedef typename NumTraits::Real Real; return int(ceil(-log(NumTraits::epsilon())/log(static_cast(2)))); } }; template struct default_digits_impl // Integer { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() { return 0; } }; } // end namespace internal namespace numext { /** \internal bit-wise cast without changing the underlying bit representation. */ // TODO: Replace by std::bit_cast (available in C++20) template EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Tgt bit_cast(const Src& src) { #if EIGEN_HAS_TYPE_TRAITS // The behaviour of memcpy is not specified for non-trivially copyable types EIGEN_STATIC_ASSERT(std::is_trivially_copyable::value, THIS_TYPE_IS_NOT_SUPPORTED); EIGEN_STATIC_ASSERT(std::is_trivially_copyable::value && std::is_default_constructible::value, THIS_TYPE_IS_NOT_SUPPORTED); #endif EIGEN_STATIC_ASSERT(sizeof(Src) == sizeof(Tgt), THIS_TYPE_IS_NOT_SUPPORTED); Tgt tgt; EIGEN_USING_STD(memcpy) memcpy(&tgt, &src, sizeof(Tgt)); return tgt; } } // namespace numext /** \class NumTraits * \ingroup Core_Module * * \brief Holds information about the various numeric (i.e. scalar) types allowed by Eigen. * * \tparam T the numeric type at hand * * This class stores enums, typedefs and static methods giving information about a numeric type. * * The provided data consists of: * \li A typedef \c Real, giving the "real part" type of \a T. If \a T is already real, * then \c Real is just a typedef to \a T. If \a T is \c std::complex then \c Real * is a typedef to \a U. * \li A typedef \c NonInteger, giving the type that should be used for operations producing non-integral values, * such as quotients, square roots, etc. If \a T is a floating-point type, then this typedef just gives * \a T again. Note however that many Eigen functions such as internal::sqrt simply refuse to * take integers. Outside of a few cases, Eigen doesn't do automatic type promotion. Thus, this typedef is * only intended as a helper for code that needs to explicitly promote types. * \li A typedef \c Literal giving the type to use for numeric literals such as "2" or "0.5". For instance, for \c std::complex, Literal is defined as \c U. * Of course, this type must be fully compatible with \a T. In doubt, just use \a T here. * \li A typedef \a Nested giving the type to use to nest a value inside of the expression tree. If you don't know what * this means, just use \a T here. * \li An enum value \a IsComplex. It is equal to 1 if \a T is a \c std::complex * type, and to 0 otherwise. * \li An enum value \a IsInteger. It is equal to \c 1 if \a T is an integer type such as \c int, * and to \c 0 otherwise. * \li Enum values ReadCost, AddCost and MulCost representing a rough estimate of the number of CPU cycles needed * to by move / add / mul instructions respectively, assuming the data is already stored in CPU registers. * Stay vague here. No need to do architecture-specific stuff. If you don't know what this means, just use \c Eigen::HugeCost. * \li An enum value \a IsSigned. It is equal to \c 1 if \a T is a signed type and to 0 if \a T is unsigned. * \li An enum value \a RequireInitialization. It is equal to \c 1 if the constructor of the numeric type \a T must * be called, and to 0 if it is safe not to call it. Default is 0 if \a T is an arithmetic type, and 1 otherwise. * \li An epsilon() function which, unlike std::numeric_limits::epsilon(), * it returns a \a Real instead of a \a T. * \li A dummy_precision() function returning a weak epsilon value. It is mainly used as a default * value by the fuzzy comparison operators. * \li highest() and lowest() functions returning the highest and lowest possible values respectively. * \li digits() function returning the number of radix digits (non-sign digits for integers, mantissa for floating-point). This is * the analogue of std::numeric_limits::digits * which is used as the default implementation if specialized. * \li digits10() function returning the number of decimal digits that can be represented without change. This is * the analogue of std::numeric_limits::digits10 * which is used as the default implementation if specialized. * \li min_exponent() and max_exponent() functions returning the highest and lowest possible values, respectively, * such that the radix raised to the power exponent-1 is a normalized floating-point number. These are equivalent to * std::numeric_limits::min_exponent/ * std::numeric_limits::max_exponent. * \li infinity() function returning a representation of positive infinity, if available. * \li quiet_NaN function returning a non-signaling "not-a-number", if available. */ template struct GenericNumTraits { enum { IsInteger = std::numeric_limits::is_integer, IsSigned = std::numeric_limits::is_signed, IsComplex = 0, RequireInitialization = internal::is_arithmetic::value ? 0 : 1, ReadCost = 1, AddCost = 1, MulCost = 1 }; typedef T Real; typedef typename internal::conditional< IsInteger, typename internal::conditional::type, T >::type NonInteger; typedef T Nested; typedef T Literal; EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline Real epsilon() { return numext::numeric_limits::epsilon(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline int digits10() { return internal::default_digits10_impl::run(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline int digits() { return internal::default_digits_impl::run(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline int min_exponent() { return numext::numeric_limits::min_exponent; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline int max_exponent() { return numext::numeric_limits::max_exponent; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline Real dummy_precision() { // make sure to override this for floating-point types return Real(0); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline T highest() { return (numext::numeric_limits::max)(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline T lowest() { return IsInteger ? (numext::numeric_limits::min)() : static_cast(-(numext::numeric_limits::max)()); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline T infinity() { return numext::numeric_limits::infinity(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline T quiet_NaN() { return numext::numeric_limits::quiet_NaN(); } }; template struct NumTraits : GenericNumTraits {}; template<> struct NumTraits : GenericNumTraits { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline float dummy_precision() { return 1e-5f; } }; template<> struct NumTraits : GenericNumTraits { EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline double dummy_precision() { return 1e-12; } }; template<> struct NumTraits : GenericNumTraits { EIGEN_CONSTEXPR static inline long double dummy_precision() { return 1e-15l; } }; template struct NumTraits > : GenericNumTraits > { typedef _Real Real; typedef typename NumTraits<_Real>::Literal Literal; enum { IsComplex = 1, RequireInitialization = NumTraits<_Real>::RequireInitialization, ReadCost = 2 * NumTraits<_Real>::ReadCost, AddCost = 2 * NumTraits::AddCost, MulCost = 4 * NumTraits::MulCost + 2 * NumTraits::AddCost }; EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline Real epsilon() { return NumTraits::epsilon(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline Real dummy_precision() { return NumTraits::dummy_precision(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline int digits10() { return NumTraits::digits10(); } }; template struct NumTraits > { typedef Array ArrayType; typedef typename NumTraits::Real RealScalar; typedef Array Real; typedef typename NumTraits::NonInteger NonIntegerScalar; typedef Array NonInteger; typedef ArrayType & Nested; typedef typename NumTraits::Literal Literal; enum { IsComplex = NumTraits::IsComplex, IsInteger = NumTraits::IsInteger, IsSigned = NumTraits::IsSigned, RequireInitialization = 1, ReadCost = ArrayType::SizeAtCompileTime==Dynamic ? HugeCost : ArrayType::SizeAtCompileTime * int(NumTraits::ReadCost), AddCost = ArrayType::SizeAtCompileTime==Dynamic ? HugeCost : ArrayType::SizeAtCompileTime * int(NumTraits::AddCost), MulCost = ArrayType::SizeAtCompileTime==Dynamic ? HugeCost : ArrayType::SizeAtCompileTime * int(NumTraits::MulCost) }; EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline RealScalar epsilon() { return NumTraits::epsilon(); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline RealScalar dummy_precision() { return NumTraits::dummy_precision(); } EIGEN_CONSTEXPR static inline int digits10() { return NumTraits::digits10(); } }; template<> struct NumTraits : GenericNumTraits { enum { RequireInitialization = 1, ReadCost = HugeCost, AddCost = HugeCost, MulCost = HugeCost }; EIGEN_CONSTEXPR static inline int digits10() { return 0; } private: static inline std::string epsilon(); static inline std::string dummy_precision(); static inline std::string lowest(); static inline std::string highest(); static inline std::string infinity(); static inline std::string quiet_NaN(); }; // Empty specialization for void to allow template specialization based on NumTraits::Real with T==void and SFINAE. template<> struct NumTraits {}; template<> struct NumTraits : GenericNumTraits {}; } // end namespace Eigen #endif // EIGEN_NUMTRAITS_H RcppEigen/inst/include/Eigen/src/Core/DiagonalProduct.h0000644000176200001440000000173414567757725022522 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008 Gael Guennebaud // Copyright (C) 2007-2009 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_DIAGONALPRODUCT_H #define EIGEN_DIAGONALPRODUCT_H namespace Eigen { /** \returns the diagonal matrix product of \c *this by the diagonal matrix \a diagonal. */ template template EIGEN_DEVICE_FUNC inline const Product MatrixBase::operator*(const DiagonalBase &a_diagonal) const { return Product(derived(),a_diagonal.derived()); } } // end namespace Eigen #endif // EIGEN_DIAGONALPRODUCT_H RcppEigen/inst/include/Eigen/src/Core/DenseCoeffsBase.h0000644000176200001440000005764414567757725022435 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2006-2010 Benoit Jacob // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_DENSECOEFFSBASE_H #define EIGEN_DENSECOEFFSBASE_H namespace Eigen { namespace internal { template struct add_const_on_value_type_if_arithmetic { typedef typename conditional::value, T, typename add_const_on_value_type::type>::type type; }; } /** \brief Base class providing read-only coefficient access to matrices and arrays. * \ingroup Core_Module * \tparam Derived Type of the derived class * * \note #ReadOnlyAccessors Constant indicating read-only access * * This class defines the \c operator() \c const function and friends, which can be used to read specific * entries of a matrix or array. * * \sa DenseCoeffsBase, DenseCoeffsBase, * \ref TopicClassHierarchy */ template class DenseCoeffsBase : public EigenBase { public: typedef typename internal::traits::StorageKind StorageKind; typedef typename internal::traits::Scalar Scalar; typedef typename internal::packet_traits::type PacketScalar; // Explanation for this CoeffReturnType typedef. // - This is the return type of the coeff() method. // - The LvalueBit means exactly that we can offer a coeffRef() method, which means exactly that we can get references // to coeffs, which means exactly that we can have coeff() return a const reference (as opposed to returning a value). // - The is_artihmetic check is required since "const int", "const double", etc. will cause warnings on some systems // while the declaration of "const T", where T is a non arithmetic type does not. Always returning "const Scalar&" is // not possible, since the underlying expressions might not offer a valid address the reference could be referring to. typedef typename internal::conditional::Flags&LvalueBit), const Scalar&, typename internal::conditional::value, Scalar, const Scalar>::type >::type CoeffReturnType; typedef typename internal::add_const_on_value_type_if_arithmetic< typename internal::packet_traits::type >::type PacketReturnType; typedef EigenBase Base; using Base::rows; using Base::cols; using Base::size; using Base::derived; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index rowIndexByOuterInner(Index outer, Index inner) const { return int(Derived::RowsAtCompileTime) == 1 ? 0 : int(Derived::ColsAtCompileTime) == 1 ? inner : int(Derived::Flags)&RowMajorBit ? outer : inner; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index colIndexByOuterInner(Index outer, Index inner) const { return int(Derived::ColsAtCompileTime) == 1 ? 0 : int(Derived::RowsAtCompileTime) == 1 ? inner : int(Derived::Flags)&RowMajorBit ? inner : outer; } /** Short version: don't use this function, use * \link operator()(Index,Index) const \endlink instead. * * Long version: this function is similar to * \link operator()(Index,Index) const \endlink, but without the assertion. * Use this for limiting the performance cost of debugging code when doing * repeated coefficient access. Only use this when it is guaranteed that the * parameters \a row and \a col are in range. * * If EIGEN_INTERNAL_DEBUGGING is defined, an assertion will be made, making this * function equivalent to \link operator()(Index,Index) const \endlink. * * \sa operator()(Index,Index) const, coeffRef(Index,Index), coeff(Index) const */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { eigen_internal_assert(row >= 0 && row < rows() && col >= 0 && col < cols()); return internal::evaluator(derived()).coeff(row,col); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeffByOuterInner(Index outer, Index inner) const { return coeff(rowIndexByOuterInner(outer, inner), colIndexByOuterInner(outer, inner)); } /** \returns the coefficient at given the given row and column. * * \sa operator()(Index,Index), operator[](Index) */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType operator()(Index row, Index col) const { eigen_assert(row >= 0 && row < rows() && col >= 0 && col < cols()); return coeff(row, col); } /** Short version: don't use this function, use * \link operator[](Index) const \endlink instead. * * Long version: this function is similar to * \link operator[](Index) const \endlink, but without the assertion. * Use this for limiting the performance cost of debugging code when doing * repeated coefficient access. Only use this when it is guaranteed that the * parameter \a index is in range. * * If EIGEN_INTERNAL_DEBUGGING is defined, an assertion will be made, making this * function equivalent to \link operator[](Index) const \endlink. * * \sa operator[](Index) const, coeffRef(Index), coeff(Index,Index) const */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { EIGEN_STATIC_ASSERT(internal::evaluator::Flags & LinearAccessBit, THIS_COEFFICIENT_ACCESSOR_TAKING_ONE_ACCESS_IS_ONLY_FOR_EXPRESSIONS_ALLOWING_LINEAR_ACCESS) eigen_internal_assert(index >= 0 && index < size()); return internal::evaluator(derived()).coeff(index); } /** \returns the coefficient at given index. * * This method is allowed only for vector expressions, and for matrix expressions having the LinearAccessBit. * * \sa operator[](Index), operator()(Index,Index) const, x() const, y() const, * z() const, w() const */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType operator[](Index index) const { EIGEN_STATIC_ASSERT(Derived::IsVectorAtCompileTime, THE_BRACKET_OPERATOR_IS_ONLY_FOR_VECTORS__USE_THE_PARENTHESIS_OPERATOR_INSTEAD) eigen_assert(index >= 0 && index < size()); return coeff(index); } /** \returns the coefficient at given index. * * This is synonymous to operator[](Index) const. * * This method is allowed only for vector expressions, and for matrix expressions having the LinearAccessBit. * * \sa operator[](Index), operator()(Index,Index) const, x() const, y() const, * z() const, w() const */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType operator()(Index index) const { eigen_assert(index >= 0 && index < size()); return coeff(index); } /** equivalent to operator[](0). */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType x() const { return (*this)[0]; } /** equivalent to operator[](1). */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType y() const { EIGEN_STATIC_ASSERT(Derived::SizeAtCompileTime==-1 || Derived::SizeAtCompileTime>=2, OUT_OF_RANGE_ACCESS); return (*this)[1]; } /** equivalent to operator[](2). */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType z() const { EIGEN_STATIC_ASSERT(Derived::SizeAtCompileTime==-1 || Derived::SizeAtCompileTime>=3, OUT_OF_RANGE_ACCESS); return (*this)[2]; } /** equivalent to operator[](3). */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType w() const { EIGEN_STATIC_ASSERT(Derived::SizeAtCompileTime==-1 || Derived::SizeAtCompileTime>=4, OUT_OF_RANGE_ACCESS); return (*this)[3]; } /** \internal * \returns the packet of coefficients starting at the given row and column. It is your responsibility * to ensure that a packet really starts there. This method is only available on expressions having the * PacketAccessBit. * * The \a LoadMode parameter may have the value \a #Aligned or \a #Unaligned. Its effect is to select * the appropriate vectorization instruction. Aligned access is faster, but is only possible for packets * starting at an address which is a multiple of the packet size. */ template EIGEN_STRONG_INLINE PacketReturnType packet(Index row, Index col) const { typedef typename internal::packet_traits::type DefaultPacketType; eigen_internal_assert(row >= 0 && row < rows() && col >= 0 && col < cols()); return internal::evaluator(derived()).template packet(row,col); } /** \internal */ template EIGEN_STRONG_INLINE PacketReturnType packetByOuterInner(Index outer, Index inner) const { return packet(rowIndexByOuterInner(outer, inner), colIndexByOuterInner(outer, inner)); } /** \internal * \returns the packet of coefficients starting at the given index. It is your responsibility * to ensure that a packet really starts there. This method is only available on expressions having the * PacketAccessBit and the LinearAccessBit. * * The \a LoadMode parameter may have the value \a #Aligned or \a #Unaligned. Its effect is to select * the appropriate vectorization instruction. Aligned access is faster, but is only possible for packets * starting at an address which is a multiple of the packet size. */ template EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { EIGEN_STATIC_ASSERT(internal::evaluator::Flags & LinearAccessBit, THIS_COEFFICIENT_ACCESSOR_TAKING_ONE_ACCESS_IS_ONLY_FOR_EXPRESSIONS_ALLOWING_LINEAR_ACCESS) typedef typename internal::packet_traits::type DefaultPacketType; eigen_internal_assert(index >= 0 && index < size()); return internal::evaluator(derived()).template packet(index); } protected: // explanation: DenseBase is doing "using ..." on the methods from DenseCoeffsBase. // But some methods are only available in the DirectAccess case. // So we add dummy methods here with these names, so that "using... " doesn't fail. // It's not private so that the child class DenseBase can access them, and it's not public // either since it's an implementation detail, so has to be protected. void coeffRef(); void coeffRefByOuterInner(); void writePacket(); void writePacketByOuterInner(); void copyCoeff(); void copyCoeffByOuterInner(); void copyPacket(); void copyPacketByOuterInner(); void stride(); void innerStride(); void outerStride(); void rowStride(); void colStride(); }; /** \brief Base class providing read/write coefficient access to matrices and arrays. * \ingroup Core_Module * \tparam Derived Type of the derived class * * \note #WriteAccessors Constant indicating read/write access * * This class defines the non-const \c operator() function and friends, which can be used to write specific * entries of a matrix or array. This class inherits DenseCoeffsBase which * defines the const variant for reading specific entries. * * \sa DenseCoeffsBase, \ref TopicClassHierarchy */ template class DenseCoeffsBase : public DenseCoeffsBase { public: typedef DenseCoeffsBase Base; typedef typename internal::traits::StorageKind StorageKind; typedef typename internal::traits::Scalar Scalar; typedef typename internal::packet_traits::type PacketScalar; typedef typename NumTraits::Real RealScalar; using Base::coeff; using Base::rows; using Base::cols; using Base::size; using Base::derived; using Base::rowIndexByOuterInner; using Base::colIndexByOuterInner; using Base::operator[]; using Base::operator(); using Base::x; using Base::y; using Base::z; using Base::w; /** Short version: don't use this function, use * \link operator()(Index,Index) \endlink instead. * * Long version: this function is similar to * \link operator()(Index,Index) \endlink, but without the assertion. * Use this for limiting the performance cost of debugging code when doing * repeated coefficient access. Only use this when it is guaranteed that the * parameters \a row and \a col are in range. * * If EIGEN_INTERNAL_DEBUGGING is defined, an assertion will be made, making this * function equivalent to \link operator()(Index,Index) \endlink. * * \sa operator()(Index,Index), coeff(Index, Index) const, coeffRef(Index) */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { eigen_internal_assert(row >= 0 && row < rows() && col >= 0 && col < cols()); return internal::evaluator(derived()).coeffRef(row,col); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRefByOuterInner(Index outer, Index inner) { return coeffRef(rowIndexByOuterInner(outer, inner), colIndexByOuterInner(outer, inner)); } /** \returns a reference to the coefficient at given the given row and column. * * \sa operator[](Index) */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& operator()(Index row, Index col) { eigen_assert(row >= 0 && row < rows() && col >= 0 && col < cols()); return coeffRef(row, col); } /** Short version: don't use this function, use * \link operator[](Index) \endlink instead. * * Long version: this function is similar to * \link operator[](Index) \endlink, but without the assertion. * Use this for limiting the performance cost of debugging code when doing * repeated coefficient access. Only use this when it is guaranteed that the * parameters \a row and \a col are in range. * * If EIGEN_INTERNAL_DEBUGGING is defined, an assertion will be made, making this * function equivalent to \link operator[](Index) \endlink. * * \sa operator[](Index), coeff(Index) const, coeffRef(Index,Index) */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { EIGEN_STATIC_ASSERT(internal::evaluator::Flags & LinearAccessBit, THIS_COEFFICIENT_ACCESSOR_TAKING_ONE_ACCESS_IS_ONLY_FOR_EXPRESSIONS_ALLOWING_LINEAR_ACCESS) eigen_internal_assert(index >= 0 && index < size()); return internal::evaluator(derived()).coeffRef(index); } /** \returns a reference to the coefficient at given index. * * This method is allowed only for vector expressions, and for matrix expressions having the LinearAccessBit. * * \sa operator[](Index) const, operator()(Index,Index), x(), y(), z(), w() */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& operator[](Index index) { EIGEN_STATIC_ASSERT(Derived::IsVectorAtCompileTime, THE_BRACKET_OPERATOR_IS_ONLY_FOR_VECTORS__USE_THE_PARENTHESIS_OPERATOR_INSTEAD) eigen_assert(index >= 0 && index < size()); return coeffRef(index); } /** \returns a reference to the coefficient at given index. * * This is synonymous to operator[](Index). * * This method is allowed only for vector expressions, and for matrix expressions having the LinearAccessBit. * * \sa operator[](Index) const, operator()(Index,Index), x(), y(), z(), w() */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& operator()(Index index) { eigen_assert(index >= 0 && index < size()); return coeffRef(index); } /** equivalent to operator[](0). */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& x() { return (*this)[0]; } /** equivalent to operator[](1). */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& y() { EIGEN_STATIC_ASSERT(Derived::SizeAtCompileTime==-1 || Derived::SizeAtCompileTime>=2, OUT_OF_RANGE_ACCESS); return (*this)[1]; } /** equivalent to operator[](2). */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& z() { EIGEN_STATIC_ASSERT(Derived::SizeAtCompileTime==-1 || Derived::SizeAtCompileTime>=3, OUT_OF_RANGE_ACCESS); return (*this)[2]; } /** equivalent to operator[](3). */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& w() { EIGEN_STATIC_ASSERT(Derived::SizeAtCompileTime==-1 || Derived::SizeAtCompileTime>=4, OUT_OF_RANGE_ACCESS); return (*this)[3]; } }; /** \brief Base class providing direct read-only coefficient access to matrices and arrays. * \ingroup Core_Module * \tparam Derived Type of the derived class * * \note #DirectAccessors Constant indicating direct access * * This class defines functions to work with strides which can be used to access entries directly. This class * inherits DenseCoeffsBase which defines functions to access entries read-only using * \c operator() . * * \sa \blank \ref TopicClassHierarchy */ template class DenseCoeffsBase : public DenseCoeffsBase { public: typedef DenseCoeffsBase Base; typedef typename internal::traits::Scalar Scalar; typedef typename NumTraits::Real RealScalar; using Base::rows; using Base::cols; using Base::size; using Base::derived; /** \returns the pointer increment between two consecutive elements within a slice in the inner direction. * * \sa outerStride(), rowStride(), colStride() */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const { return derived().innerStride(); } /** \returns the pointer increment between two consecutive inner slices (for example, between two consecutive columns * in a column-major matrix). * * \sa innerStride(), rowStride(), colStride() */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const { return derived().outerStride(); } // FIXME shall we remove it ? EIGEN_CONSTEXPR inline Index stride() const { return Derived::IsVectorAtCompileTime ? innerStride() : outerStride(); } /** \returns the pointer increment between two consecutive rows. * * \sa innerStride(), outerStride(), colStride() */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rowStride() const { return Derived::IsRowMajor ? outerStride() : innerStride(); } /** \returns the pointer increment between two consecutive columns. * * \sa innerStride(), outerStride(), rowStride() */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index colStride() const { return Derived::IsRowMajor ? innerStride() : outerStride(); } }; /** \brief Base class providing direct read/write coefficient access to matrices and arrays. * \ingroup Core_Module * \tparam Derived Type of the derived class * * \note #DirectWriteAccessors Constant indicating direct access * * This class defines functions to work with strides which can be used to access entries directly. This class * inherits DenseCoeffsBase which defines functions to access entries read/write using * \c operator(). * * \sa \blank \ref TopicClassHierarchy */ template class DenseCoeffsBase : public DenseCoeffsBase { public: typedef DenseCoeffsBase Base; typedef typename internal::traits::Scalar Scalar; typedef typename NumTraits::Real RealScalar; using Base::rows; using Base::cols; using Base::size; using Base::derived; /** \returns the pointer increment between two consecutive elements within a slice in the inner direction. * * \sa outerStride(), rowStride(), colStride() */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT { return derived().innerStride(); } /** \returns the pointer increment between two consecutive inner slices (for example, between two consecutive columns * in a column-major matrix). * * \sa innerStride(), rowStride(), colStride() */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT { return derived().outerStride(); } // FIXME shall we remove it ? EIGEN_CONSTEXPR inline Index stride() const EIGEN_NOEXCEPT { return Derived::IsVectorAtCompileTime ? innerStride() : outerStride(); } /** \returns the pointer increment between two consecutive rows. * * \sa innerStride(), outerStride(), colStride() */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rowStride() const EIGEN_NOEXCEPT { return Derived::IsRowMajor ? outerStride() : innerStride(); } /** \returns the pointer increment between two consecutive columns. * * \sa innerStride(), outerStride(), rowStride() */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index colStride() const EIGEN_NOEXCEPT { return Derived::IsRowMajor ? innerStride() : outerStride(); } }; namespace internal { template struct first_aligned_impl { static EIGEN_CONSTEXPR inline Index run(const Derived&) EIGEN_NOEXCEPT { return 0; } }; template struct first_aligned_impl { static inline Index run(const Derived& m) { return internal::first_aligned(m.data(), m.size()); } }; /** \internal \returns the index of the first element of the array stored by \a m that is properly aligned with respect to \a Alignment for vectorization. * * \tparam Alignment requested alignment in Bytes. * * There is also the variant first_aligned(const Scalar*, Integer) defined in Memory.h. See it for more * documentation. */ template static inline Index first_aligned(const DenseBase& m) { enum { ReturnZero = (int(evaluator::Alignment) >= Alignment) || !(Derived::Flags & DirectAccessBit) }; return first_aligned_impl::run(m.derived()); } template static inline Index first_default_aligned(const DenseBase& m) { typedef typename Derived::Scalar Scalar; typedef typename packet_traits::type DefaultPacketType; return internal::first_aligned::alignment),Derived>(m); } template::ret> struct inner_stride_at_compile_time { enum { ret = traits::InnerStrideAtCompileTime }; }; template struct inner_stride_at_compile_time { enum { ret = 0 }; }; template::ret> struct outer_stride_at_compile_time { enum { ret = traits::OuterStrideAtCompileTime }; }; template struct outer_stride_at_compile_time { enum { ret = 0 }; }; } // end namespace internal } // end namespace Eigen #endif // EIGEN_DENSECOEFFSBASE_H RcppEigen/inst/include/Eigen/src/Core/Map.h0000644000176200001440000001613014567757725020154 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2007-2010 Benoit Jacob // Copyright (C) 2008 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_MAP_H #define EIGEN_MAP_H namespace Eigen { namespace internal { template struct traits > : public traits { typedef traits TraitsBase; enum { PlainObjectTypeInnerSize = ((traits::Flags&RowMajorBit)==RowMajorBit) ? PlainObjectType::ColsAtCompileTime : PlainObjectType::RowsAtCompileTime, InnerStrideAtCompileTime = StrideType::InnerStrideAtCompileTime == 0 ? int(PlainObjectType::InnerStrideAtCompileTime) : int(StrideType::InnerStrideAtCompileTime), OuterStrideAtCompileTime = StrideType::OuterStrideAtCompileTime == 0 ? (InnerStrideAtCompileTime==Dynamic || PlainObjectTypeInnerSize==Dynamic ? Dynamic : int(InnerStrideAtCompileTime) * int(PlainObjectTypeInnerSize)) : int(StrideType::OuterStrideAtCompileTime), Alignment = int(MapOptions)&int(AlignedMask), Flags0 = TraitsBase::Flags & (~NestByRefBit), Flags = is_lvalue::value ? int(Flags0) : (int(Flags0) & ~LvalueBit) }; private: enum { Options }; // Expressions don't have Options }; } /** \class Map * \ingroup Core_Module * * \brief A matrix or vector expression mapping an existing array of data. * * \tparam PlainObjectType the equivalent matrix type of the mapped data * \tparam MapOptions specifies the pointer alignment in bytes. It can be: \c #Aligned128, \c #Aligned64, \c #Aligned32, \c #Aligned16, \c #Aligned8 or \c #Unaligned. * The default is \c #Unaligned. * \tparam StrideType optionally specifies strides. By default, Map assumes the memory layout * of an ordinary, contiguous array. This can be overridden by specifying strides. * The type passed here must be a specialization of the Stride template, see examples below. * * This class represents a matrix or vector expression mapping an existing array of data. * It can be used to let Eigen interface without any overhead with non-Eigen data structures, * such as plain C arrays or structures from other libraries. By default, it assumes that the * data is laid out contiguously in memory. You can however override this by explicitly specifying * inner and outer strides. * * Here's an example of simply mapping a contiguous array as a \ref TopicStorageOrders "column-major" matrix: * \include Map_simple.cpp * Output: \verbinclude Map_simple.out * * If you need to map non-contiguous arrays, you can do so by specifying strides: * * Here's an example of mapping an array as a vector, specifying an inner stride, that is, the pointer * increment between two consecutive coefficients. Here, we're specifying the inner stride as a compile-time * fixed value. * \include Map_inner_stride.cpp * Output: \verbinclude Map_inner_stride.out * * Here's an example of mapping an array while specifying an outer stride. Here, since we're mapping * as a column-major matrix, 'outer stride' means the pointer increment between two consecutive columns. * Here, we're specifying the outer stride as a runtime parameter. Note that here \c OuterStride<> is * a short version of \c OuterStride because the default template parameter of OuterStride * is \c Dynamic * \include Map_outer_stride.cpp * Output: \verbinclude Map_outer_stride.out * * For more details and for an example of specifying both an inner and an outer stride, see class Stride. * * \b Tip: to change the array of data mapped by a Map object, you can use the C++ * placement new syntax: * * Example: \include Map_placement_new.cpp * Output: \verbinclude Map_placement_new.out * * This class is the return type of PlainObjectBase::Map() but can also be used directly. * * \sa PlainObjectBase::Map(), \ref TopicStorageOrders */ template class Map : public MapBase > { public: typedef MapBase Base; EIGEN_DENSE_PUBLIC_INTERFACE(Map) typedef typename Base::PointerType PointerType; typedef PointerType PointerArgType; EIGEN_DEVICE_FUNC inline PointerType cast_to_pointer_type(PointerArgType ptr) { return ptr; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const { return StrideType::InnerStrideAtCompileTime != 0 ? m_stride.inner() : 1; } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const { return StrideType::OuterStrideAtCompileTime != 0 ? m_stride.outer() : internal::traits::OuterStrideAtCompileTime != Dynamic ? Index(internal::traits::OuterStrideAtCompileTime) : IsVectorAtCompileTime ? (this->size() * innerStride()) : int(Flags)&RowMajorBit ? (this->cols() * innerStride()) : (this->rows() * innerStride()); } /** Constructor in the fixed-size case. * * \param dataPtr pointer to the array to map * \param stride optional Stride object, passing the strides. */ EIGEN_DEVICE_FUNC explicit inline Map(PointerArgType dataPtr, const StrideType& stride = StrideType()) : Base(cast_to_pointer_type(dataPtr)), m_stride(stride) { PlainObjectType::Base::_check_template_params(); } /** Constructor in the dynamic-size vector case. * * \param dataPtr pointer to the array to map * \param size the size of the vector expression * \param stride optional Stride object, passing the strides. */ EIGEN_DEVICE_FUNC inline Map(PointerArgType dataPtr, Index size, const StrideType& stride = StrideType()) : Base(cast_to_pointer_type(dataPtr), size), m_stride(stride) { PlainObjectType::Base::_check_template_params(); } /** Constructor in the dynamic-size matrix case. * * \param dataPtr pointer to the array to map * \param rows the number of rows of the matrix expression * \param cols the number of columns of the matrix expression * \param stride optional Stride object, passing the strides. */ EIGEN_DEVICE_FUNC inline Map(PointerArgType dataPtr, Index rows, Index cols, const StrideType& stride = StrideType()) : Base(cast_to_pointer_type(dataPtr), rows, cols), m_stride(stride) { PlainObjectType::Base::_check_template_params(); } EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Map) protected: StrideType m_stride; }; } // end namespace Eigen #endif // EIGEN_MAP_H RcppEigen/inst/include/Eigen/src/Core/CwiseNullaryOp.h0000644000176200001440000010667214567757725022372 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2010 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_CWISE_NULLARY_OP_H #define EIGEN_CWISE_NULLARY_OP_H namespace Eigen { namespace internal { template struct traits > : traits { enum { Flags = traits::Flags & RowMajorBit }; }; } // namespace internal /** \class CwiseNullaryOp * \ingroup Core_Module * * \brief Generic expression of a matrix where all coefficients are defined by a functor * * \tparam NullaryOp template functor implementing the operator * \tparam PlainObjectType the underlying plain matrix/array type * * This class represents an expression of a generic nullary operator. * It is the return type of the Ones(), Zero(), Constant(), Identity() and Random() methods, * and most of the time this is the only way it is used. * * However, if you want to write a function returning such an expression, you * will need to use this class. * * The functor NullaryOp must expose one of the following method:
\c operator()() if the procedural generation does not depend on the coefficient entries (e.g., random numbers)
\c operator()(Index i)if the procedural generation makes sense for vectors only and that it depends on the coefficient index \c i (e.g., linspace)
\c operator()(Index i,Index j)if the procedural generation depends on the matrix coordinates \c i, \c j (e.g., to generate a checkerboard with 0 and 1)
* It is also possible to expose the last two operators if the generation makes sense for matrices but can be optimized for vectors. * * See DenseBase::NullaryExpr(Index,const CustomNullaryOp&) for an example binding * C++11 random number generators. * * A nullary expression can also be used to implement custom sophisticated matrix manipulations * that cannot be covered by the existing set of natively supported matrix manipulations. * See this \ref TopicCustomizing_NullaryExpr "page" for some examples and additional explanations * on the behavior of CwiseNullaryOp. * * \sa class CwiseUnaryOp, class CwiseBinaryOp, DenseBase::NullaryExpr */ template class CwiseNullaryOp : public internal::dense_xpr_base< CwiseNullaryOp >::type, internal::no_assignment_operator { public: typedef typename internal::dense_xpr_base::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(CwiseNullaryOp) EIGEN_DEVICE_FUNC CwiseNullaryOp(Index rows, Index cols, const NullaryOp& func = NullaryOp()) : m_rows(rows), m_cols(cols), m_functor(func) { eigen_assert(rows >= 0 && (RowsAtCompileTime == Dynamic || RowsAtCompileTime == rows) && cols >= 0 && (ColsAtCompileTime == Dynamic || ColsAtCompileTime == cols)); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rows() const { return m_rows.value(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index cols() const { return m_cols.value(); } /** \returns the functor representing the nullary operation */ EIGEN_DEVICE_FUNC const NullaryOp& functor() const { return m_functor; } protected: const internal::variable_if_dynamic m_rows; const internal::variable_if_dynamic m_cols; const NullaryOp m_functor; }; /** \returns an expression of a matrix defined by a custom functor \a func * * The parameters \a rows and \a cols are the number of rows and of columns of * the returned matrix. Must be compatible with this MatrixBase type. * * This variant is meant to be used for dynamic-size matrix types. For fixed-size types, * it is redundant to pass \a rows and \a cols as arguments, so Zero() should be used * instead. * * The template parameter \a CustomNullaryOp is the type of the functor. * * \sa class CwiseNullaryOp */ template template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE #ifndef EIGEN_PARSED_BY_DOXYGEN const CwiseNullaryOp::PlainObject> #else const CwiseNullaryOp #endif DenseBase::NullaryExpr(Index rows, Index cols, const CustomNullaryOp& func) { return CwiseNullaryOp(rows, cols, func); } /** \returns an expression of a matrix defined by a custom functor \a func * * The parameter \a size is the size of the returned vector. * Must be compatible with this MatrixBase type. * * \only_for_vectors * * This variant is meant to be used for dynamic-size vector types. For fixed-size types, * it is redundant to pass \a size as argument, so Zero() should be used * instead. * * The template parameter \a CustomNullaryOp is the type of the functor. * * Here is an example with C++11 random generators: \include random_cpp11.cpp * Output: \verbinclude random_cpp11.out * * \sa class CwiseNullaryOp */ template template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE #ifndef EIGEN_PARSED_BY_DOXYGEN const CwiseNullaryOp::PlainObject> #else const CwiseNullaryOp #endif DenseBase::NullaryExpr(Index size, const CustomNullaryOp& func) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) if(RowsAtCompileTime == 1) return CwiseNullaryOp(1, size, func); else return CwiseNullaryOp(size, 1, func); } /** \returns an expression of a matrix defined by a custom functor \a func * * This variant is only for fixed-size DenseBase types. For dynamic-size types, you * need to use the variants taking size arguments. * * The template parameter \a CustomNullaryOp is the type of the functor. * * \sa class CwiseNullaryOp */ template template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE #ifndef EIGEN_PARSED_BY_DOXYGEN const CwiseNullaryOp::PlainObject> #else const CwiseNullaryOp #endif DenseBase::NullaryExpr(const CustomNullaryOp& func) { return CwiseNullaryOp(RowsAtCompileTime, ColsAtCompileTime, func); } /** \returns an expression of a constant matrix of value \a value * * The parameters \a rows and \a cols are the number of rows and of columns of * the returned matrix. Must be compatible with this DenseBase type. * * This variant is meant to be used for dynamic-size matrix types. For fixed-size types, * it is redundant to pass \a rows and \a cols as arguments, so Zero() should be used * instead. * * The template parameter \a CustomNullaryOp is the type of the functor. * * \sa class CwiseNullaryOp */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase::ConstantReturnType DenseBase::Constant(Index rows, Index cols, const Scalar& value) { return DenseBase::NullaryExpr(rows, cols, internal::scalar_constant_op(value)); } /** \returns an expression of a constant matrix of value \a value * * The parameter \a size is the size of the returned vector. * Must be compatible with this DenseBase type. * * \only_for_vectors * * This variant is meant to be used for dynamic-size vector types. For fixed-size types, * it is redundant to pass \a size as argument, so Zero() should be used * instead. * * The template parameter \a CustomNullaryOp is the type of the functor. * * \sa class CwiseNullaryOp */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase::ConstantReturnType DenseBase::Constant(Index size, const Scalar& value) { return DenseBase::NullaryExpr(size, internal::scalar_constant_op(value)); } /** \returns an expression of a constant matrix of value \a value * * This variant is only for fixed-size DenseBase types. For dynamic-size types, you * need to use the variants taking size arguments. * * The template parameter \a CustomNullaryOp is the type of the functor. * * \sa class CwiseNullaryOp */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase::ConstantReturnType DenseBase::Constant(const Scalar& value) { EIGEN_STATIC_ASSERT_FIXED_SIZE(Derived) return DenseBase::NullaryExpr(RowsAtCompileTime, ColsAtCompileTime, internal::scalar_constant_op(value)); } /** \deprecated because of accuracy loss. In Eigen 3.3, it is an alias for LinSpaced(Index,const Scalar&,const Scalar&) * * \only_for_vectors * * Example: \include DenseBase_LinSpaced_seq_deprecated.cpp * Output: \verbinclude DenseBase_LinSpaced_seq_deprecated.out * * \sa LinSpaced(Index,const Scalar&, const Scalar&), setLinSpaced(Index,const Scalar&,const Scalar&) */ template EIGEN_DEPRECATED EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase::RandomAccessLinSpacedReturnType DenseBase::LinSpaced(Sequential_t, Index size, const Scalar& low, const Scalar& high) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return DenseBase::NullaryExpr(size, internal::linspaced_op(low,high,size)); } /** \deprecated because of accuracy loss. In Eigen 3.3, it is an alias for LinSpaced(const Scalar&,const Scalar&) * * \sa LinSpaced(const Scalar&, const Scalar&) */ template EIGEN_DEPRECATED EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase::RandomAccessLinSpacedReturnType DenseBase::LinSpaced(Sequential_t, const Scalar& low, const Scalar& high) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) EIGEN_STATIC_ASSERT_FIXED_SIZE(Derived) return DenseBase::NullaryExpr(Derived::SizeAtCompileTime, internal::linspaced_op(low,high,Derived::SizeAtCompileTime)); } /** * \brief Sets a linearly spaced vector. * * The function generates 'size' equally spaced values in the closed interval [low,high]. * When size is set to 1, a vector of length 1 containing 'high' is returned. * * \only_for_vectors * * Example: \include DenseBase_LinSpaced.cpp * Output: \verbinclude DenseBase_LinSpaced.out * * For integer scalar types, an even spacing is possible if and only if the length of the range, * i.e., \c high-low is a scalar multiple of \c size-1, or if \c size is a scalar multiple of the * number of values \c high-low+1 (meaning each value can be repeated the same number of time). * If one of these two considions is not satisfied, then \c high is lowered to the largest value * satisfying one of this constraint. * Here are some examples: * * Example: \include DenseBase_LinSpacedInt.cpp * Output: \verbinclude DenseBase_LinSpacedInt.out * * \sa setLinSpaced(Index,const Scalar&,const Scalar&), CwiseNullaryOp */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase::RandomAccessLinSpacedReturnType DenseBase::LinSpaced(Index size, const Scalar& low, const Scalar& high) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return DenseBase::NullaryExpr(size, internal::linspaced_op(low,high,size)); } /** * \copydoc DenseBase::LinSpaced(Index, const Scalar&, const Scalar&) * Special version for fixed size types which does not require the size parameter. */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase::RandomAccessLinSpacedReturnType DenseBase::LinSpaced(const Scalar& low, const Scalar& high) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) EIGEN_STATIC_ASSERT_FIXED_SIZE(Derived) return DenseBase::NullaryExpr(Derived::SizeAtCompileTime, internal::linspaced_op(low,high,Derived::SizeAtCompileTime)); } /** \returns true if all coefficients in this matrix are approximately equal to \a val, to within precision \a prec */ template EIGEN_DEVICE_FUNC bool DenseBase::isApproxToConstant (const Scalar& val, const RealScalar& prec) const { typename internal::nested_eval::type self(derived()); for(Index j = 0; j < cols(); ++j) for(Index i = 0; i < rows(); ++i) if(!internal::isApprox(self.coeff(i, j), val, prec)) return false; return true; } /** This is just an alias for isApproxToConstant(). * * \returns true if all coefficients in this matrix are approximately equal to \a value, to within precision \a prec */ template EIGEN_DEVICE_FUNC bool DenseBase::isConstant (const Scalar& val, const RealScalar& prec) const { return isApproxToConstant(val, prec); } /** Alias for setConstant(): sets all coefficients in this expression to \a val. * * \sa setConstant(), Constant(), class CwiseNullaryOp */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void DenseBase::fill(const Scalar& val) { setConstant(val); } /** Sets all coefficients in this expression to value \a val. * * \sa fill(), setConstant(Index,const Scalar&), setConstant(Index,Index,const Scalar&), setZero(), setOnes(), Constant(), class CwiseNullaryOp, setZero(), setOnes() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase::setConstant(const Scalar& val) { return derived() = Constant(rows(), cols(), val); } /** Resizes to the given \a size, and sets all coefficients in this expression to the given value \a val. * * \only_for_vectors * * Example: \include Matrix_setConstant_int.cpp * Output: \verbinclude Matrix_setConstant_int.out * * \sa MatrixBase::setConstant(const Scalar&), setConstant(Index,Index,const Scalar&), class CwiseNullaryOp, MatrixBase::Constant(const Scalar&) */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase::setConstant(Index size, const Scalar& val) { resize(size); return setConstant(val); } /** Resizes to the given size, and sets all coefficients in this expression to the given value \a val. * * \param rows the new number of rows * \param cols the new number of columns * \param val the value to which all coefficients are set * * Example: \include Matrix_setConstant_int_int.cpp * Output: \verbinclude Matrix_setConstant_int_int.out * * \sa MatrixBase::setConstant(const Scalar&), setConstant(Index,const Scalar&), class CwiseNullaryOp, MatrixBase::Constant(const Scalar&) */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase::setConstant(Index rows, Index cols, const Scalar& val) { resize(rows, cols); return setConstant(val); } /** Resizes to the given size, changing only the number of columns, and sets all * coefficients in this expression to the given value \a val. For the parameter * of type NoChange_t, just pass the special value \c NoChange. * * \sa MatrixBase::setConstant(const Scalar&), setConstant(Index,const Scalar&), class CwiseNullaryOp, MatrixBase::Constant(const Scalar&) */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase::setConstant(NoChange_t, Index cols, const Scalar& val) { return setConstant(rows(), cols, val); } /** Resizes to the given size, changing only the number of rows, and sets all * coefficients in this expression to the given value \a val. For the parameter * of type NoChange_t, just pass the special value \c NoChange. * * \sa MatrixBase::setConstant(const Scalar&), setConstant(Index,const Scalar&), class CwiseNullaryOp, MatrixBase::Constant(const Scalar&) */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase::setConstant(Index rows, NoChange_t, const Scalar& val) { return setConstant(rows, cols(), val); } /** * \brief Sets a linearly spaced vector. * * The function generates 'size' equally spaced values in the closed interval [low,high]. * When size is set to 1, a vector of length 1 containing 'high' is returned. * * \only_for_vectors * * Example: \include DenseBase_setLinSpaced.cpp * Output: \verbinclude DenseBase_setLinSpaced.out * * For integer scalar types, do not miss the explanations on the definition * of \link LinSpaced(Index,const Scalar&,const Scalar&) even spacing \endlink. * * \sa LinSpaced(Index,const Scalar&,const Scalar&), CwiseNullaryOp */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase::setLinSpaced(Index newSize, const Scalar& low, const Scalar& high) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return derived() = Derived::NullaryExpr(newSize, internal::linspaced_op(low,high,newSize)); } /** * \brief Sets a linearly spaced vector. * * The function fills \c *this with equally spaced values in the closed interval [low,high]. * When size is set to 1, a vector of length 1 containing 'high' is returned. * * \only_for_vectors * * For integer scalar types, do not miss the explanations on the definition * of \link LinSpaced(Index,const Scalar&,const Scalar&) even spacing \endlink. * * \sa LinSpaced(Index,const Scalar&,const Scalar&), setLinSpaced(Index, const Scalar&, const Scalar&), CwiseNullaryOp */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase::setLinSpaced(const Scalar& low, const Scalar& high) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return setLinSpaced(size(), low, high); } // zero: /** \returns an expression of a zero matrix. * * The parameters \a rows and \a cols are the number of rows and of columns of * the returned matrix. Must be compatible with this MatrixBase type. * * This variant is meant to be used for dynamic-size matrix types. For fixed-size types, * it is redundant to pass \a rows and \a cols as arguments, so Zero() should be used * instead. * * Example: \include MatrixBase_zero_int_int.cpp * Output: \verbinclude MatrixBase_zero_int_int.out * * \sa Zero(), Zero(Index) */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase::ConstantReturnType DenseBase::Zero(Index rows, Index cols) { return Constant(rows, cols, Scalar(0)); } /** \returns an expression of a zero vector. * * The parameter \a size is the size of the returned vector. * Must be compatible with this MatrixBase type. * * \only_for_vectors * * This variant is meant to be used for dynamic-size vector types. For fixed-size types, * it is redundant to pass \a size as argument, so Zero() should be used * instead. * * Example: \include MatrixBase_zero_int.cpp * Output: \verbinclude MatrixBase_zero_int.out * * \sa Zero(), Zero(Index,Index) */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase::ConstantReturnType DenseBase::Zero(Index size) { return Constant(size, Scalar(0)); } /** \returns an expression of a fixed-size zero matrix or vector. * * This variant is only for fixed-size MatrixBase types. For dynamic-size types, you * need to use the variants taking size arguments. * * Example: \include MatrixBase_zero.cpp * Output: \verbinclude MatrixBase_zero.out * * \sa Zero(Index), Zero(Index,Index) */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase::ConstantReturnType DenseBase::Zero() { return Constant(Scalar(0)); } /** \returns true if *this is approximately equal to the zero matrix, * within the precision given by \a prec. * * Example: \include MatrixBase_isZero.cpp * Output: \verbinclude MatrixBase_isZero.out * * \sa class CwiseNullaryOp, Zero() */ template EIGEN_DEVICE_FUNC bool DenseBase::isZero(const RealScalar& prec) const { typename internal::nested_eval::type self(derived()); for(Index j = 0; j < cols(); ++j) for(Index i = 0; i < rows(); ++i) if(!internal::isMuchSmallerThan(self.coeff(i, j), static_cast(1), prec)) return false; return true; } /** Sets all coefficients in this expression to zero. * * Example: \include MatrixBase_setZero.cpp * Output: \verbinclude MatrixBase_setZero.out * * \sa class CwiseNullaryOp, Zero() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase::setZero() { return setConstant(Scalar(0)); } /** Resizes to the given \a size, and sets all coefficients in this expression to zero. * * \only_for_vectors * * Example: \include Matrix_setZero_int.cpp * Output: \verbinclude Matrix_setZero_int.out * * \sa DenseBase::setZero(), setZero(Index,Index), class CwiseNullaryOp, DenseBase::Zero() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase::setZero(Index newSize) { resize(newSize); return setConstant(Scalar(0)); } /** Resizes to the given size, and sets all coefficients in this expression to zero. * * \param rows the new number of rows * \param cols the new number of columns * * Example: \include Matrix_setZero_int_int.cpp * Output: \verbinclude Matrix_setZero_int_int.out * * \sa DenseBase::setZero(), setZero(Index), class CwiseNullaryOp, DenseBase::Zero() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase::setZero(Index rows, Index cols) { resize(rows, cols); return setConstant(Scalar(0)); } /** Resizes to the given size, changing only the number of columns, and sets all * coefficients in this expression to zero. For the parameter of type NoChange_t, * just pass the special value \c NoChange. * * \sa DenseBase::setZero(), setZero(Index), setZero(Index, Index), setZero(Index, NoChange_t), class CwiseNullaryOp, DenseBase::Zero() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase::setZero(NoChange_t, Index cols) { return setZero(rows(), cols); } /** Resizes to the given size, changing only the number of rows, and sets all * coefficients in this expression to zero. For the parameter of type NoChange_t, * just pass the special value \c NoChange. * * \sa DenseBase::setZero(), setZero(Index), setZero(Index, Index), setZero(NoChange_t, Index), class CwiseNullaryOp, DenseBase::Zero() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase::setZero(Index rows, NoChange_t) { return setZero(rows, cols()); } // ones: /** \returns an expression of a matrix where all coefficients equal one. * * The parameters \a rows and \a cols are the number of rows and of columns of * the returned matrix. Must be compatible with this MatrixBase type. * * This variant is meant to be used for dynamic-size matrix types. For fixed-size types, * it is redundant to pass \a rows and \a cols as arguments, so Ones() should be used * instead. * * Example: \include MatrixBase_ones_int_int.cpp * Output: \verbinclude MatrixBase_ones_int_int.out * * \sa Ones(), Ones(Index), isOnes(), class Ones */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase::ConstantReturnType DenseBase::Ones(Index rows, Index cols) { return Constant(rows, cols, Scalar(1)); } /** \returns an expression of a vector where all coefficients equal one. * * The parameter \a newSize is the size of the returned vector. * Must be compatible with this MatrixBase type. * * \only_for_vectors * * This variant is meant to be used for dynamic-size vector types. For fixed-size types, * it is redundant to pass \a size as argument, so Ones() should be used * instead. * * Example: \include MatrixBase_ones_int.cpp * Output: \verbinclude MatrixBase_ones_int.out * * \sa Ones(), Ones(Index,Index), isOnes(), class Ones */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase::ConstantReturnType DenseBase::Ones(Index newSize) { return Constant(newSize, Scalar(1)); } /** \returns an expression of a fixed-size matrix or vector where all coefficients equal one. * * This variant is only for fixed-size MatrixBase types. For dynamic-size types, you * need to use the variants taking size arguments. * * Example: \include MatrixBase_ones.cpp * Output: \verbinclude MatrixBase_ones.out * * \sa Ones(Index), Ones(Index,Index), isOnes(), class Ones */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase::ConstantReturnType DenseBase::Ones() { return Constant(Scalar(1)); } /** \returns true if *this is approximately equal to the matrix where all coefficients * are equal to 1, within the precision given by \a prec. * * Example: \include MatrixBase_isOnes.cpp * Output: \verbinclude MatrixBase_isOnes.out * * \sa class CwiseNullaryOp, Ones() */ template EIGEN_DEVICE_FUNC bool DenseBase::isOnes (const RealScalar& prec) const { return isApproxToConstant(Scalar(1), prec); } /** Sets all coefficients in this expression to one. * * Example: \include MatrixBase_setOnes.cpp * Output: \verbinclude MatrixBase_setOnes.out * * \sa class CwiseNullaryOp, Ones() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase::setOnes() { return setConstant(Scalar(1)); } /** Resizes to the given \a newSize, and sets all coefficients in this expression to one. * * \only_for_vectors * * Example: \include Matrix_setOnes_int.cpp * Output: \verbinclude Matrix_setOnes_int.out * * \sa MatrixBase::setOnes(), setOnes(Index,Index), class CwiseNullaryOp, MatrixBase::Ones() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase::setOnes(Index newSize) { resize(newSize); return setConstant(Scalar(1)); } /** Resizes to the given size, and sets all coefficients in this expression to one. * * \param rows the new number of rows * \param cols the new number of columns * * Example: \include Matrix_setOnes_int_int.cpp * Output: \verbinclude Matrix_setOnes_int_int.out * * \sa MatrixBase::setOnes(), setOnes(Index), class CwiseNullaryOp, MatrixBase::Ones() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase::setOnes(Index rows, Index cols) { resize(rows, cols); return setConstant(Scalar(1)); } /** Resizes to the given size, changing only the number of rows, and sets all * coefficients in this expression to one. For the parameter of type NoChange_t, * just pass the special value \c NoChange. * * \sa MatrixBase::setOnes(), setOnes(Index), setOnes(Index, Index), setOnes(NoChange_t, Index), class CwiseNullaryOp, MatrixBase::Ones() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase::setOnes(Index rows, NoChange_t) { return setOnes(rows, cols()); } /** Resizes to the given size, changing only the number of columns, and sets all * coefficients in this expression to one. For the parameter of type NoChange_t, * just pass the special value \c NoChange. * * \sa MatrixBase::setOnes(), setOnes(Index), setOnes(Index, Index), setOnes(Index, NoChange_t) class CwiseNullaryOp, MatrixBase::Ones() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase::setOnes(NoChange_t, Index cols) { return setOnes(rows(), cols); } // Identity: /** \returns an expression of the identity matrix (not necessarily square). * * The parameters \a rows and \a cols are the number of rows and of columns of * the returned matrix. Must be compatible with this MatrixBase type. * * This variant is meant to be used for dynamic-size matrix types. For fixed-size types, * it is redundant to pass \a rows and \a cols as arguments, so Identity() should be used * instead. * * Example: \include MatrixBase_identity_int_int.cpp * Output: \verbinclude MatrixBase_identity_int_int.out * * \sa Identity(), setIdentity(), isIdentity() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase::IdentityReturnType MatrixBase::Identity(Index rows, Index cols) { return DenseBase::NullaryExpr(rows, cols, internal::scalar_identity_op()); } /** \returns an expression of the identity matrix (not necessarily square). * * This variant is only for fixed-size MatrixBase types. For dynamic-size types, you * need to use the variant taking size arguments. * * Example: \include MatrixBase_identity.cpp * Output: \verbinclude MatrixBase_identity.out * * \sa Identity(Index,Index), setIdentity(), isIdentity() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase::IdentityReturnType MatrixBase::Identity() { EIGEN_STATIC_ASSERT_FIXED_SIZE(Derived) return MatrixBase::NullaryExpr(RowsAtCompileTime, ColsAtCompileTime, internal::scalar_identity_op()); } /** \returns true if *this is approximately equal to the identity matrix * (not necessarily square), * within the precision given by \a prec. * * Example: \include MatrixBase_isIdentity.cpp * Output: \verbinclude MatrixBase_isIdentity.out * * \sa class CwiseNullaryOp, Identity(), Identity(Index,Index), setIdentity() */ template bool MatrixBase::isIdentity (const RealScalar& prec) const { typename internal::nested_eval::type self(derived()); for(Index j = 0; j < cols(); ++j) { for(Index i = 0; i < rows(); ++i) { if(i == j) { if(!internal::isApprox(self.coeff(i, j), static_cast(1), prec)) return false; } else { if(!internal::isMuchSmallerThan(self.coeff(i, j), static_cast(1), prec)) return false; } } } return true; } namespace internal { template=16)> struct setIdentity_impl { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Derived& run(Derived& m) { return m = Derived::Identity(m.rows(), m.cols()); } }; template struct setIdentity_impl { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Derived& run(Derived& m) { m.setZero(); const Index size = numext::mini(m.rows(), m.cols()); for(Index i = 0; i < size; ++i) m.coeffRef(i,i) = typename Derived::Scalar(1); return m; } }; } // end namespace internal /** Writes the identity expression (not necessarily square) into *this. * * Example: \include MatrixBase_setIdentity.cpp * Output: \verbinclude MatrixBase_setIdentity.out * * \sa class CwiseNullaryOp, Identity(), Identity(Index,Index), isIdentity() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase::setIdentity() { return internal::setIdentity_impl::run(derived()); } /** \brief Resizes to the given size, and writes the identity expression (not necessarily square) into *this. * * \param rows the new number of rows * \param cols the new number of columns * * Example: \include Matrix_setIdentity_int_int.cpp * Output: \verbinclude Matrix_setIdentity_int_int.out * * \sa MatrixBase::setIdentity(), class CwiseNullaryOp, MatrixBase::Identity() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase::setIdentity(Index rows, Index cols) { derived().resize(rows, cols); return setIdentity(); } /** \returns an expression of the i-th unit (basis) vector. * * \only_for_vectors * * \sa MatrixBase::Unit(Index), MatrixBase::UnitX(), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase::BasisReturnType MatrixBase::Unit(Index newSize, Index i) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return BasisReturnType(SquareMatrixType::Identity(newSize,newSize), i); } /** \returns an expression of the i-th unit (basis) vector. * * \only_for_vectors * * This variant is for fixed-size vector only. * * \sa MatrixBase::Unit(Index,Index), MatrixBase::UnitX(), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase::BasisReturnType MatrixBase::Unit(Index i) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) return BasisReturnType(SquareMatrixType::Identity(),i); } /** \returns an expression of the X axis unit vector (1{,0}^*) * * \only_for_vectors * * \sa MatrixBase::Unit(Index,Index), MatrixBase::Unit(Index), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase::BasisReturnType MatrixBase::UnitX() { return Derived::Unit(0); } /** \returns an expression of the Y axis unit vector (0,1{,0}^*) * * \only_for_vectors * * \sa MatrixBase::Unit(Index,Index), MatrixBase::Unit(Index), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase::BasisReturnType MatrixBase::UnitY() { return Derived::Unit(1); } /** \returns an expression of the Z axis unit vector (0,0,1{,0}^*) * * \only_for_vectors * * \sa MatrixBase::Unit(Index,Index), MatrixBase::Unit(Index), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase::BasisReturnType MatrixBase::UnitZ() { return Derived::Unit(2); } /** \returns an expression of the W axis unit vector (0,0,0,1) * * \only_for_vectors * * \sa MatrixBase::Unit(Index,Index), MatrixBase::Unit(Index), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW() */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase::BasisReturnType MatrixBase::UnitW() { return Derived::Unit(3); } /** \brief Set the coefficients of \c *this to the i-th unit (basis) vector * * \param i index of the unique coefficient to be set to 1 * * \only_for_vectors * * \sa MatrixBase::setIdentity(), class CwiseNullaryOp, MatrixBase::Unit(Index,Index) */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase::setUnit(Index i) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived); eigen_assert(i EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase::setUnit(Index newSize, Index i) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived); eigen_assert(i // Copyright (C) 2008-2010 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_DENSEBASE_H #define EIGEN_DENSEBASE_H namespace Eigen { namespace internal { // The index type defined by EIGEN_DEFAULT_DENSE_INDEX_TYPE must be a signed type. // This dummy function simply aims at checking that at compile time. static inline void check_DenseIndex_is_signed() { EIGEN_STATIC_ASSERT(NumTraits::IsSigned,THE_INDEX_TYPE_MUST_BE_A_SIGNED_TYPE) } } // end namespace internal /** \class DenseBase * \ingroup Core_Module * * \brief Base class for all dense matrices, vectors, and arrays * * This class is the base that is inherited by all dense objects (matrix, vector, arrays, * and related expression types). The common Eigen API for dense objects is contained in this class. * * \tparam Derived is the derived type, e.g., a matrix type or an expression. * * This class can be extended with the help of the plugin mechanism described on the page * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_DENSEBASE_PLUGIN. * * \sa \blank \ref TopicClassHierarchy */ template class DenseBase #ifndef EIGEN_PARSED_BY_DOXYGEN : public DenseCoeffsBase::value> #else : public DenseCoeffsBase #endif // not EIGEN_PARSED_BY_DOXYGEN { public: /** Inner iterator type to iterate over the coefficients of a row or column. * \sa class InnerIterator */ typedef Eigen::InnerIterator InnerIterator; typedef typename internal::traits::StorageKind StorageKind; /** * \brief The type used to store indices * \details This typedef is relevant for types that store multiple indices such as * PermutationMatrix or Transpositions, otherwise it defaults to Eigen::Index * \sa \blank \ref TopicPreprocessorDirectives, Eigen::Index, SparseMatrixBase. */ typedef typename internal::traits::StorageIndex StorageIndex; /** The numeric type of the expression' coefficients, e.g. float, double, int or std::complex, etc. */ typedef typename internal::traits::Scalar Scalar; /** The numeric type of the expression' coefficients, e.g. float, double, int or std::complex, etc. * * It is an alias for the Scalar type */ typedef Scalar value_type; typedef typename NumTraits::Real RealScalar; typedef DenseCoeffsBase::value> Base; using Base::derived; using Base::const_cast_derived; using Base::rows; using Base::cols; using Base::size; using Base::rowIndexByOuterInner; using Base::colIndexByOuterInner; using Base::coeff; using Base::coeffByOuterInner; using Base::operator(); using Base::operator[]; using Base::x; using Base::y; using Base::z; using Base::w; using Base::stride; using Base::innerStride; using Base::outerStride; using Base::rowStride; using Base::colStride; typedef typename Base::CoeffReturnType CoeffReturnType; enum { RowsAtCompileTime = internal::traits::RowsAtCompileTime, /**< The number of rows at compile-time. This is just a copy of the value provided * by the \a Derived type. If a value is not known at compile-time, * it is set to the \a Dynamic constant. * \sa MatrixBase::rows(), MatrixBase::cols(), ColsAtCompileTime, SizeAtCompileTime */ ColsAtCompileTime = internal::traits::ColsAtCompileTime, /**< The number of columns at compile-time. This is just a copy of the value provided * by the \a Derived type. If a value is not known at compile-time, * it is set to the \a Dynamic constant. * \sa MatrixBase::rows(), MatrixBase::cols(), RowsAtCompileTime, SizeAtCompileTime */ SizeAtCompileTime = (internal::size_at_compile_time::RowsAtCompileTime, internal::traits::ColsAtCompileTime>::ret), /**< This is equal to the number of coefficients, i.e. the number of * rows times the number of columns, or to \a Dynamic if this is not * known at compile-time. \sa RowsAtCompileTime, ColsAtCompileTime */ MaxRowsAtCompileTime = internal::traits::MaxRowsAtCompileTime, /**< This value is equal to the maximum possible number of rows that this expression * might have. If this expression might have an arbitrarily high number of rows, * this value is set to \a Dynamic. * * This value is useful to know when evaluating an expression, in order to determine * whether it is possible to avoid doing a dynamic memory allocation. * * \sa RowsAtCompileTime, MaxColsAtCompileTime, MaxSizeAtCompileTime */ MaxColsAtCompileTime = internal::traits::MaxColsAtCompileTime, /**< This value is equal to the maximum possible number of columns that this expression * might have. If this expression might have an arbitrarily high number of columns, * this value is set to \a Dynamic. * * This value is useful to know when evaluating an expression, in order to determine * whether it is possible to avoid doing a dynamic memory allocation. * * \sa ColsAtCompileTime, MaxRowsAtCompileTime, MaxSizeAtCompileTime */ MaxSizeAtCompileTime = (internal::size_at_compile_time::MaxRowsAtCompileTime, internal::traits::MaxColsAtCompileTime>::ret), /**< This value is equal to the maximum possible number of coefficients that this expression * might have. If this expression might have an arbitrarily high number of coefficients, * this value is set to \a Dynamic. * * This value is useful to know when evaluating an expression, in order to determine * whether it is possible to avoid doing a dynamic memory allocation. * * \sa SizeAtCompileTime, MaxRowsAtCompileTime, MaxColsAtCompileTime */ IsVectorAtCompileTime = internal::traits::RowsAtCompileTime == 1 || internal::traits::ColsAtCompileTime == 1, /**< This is set to true if either the number of rows or the number of * columns is known at compile-time to be equal to 1. Indeed, in that case, * we are dealing with a column-vector (if there is only one column) or with * a row-vector (if there is only one row). */ NumDimensions = int(MaxSizeAtCompileTime) == 1 ? 0 : bool(IsVectorAtCompileTime) ? 1 : 2, /**< This value is equal to Tensor::NumDimensions, i.e. 0 for scalars, 1 for vectors, * and 2 for matrices. */ Flags = internal::traits::Flags, /**< This stores expression \ref flags flags which may or may not be inherited by new expressions * constructed from this one. See the \ref flags "list of flags". */ IsRowMajor = int(Flags) & RowMajorBit, /**< True if this expression has row-major storage order. */ InnerSizeAtCompileTime = int(IsVectorAtCompileTime) ? int(SizeAtCompileTime) : int(IsRowMajor) ? int(ColsAtCompileTime) : int(RowsAtCompileTime), InnerStrideAtCompileTime = internal::inner_stride_at_compile_time::ret, OuterStrideAtCompileTime = internal::outer_stride_at_compile_time::ret }; typedef typename internal::find_best_packet::type PacketScalar; enum { IsPlainObjectBase = 0 }; /** The plain matrix type corresponding to this expression. * \sa PlainObject */ typedef Matrix::Scalar, internal::traits::RowsAtCompileTime, internal::traits::ColsAtCompileTime, AutoAlign | (internal::traits::Flags&RowMajorBit ? RowMajor : ColMajor), internal::traits::MaxRowsAtCompileTime, internal::traits::MaxColsAtCompileTime > PlainMatrix; /** The plain array type corresponding to this expression. * \sa PlainObject */ typedef Array::Scalar, internal::traits::RowsAtCompileTime, internal::traits::ColsAtCompileTime, AutoAlign | (internal::traits::Flags&RowMajorBit ? RowMajor : ColMajor), internal::traits::MaxRowsAtCompileTime, internal::traits::MaxColsAtCompileTime > PlainArray; /** \brief The plain matrix or array type corresponding to this expression. * * This is not necessarily exactly the return type of eval(). In the case of plain matrices, * the return type of eval() is a const reference to a matrix, not a matrix! It is however guaranteed * that the return type of eval() is either PlainObject or const PlainObject&. */ typedef typename internal::conditional::XprKind,MatrixXpr >::value, PlainMatrix, PlainArray>::type PlainObject; /** \returns the number of nonzero coefficients which is in practice the number * of stored coefficients. */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index nonZeros() const { return size(); } /** \returns the outer size. * * \note For a vector, this returns just 1. For a matrix (non-vector), this is the major dimension * with respect to the \ref TopicStorageOrders "storage order", i.e., the number of columns for a * column-major matrix, and the number of rows for a row-major matrix. */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index outerSize() const { return IsVectorAtCompileTime ? 1 : int(IsRowMajor) ? this->rows() : this->cols(); } /** \returns the inner size. * * \note For a vector, this is just the size. For a matrix (non-vector), this is the minor dimension * with respect to the \ref TopicStorageOrders "storage order", i.e., the number of rows for a * column-major matrix, and the number of columns for a row-major matrix. */ EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index innerSize() const { return IsVectorAtCompileTime ? this->size() : int(IsRowMajor) ? this->cols() : this->rows(); } /** Only plain matrices/arrays, not expressions, may be resized; therefore the only useful resize methods are * Matrix::resize() and Array::resize(). The present method only asserts that the new size equals the old size, and does * nothing else. */ EIGEN_DEVICE_FUNC void resize(Index newSize) { EIGEN_ONLY_USED_FOR_DEBUG(newSize); eigen_assert(newSize == this->size() && "DenseBase::resize() does not actually allow to resize."); } /** Only plain matrices/arrays, not expressions, may be resized; therefore the only useful resize methods are * Matrix::resize() and Array::resize(). The present method only asserts that the new size equals the old size, and does * nothing else. */ EIGEN_DEVICE_FUNC void resize(Index rows, Index cols) { EIGEN_ONLY_USED_FOR_DEBUG(rows); EIGEN_ONLY_USED_FOR_DEBUG(cols); eigen_assert(rows == this->rows() && cols == this->cols() && "DenseBase::resize() does not actually allow to resize."); } #ifndef EIGEN_PARSED_BY_DOXYGEN /** \internal Represents a matrix with all coefficients equal to one another*/ typedef CwiseNullaryOp,PlainObject> ConstantReturnType; /** \internal \deprecated Represents a vector with linearly spaced coefficients that allows sequential access only. */ EIGEN_DEPRECATED typedef CwiseNullaryOp,PlainObject> SequentialLinSpacedReturnType; /** \internal Represents a vector with linearly spaced coefficients that allows random access. */ typedef CwiseNullaryOp,PlainObject> RandomAccessLinSpacedReturnType; /** \internal the return type of MatrixBase::eigenvalues() */ typedef Matrix::Scalar>::Real, internal::traits::ColsAtCompileTime, 1> EigenvaluesReturnType; #endif // not EIGEN_PARSED_BY_DOXYGEN /** Copies \a other into *this. \returns a reference to *this. */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const DenseBase& other); /** Special case of the template operator=, in order to prevent the compiler * from generating a default operator= (issue hit with g++ 4.1) */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const DenseBase& other); template EIGEN_DEVICE_FUNC Derived& operator=(const EigenBase &other); template EIGEN_DEVICE_FUNC Derived& operator+=(const EigenBase &other); template EIGEN_DEVICE_FUNC Derived& operator-=(const EigenBase &other); template EIGEN_DEVICE_FUNC Derived& operator=(const ReturnByValue& func); /** \internal * Copies \a other into *this without evaluating other. \returns a reference to *this. */ template /** \deprecated */ EIGEN_DEPRECATED EIGEN_DEVICE_FUNC Derived& lazyAssign(const DenseBase& other); EIGEN_DEVICE_FUNC CommaInitializer operator<< (const Scalar& s); template /** \deprecated it now returns \c *this */ EIGEN_DEPRECATED const Derived& flagged() const { return derived(); } template EIGEN_DEVICE_FUNC CommaInitializer operator<< (const DenseBase& other); typedef Transpose TransposeReturnType; EIGEN_DEVICE_FUNC TransposeReturnType transpose(); typedef typename internal::add_const >::type ConstTransposeReturnType; EIGEN_DEVICE_FUNC ConstTransposeReturnType transpose() const; EIGEN_DEVICE_FUNC void transposeInPlace(); EIGEN_DEVICE_FUNC static const ConstantReturnType Constant(Index rows, Index cols, const Scalar& value); EIGEN_DEVICE_FUNC static const ConstantReturnType Constant(Index size, const Scalar& value); EIGEN_DEVICE_FUNC static const ConstantReturnType Constant(const Scalar& value); EIGEN_DEPRECATED EIGEN_DEVICE_FUNC static const RandomAccessLinSpacedReturnType LinSpaced(Sequential_t, Index size, const Scalar& low, const Scalar& high); EIGEN_DEPRECATED EIGEN_DEVICE_FUNC static const RandomAccessLinSpacedReturnType LinSpaced(Sequential_t, const Scalar& low, const Scalar& high); EIGEN_DEVICE_FUNC static const RandomAccessLinSpacedReturnType LinSpaced(Index size, const Scalar& low, const Scalar& high); EIGEN_DEVICE_FUNC static const RandomAccessLinSpacedReturnType LinSpaced(const Scalar& low, const Scalar& high); template EIGEN_DEVICE_FUNC static const CwiseNullaryOp NullaryExpr(Index rows, Index cols, const CustomNullaryOp& func); template EIGEN_DEVICE_FUNC static const CwiseNullaryOp NullaryExpr(Index size, const CustomNullaryOp& func); template EIGEN_DEVICE_FUNC static const CwiseNullaryOp NullaryExpr(const CustomNullaryOp& func); EIGEN_DEVICE_FUNC static const ConstantReturnType Zero(Index rows, Index cols); EIGEN_DEVICE_FUNC static const ConstantReturnType Zero(Index size); EIGEN_DEVICE_FUNC static const ConstantReturnType Zero(); EIGEN_DEVICE_FUNC static const ConstantReturnType Ones(Index rows, Index cols); EIGEN_DEVICE_FUNC static const ConstantReturnType Ones(Index size); EIGEN_DEVICE_FUNC static const ConstantReturnType Ones(); EIGEN_DEVICE_FUNC void fill(const Scalar& value); EIGEN_DEVICE_FUNC Derived& setConstant(const Scalar& value); EIGEN_DEVICE_FUNC Derived& setLinSpaced(Index size, const Scalar& low, const Scalar& high); EIGEN_DEVICE_FUNC Derived& setLinSpaced(const Scalar& low, const Scalar& high); EIGEN_DEVICE_FUNC Derived& setZero(); EIGEN_DEVICE_FUNC Derived& setOnes(); EIGEN_DEVICE_FUNC Derived& setRandom(); template EIGEN_DEVICE_FUNC bool isApprox(const DenseBase& other, const RealScalar& prec = NumTraits::dummy_precision()) const; EIGEN_DEVICE_FUNC bool isMuchSmallerThan(const RealScalar& other, const RealScalar& prec = NumTraits::dummy_precision()) const; template EIGEN_DEVICE_FUNC bool isMuchSmallerThan(const DenseBase& other, const RealScalar& prec = NumTraits::dummy_precision()) const; EIGEN_DEVICE_FUNC bool isApproxToConstant(const Scalar& value, const RealScalar& prec = NumTraits::dummy_precision()) const; EIGEN_DEVICE_FUNC bool isConstant(const Scalar& value, const RealScalar& prec = NumTraits::dummy_precision()) const; EIGEN_DEVICE_FUNC bool isZero(const RealScalar& prec = NumTraits::dummy_precision()) const; EIGEN_DEVICE_FUNC bool isOnes(const RealScalar& prec = NumTraits::dummy_precision()) const; inline bool hasNaN() const; inline bool allFinite() const; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator*=(const Scalar& other); EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator/=(const Scalar& other); typedef typename internal::add_const_on_value_type::type>::type EvalReturnType; /** \returns the matrix or vector obtained by evaluating this expression. * * Notice that in the case of a plain matrix or vector (not an expression) this function just returns * a const reference, in order to avoid a useless copy. * * \warning Be careful with eval() and the auto C++ keyword, as detailed in this \link TopicPitfalls_auto_keyword page \endlink. */ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EvalReturnType eval() const { // Even though MSVC does not honor strong inlining when the return type // is a dynamic matrix, we desperately need strong inlining for fixed // size types on MSVC. return typename internal::eval::type(derived()); } /** swaps *this with the expression \a other. * */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void swap(const DenseBase& other) { EIGEN_STATIC_ASSERT(!OtherDerived::IsPlainObjectBase,THIS_EXPRESSION_IS_NOT_A_LVALUE__IT_IS_READ_ONLY); eigen_assert(rows()==other.rows() && cols()==other.cols()); call_assignment(derived(), other.const_cast_derived(), internal::swap_assign_op()); } /** swaps *this with the matrix or array \a other. * */ template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void swap(PlainObjectBase& other) { eigen_assert(rows()==other.rows() && cols()==other.cols()); call_assignment(derived(), other.derived(), internal::swap_assign_op()); } EIGEN_DEVICE_FUNC inline const NestByValue nestByValue() const; EIGEN_DEVICE_FUNC inline const ForceAlignedAccess forceAlignedAccess() const; EIGEN_DEVICE_FUNC inline ForceAlignedAccess forceAlignedAccess(); template EIGEN_DEVICE_FUNC inline const typename internal::conditional,Derived&>::type forceAlignedAccessIf() const; template EIGEN_DEVICE_FUNC inline typename internal::conditional,Derived&>::type forceAlignedAccessIf(); EIGEN_DEVICE_FUNC Scalar sum() const; EIGEN_DEVICE_FUNC Scalar mean() const; EIGEN_DEVICE_FUNC Scalar trace() const; EIGEN_DEVICE_FUNC Scalar prod() const; template EIGEN_DEVICE_FUNC typename internal::traits::Scalar minCoeff() const; template EIGEN_DEVICE_FUNC typename internal::traits::Scalar maxCoeff() const; // By default, the fastest version with undefined NaN propagation semantics is // used. // TODO(rmlarsen): Replace with default template argument when we move to // c++11 or beyond. EIGEN_DEVICE_FUNC inline typename internal::traits::Scalar minCoeff() const { return minCoeff(); } EIGEN_DEVICE_FUNC inline typename internal::traits::Scalar maxCoeff() const { return maxCoeff(); } template EIGEN_DEVICE_FUNC typename internal::traits::Scalar minCoeff(IndexType* row, IndexType* col) const; template EIGEN_DEVICE_FUNC typename internal::traits::Scalar maxCoeff(IndexType* row, IndexType* col) const; template EIGEN_DEVICE_FUNC typename internal::traits::Scalar minCoeff(IndexType* index) const; template EIGEN_DEVICE_FUNC typename internal::traits::Scalar maxCoeff(IndexType* index) const; // TODO(rmlarsen): Replace these methods with a default template argument. template EIGEN_DEVICE_FUNC inline typename internal::traits::Scalar minCoeff(IndexType* row, IndexType* col) const { return minCoeff(row, col); } template EIGEN_DEVICE_FUNC inline typename internal::traits::Scalar maxCoeff(IndexType* row, IndexType* col) const { return maxCoeff(row, col); } template EIGEN_DEVICE_FUNC inline typename internal::traits::Scalar minCoeff(IndexType* index) const { return minCoeff(index); } template EIGEN_DEVICE_FUNC inline typename internal::traits::Scalar maxCoeff(IndexType* index) const { return maxCoeff(index); } template EIGEN_DEVICE_FUNC Scalar redux(const BinaryOp& func) const; template EIGEN_DEVICE_FUNC void visit(Visitor& func) const; /** \returns a WithFormat proxy object allowing to print a matrix the with given * format \a fmt. * * See class IOFormat for some examples. * * \sa class IOFormat, class WithFormat */ inline const WithFormat format(const IOFormat& fmt) const { return WithFormat(derived(), fmt); } /** \returns the unique coefficient of a 1x1 expression */ EIGEN_DEVICE_FUNC CoeffReturnType value() const { EIGEN_STATIC_ASSERT_SIZE_1x1(Derived) eigen_assert(this->rows() == 1 && this->cols() == 1); return derived().coeff(0,0); } EIGEN_DEVICE_FUNC bool all() const; EIGEN_DEVICE_FUNC bool any() const; EIGEN_DEVICE_FUNC Index count() const; typedef VectorwiseOp RowwiseReturnType; typedef const VectorwiseOp ConstRowwiseReturnType; typedef VectorwiseOp ColwiseReturnType; typedef const VectorwiseOp ConstColwiseReturnType; /** \returns a VectorwiseOp wrapper of *this for broadcasting and partial reductions * * Example: \include MatrixBase_rowwise.cpp * Output: \verbinclude MatrixBase_rowwise.out * * \sa colwise(), class VectorwiseOp, \ref TutorialReductionsVisitorsBroadcasting */ //Code moved here due to a CUDA compiler bug EIGEN_DEVICE_FUNC inline ConstRowwiseReturnType rowwise() const { return ConstRowwiseReturnType(derived()); } EIGEN_DEVICE_FUNC RowwiseReturnType rowwise(); /** \returns a VectorwiseOp wrapper of *this broadcasting and partial reductions * * Example: \include MatrixBase_colwise.cpp * Output: \verbinclude MatrixBase_colwise.out * * \sa rowwise(), class VectorwiseOp, \ref TutorialReductionsVisitorsBroadcasting */ EIGEN_DEVICE_FUNC inline ConstColwiseReturnType colwise() const { return ConstColwiseReturnType(derived()); } EIGEN_DEVICE_FUNC ColwiseReturnType colwise(); typedef CwiseNullaryOp,PlainObject> RandomReturnType; static const RandomReturnType Random(Index rows, Index cols); static const RandomReturnType Random(Index size); static const RandomReturnType Random(); template inline EIGEN_DEVICE_FUNC const Select select(const DenseBase& thenMatrix, const DenseBase& elseMatrix) const; template inline EIGEN_DEVICE_FUNC const Select select(const DenseBase& thenMatrix, const typename ThenDerived::Scalar& elseScalar) const; template inline EIGEN_DEVICE_FUNC const Select select(const typename ElseDerived::Scalar& thenScalar, const DenseBase& elseMatrix) const; template RealScalar lpNorm() const; template EIGEN_DEVICE_FUNC const Replicate replicate() const; /** * \return an expression of the replication of \c *this * * Example: \include MatrixBase_replicate_int_int.cpp * Output: \verbinclude MatrixBase_replicate_int_int.out * * \sa VectorwiseOp::replicate(), DenseBase::replicate(), class Replicate */ //Code moved here due to a CUDA compiler bug EIGEN_DEVICE_FUNC const Replicate replicate(Index rowFactor, Index colFactor) const { return Replicate(derived(), rowFactor, colFactor); } typedef Reverse ReverseReturnType; typedef const Reverse ConstReverseReturnType; EIGEN_DEVICE_FUNC ReverseReturnType reverse(); /** This is the const version of reverse(). */ //Code moved here due to a CUDA compiler bug EIGEN_DEVICE_FUNC ConstReverseReturnType reverse() const { return ConstReverseReturnType(derived()); } EIGEN_DEVICE_FUNC void reverseInPlace(); #ifdef EIGEN_PARSED_BY_DOXYGEN /** STL-like RandomAccessIterator * iterator type as returned by the begin() and end() methods. */ typedef random_access_iterator_type iterator; /** This is the const version of iterator (aka read-only) */ typedef random_access_iterator_type const_iterator; #else typedef typename internal::conditional< (Flags&DirectAccessBit)==DirectAccessBit, internal::pointer_based_stl_iterator, internal::generic_randaccess_stl_iterator >::type iterator_type; typedef typename internal::conditional< (Flags&DirectAccessBit)==DirectAccessBit, internal::pointer_based_stl_iterator, internal::generic_randaccess_stl_iterator >::type const_iterator_type; // Stl-style iterators are supported only for vectors. typedef typename internal::conditional< IsVectorAtCompileTime, iterator_type, void >::type iterator; typedef typename internal::conditional< IsVectorAtCompileTime, const_iterator_type, void >::type const_iterator; #endif inline iterator begin(); inline const_iterator begin() const; inline const_iterator cbegin() const; inline iterator end(); inline const_iterator end() const; inline const_iterator cend() const; #define EIGEN_CURRENT_STORAGE_BASE_CLASS Eigen::DenseBase #define EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL #define EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(COND) #define EIGEN_DOC_UNARY_ADDONS(X,Y) # include "../plugins/CommonCwiseUnaryOps.h" # include "../plugins/BlockMethods.h" # include "../plugins/IndexedViewMethods.h" # include "../plugins/ReshapedMethods.h" # ifdef EIGEN_DENSEBASE_PLUGIN # include EIGEN_DENSEBASE_PLUGIN # endif #undef EIGEN_CURRENT_STORAGE_BASE_CLASS #undef EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL #undef EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF #undef EIGEN_DOC_UNARY_ADDONS // disable the use of evalTo for dense objects with a nice compilation error template EIGEN_DEVICE_FUNC inline void evalTo(Dest& ) const { EIGEN_STATIC_ASSERT((internal::is_same::value),THE_EVAL_EVALTO_FUNCTION_SHOULD_NEVER_BE_CALLED_FOR_DENSE_OBJECTS); } protected: EIGEN_DEFAULT_COPY_CONSTRUCTOR(DenseBase) /** Default constructor. Do nothing. */ EIGEN_DEVICE_FUNC DenseBase() { /* Just checks for self-consistency of the flags. * Only do it when debugging Eigen, as this borders on paranoia and could slow compilation down */ #ifdef EIGEN_INTERNAL_DEBUGGING EIGEN_STATIC_ASSERT((EIGEN_IMPLIES(MaxRowsAtCompileTime==1 && MaxColsAtCompileTime!=1, int(IsRowMajor)) && EIGEN_IMPLIES(MaxColsAtCompileTime==1 && MaxRowsAtCompileTime!=1, int(!IsRowMajor))), INVALID_STORAGE_ORDER_FOR_THIS_VECTOR_EXPRESSION) #endif } private: EIGEN_DEVICE_FUNC explicit DenseBase(int); EIGEN_DEVICE_FUNC DenseBase(int,int); template EIGEN_DEVICE_FUNC explicit DenseBase(const DenseBase&); }; } // end namespace Eigen #endif // EIGEN_DENSEBASE_H RcppEigen/inst/include/Eigen/src/Core/DenseStorage.h0000644000176200001440000006142014567757725022024 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008 Gael Guennebaud // Copyright (C) 2006-2009 Benoit Jacob // Copyright (C) 2010-2013 Hauke Heibel // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_MATRIXSTORAGE_H #define EIGEN_MATRIXSTORAGE_H #ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN #define EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(X) X; EIGEN_DENSE_STORAGE_CTOR_PLUGIN; #else #define EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(X) #endif namespace Eigen { namespace internal { struct constructor_without_unaligned_array_assert {}; template EIGEN_DEVICE_FUNC void check_static_allocation_size() { // if EIGEN_STACK_ALLOCATION_LIMIT is defined to 0, then no limit #if EIGEN_STACK_ALLOCATION_LIMIT EIGEN_STATIC_ASSERT(Size * sizeof(T) <= EIGEN_STACK_ALLOCATION_LIMIT, OBJECT_ALLOCATED_ON_STACK_IS_TOO_BIG); #endif } /** \internal * Static array. If the MatrixOrArrayOptions require auto-alignment, the array will be automatically aligned: * to 16 bytes boundary if the total size is a multiple of 16 bytes. */ template ::value > struct plain_array { T array[Size]; EIGEN_DEVICE_FUNC plain_array() { check_static_allocation_size(); } EIGEN_DEVICE_FUNC plain_array(constructor_without_unaligned_array_assert) { check_static_allocation_size(); } }; #if defined(EIGEN_DISABLE_UNALIGNED_ARRAY_ASSERT) #define EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(sizemask) #elif EIGEN_GNUC_AT_LEAST(4,7) // GCC 4.7 is too aggressive in its optimizations and remove the alignment test based on the fact the array is declared to be aligned. // See this bug report: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=53900 // Hiding the origin of the array pointer behind a function argument seems to do the trick even if the function is inlined: template EIGEN_ALWAYS_INLINE PtrType eigen_unaligned_array_assert_workaround_gcc47(PtrType array) { return array; } #define EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(sizemask) \ eigen_assert((internal::UIntPtr(eigen_unaligned_array_assert_workaround_gcc47(array)) & (sizemask)) == 0 \ && "this assertion is explained here: " \ "http://eigen.tuxfamily.org/dox-devel/group__TopicUnalignedArrayAssert.html" \ " **** READ THIS WEB PAGE !!! ****"); #else #define EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(sizemask) \ eigen_assert((internal::UIntPtr(array) & (sizemask)) == 0 \ && "this assertion is explained here: " \ "http://eigen.tuxfamily.org/dox-devel/group__TopicUnalignedArrayAssert.html" \ " **** READ THIS WEB PAGE !!! ****"); #endif template struct plain_array { EIGEN_ALIGN_TO_BOUNDARY(8) T array[Size]; EIGEN_DEVICE_FUNC plain_array() { EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(7); check_static_allocation_size(); } EIGEN_DEVICE_FUNC plain_array(constructor_without_unaligned_array_assert) { check_static_allocation_size(); } }; template struct plain_array { EIGEN_ALIGN_TO_BOUNDARY(16) T array[Size]; EIGEN_DEVICE_FUNC plain_array() { EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(15); check_static_allocation_size(); } EIGEN_DEVICE_FUNC plain_array(constructor_without_unaligned_array_assert) { check_static_allocation_size(); } }; template struct plain_array { EIGEN_ALIGN_TO_BOUNDARY(32) T array[Size]; EIGEN_DEVICE_FUNC plain_array() { EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(31); check_static_allocation_size(); } EIGEN_DEVICE_FUNC plain_array(constructor_without_unaligned_array_assert) { check_static_allocation_size(); } }; template struct plain_array { EIGEN_ALIGN_TO_BOUNDARY(64) T array[Size]; EIGEN_DEVICE_FUNC plain_array() { EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(63); check_static_allocation_size(); } EIGEN_DEVICE_FUNC plain_array(constructor_without_unaligned_array_assert) { check_static_allocation_size(); } }; template struct plain_array { T array[1]; EIGEN_DEVICE_FUNC plain_array() {} EIGEN_DEVICE_FUNC plain_array(constructor_without_unaligned_array_assert) {} }; struct plain_array_helper { template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void copy(const plain_array& src, const Eigen::Index size, plain_array& dst) { smart_copy(src.array, src.array + size, dst.array); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void swap(plain_array& a, const Eigen::Index a_size, plain_array& b, const Eigen::Index b_size) { if (a_size < b_size) { std::swap_ranges(b.array, b.array + a_size, a.array); smart_move(b.array + a_size, b.array + b_size, a.array + a_size); } else if (a_size > b_size) { std::swap_ranges(a.array, a.array + b_size, b.array); smart_move(a.array + b_size, a.array + a_size, b.array + b_size); } else { std::swap_ranges(a.array, a.array + a_size, b.array); } } }; } // end namespace internal /** \internal * * \class DenseStorage * \ingroup Core_Module * * \brief Stores the data of a matrix * * This class stores the data of fixed-size, dynamic-size or mixed matrices * in a way as compact as possible. * * \sa Matrix */ template class DenseStorage; // purely fixed-size matrix template class DenseStorage { internal::plain_array m_data; public: EIGEN_DEVICE_FUNC DenseStorage() { EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = Size) } EIGEN_DEVICE_FUNC explicit DenseStorage(internal::constructor_without_unaligned_array_assert) : m_data(internal::constructor_without_unaligned_array_assert()) {} #if !EIGEN_HAS_CXX11 || defined(EIGEN_DENSE_STORAGE_CTOR_PLUGIN) EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) : m_data(other.m_data) { EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = Size) } #else EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage&) = default; #endif #if !EIGEN_HAS_CXX11 EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) { if (this != &other) m_data = other.m_data; return *this; } #else EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage&) = default; #endif #if EIGEN_HAS_RVALUE_REFERENCES #if !EIGEN_HAS_CXX11 EIGEN_DEVICE_FUNC DenseStorage(DenseStorage&& other) EIGEN_NOEXCEPT : m_data(std::move(other.m_data)) { } EIGEN_DEVICE_FUNC DenseStorage& operator=(DenseStorage&& other) EIGEN_NOEXCEPT { if (this != &other) m_data = std::move(other.m_data); return *this; } #else EIGEN_DEVICE_FUNC DenseStorage(DenseStorage&&) = default; EIGEN_DEVICE_FUNC DenseStorage& operator=(DenseStorage&&) = default; #endif #endif EIGEN_DEVICE_FUNC DenseStorage(Index size, Index rows, Index cols) { EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) eigen_internal_assert(size==rows*cols && rows==_Rows && cols==_Cols); EIGEN_UNUSED_VARIABLE(size); EIGEN_UNUSED_VARIABLE(rows); EIGEN_UNUSED_VARIABLE(cols); } EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { numext::swap(m_data, other.m_data); } EIGEN_DEVICE_FUNC static EIGEN_CONSTEXPR Index rows(void) EIGEN_NOEXCEPT {return _Rows;} EIGEN_DEVICE_FUNC static EIGEN_CONSTEXPR Index cols(void) EIGEN_NOEXCEPT {return _Cols;} EIGEN_DEVICE_FUNC void conservativeResize(Index,Index,Index) {} EIGEN_DEVICE_FUNC void resize(Index,Index,Index) {} EIGEN_DEVICE_FUNC const T *data() const { return m_data.array; } EIGEN_DEVICE_FUNC T *data() { return m_data.array; } }; // null matrix template class DenseStorage { public: EIGEN_DEVICE_FUNC DenseStorage() {} EIGEN_DEVICE_FUNC explicit DenseStorage(internal::constructor_without_unaligned_array_assert) {} EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage&) {} EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage&) { return *this; } EIGEN_DEVICE_FUNC DenseStorage(Index,Index,Index) {} EIGEN_DEVICE_FUNC void swap(DenseStorage& ) {} EIGEN_DEVICE_FUNC static EIGEN_CONSTEXPR Index rows(void) EIGEN_NOEXCEPT {return _Rows;} EIGEN_DEVICE_FUNC static EIGEN_CONSTEXPR Index cols(void) EIGEN_NOEXCEPT {return _Cols;} EIGEN_DEVICE_FUNC void conservativeResize(Index,Index,Index) {} EIGEN_DEVICE_FUNC void resize(Index,Index,Index) {} EIGEN_DEVICE_FUNC const T *data() const { return 0; } EIGEN_DEVICE_FUNC T *data() { return 0; } }; // more specializations for null matrices; these are necessary to resolve ambiguities template class DenseStorage : public DenseStorage { }; template class DenseStorage : public DenseStorage { }; template class DenseStorage : public DenseStorage { }; // dynamic-size matrix with fixed-size storage template class DenseStorage { internal::plain_array m_data; Index m_rows; Index m_cols; public: EIGEN_DEVICE_FUNC DenseStorage() : m_rows(0), m_cols(0) {} EIGEN_DEVICE_FUNC explicit DenseStorage(internal::constructor_without_unaligned_array_assert) : m_data(internal::constructor_without_unaligned_array_assert()), m_rows(0), m_cols(0) {} EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) : m_data(internal::constructor_without_unaligned_array_assert()), m_rows(other.m_rows), m_cols(other.m_cols) { internal::plain_array_helper::copy(other.m_data, m_rows * m_cols, m_data); } EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) { if (this != &other) { m_rows = other.m_rows; m_cols = other.m_cols; internal::plain_array_helper::copy(other.m_data, m_rows * m_cols, m_data); } return *this; } EIGEN_DEVICE_FUNC DenseStorage(Index, Index rows, Index cols) : m_rows(rows), m_cols(cols) {} EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { internal::plain_array_helper::swap(m_data, m_rows * m_cols, other.m_data, other.m_rows * other.m_cols); numext::swap(m_rows,other.m_rows); numext::swap(m_cols,other.m_cols); } EIGEN_DEVICE_FUNC Index rows() const {return m_rows;} EIGEN_DEVICE_FUNC Index cols() const {return m_cols;} EIGEN_DEVICE_FUNC void conservativeResize(Index, Index rows, Index cols) { m_rows = rows; m_cols = cols; } EIGEN_DEVICE_FUNC void resize(Index, Index rows, Index cols) { m_rows = rows; m_cols = cols; } EIGEN_DEVICE_FUNC const T *data() const { return m_data.array; } EIGEN_DEVICE_FUNC T *data() { return m_data.array; } }; // dynamic-size matrix with fixed-size storage and fixed width template class DenseStorage { internal::plain_array m_data; Index m_rows; public: EIGEN_DEVICE_FUNC DenseStorage() : m_rows(0) {} EIGEN_DEVICE_FUNC explicit DenseStorage(internal::constructor_without_unaligned_array_assert) : m_data(internal::constructor_without_unaligned_array_assert()), m_rows(0) {} EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) : m_data(internal::constructor_without_unaligned_array_assert()), m_rows(other.m_rows) { internal::plain_array_helper::copy(other.m_data, m_rows * _Cols, m_data); } EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) { if (this != &other) { m_rows = other.m_rows; internal::plain_array_helper::copy(other.m_data, m_rows * _Cols, m_data); } return *this; } EIGEN_DEVICE_FUNC DenseStorage(Index, Index rows, Index) : m_rows(rows) {} EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { internal::plain_array_helper::swap(m_data, m_rows * _Cols, other.m_data, other.m_rows * _Cols); numext::swap(m_rows, other.m_rows); } EIGEN_DEVICE_FUNC Index rows(void) const EIGEN_NOEXCEPT {return m_rows;} EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols(void) const EIGEN_NOEXCEPT {return _Cols;} EIGEN_DEVICE_FUNC void conservativeResize(Index, Index rows, Index) { m_rows = rows; } EIGEN_DEVICE_FUNC void resize(Index, Index rows, Index) { m_rows = rows; } EIGEN_DEVICE_FUNC const T *data() const { return m_data.array; } EIGEN_DEVICE_FUNC T *data() { return m_data.array; } }; // dynamic-size matrix with fixed-size storage and fixed height template class DenseStorage { internal::plain_array m_data; Index m_cols; public: EIGEN_DEVICE_FUNC DenseStorage() : m_cols(0) {} EIGEN_DEVICE_FUNC explicit DenseStorage(internal::constructor_without_unaligned_array_assert) : m_data(internal::constructor_without_unaligned_array_assert()), m_cols(0) {} EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) : m_data(internal::constructor_without_unaligned_array_assert()), m_cols(other.m_cols) { internal::plain_array_helper::copy(other.m_data, _Rows * m_cols, m_data); } EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) { if (this != &other) { m_cols = other.m_cols; internal::plain_array_helper::copy(other.m_data, _Rows * m_cols, m_data); } return *this; } EIGEN_DEVICE_FUNC DenseStorage(Index, Index, Index cols) : m_cols(cols) {} EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { internal::plain_array_helper::swap(m_data, _Rows * m_cols, other.m_data, _Rows * other.m_cols); numext::swap(m_cols, other.m_cols); } EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows(void) const EIGEN_NOEXCEPT {return _Rows;} EIGEN_DEVICE_FUNC Index cols(void) const EIGEN_NOEXCEPT {return m_cols;} EIGEN_DEVICE_FUNC void conservativeResize(Index, Index, Index cols) { m_cols = cols; } EIGEN_DEVICE_FUNC void resize(Index, Index, Index cols) { m_cols = cols; } EIGEN_DEVICE_FUNC const T *data() const { return m_data.array; } EIGEN_DEVICE_FUNC T *data() { return m_data.array; } }; // purely dynamic matrix. template class DenseStorage { T *m_data; Index m_rows; Index m_cols; public: EIGEN_DEVICE_FUNC DenseStorage() : m_data(0), m_rows(0), m_cols(0) {} EIGEN_DEVICE_FUNC explicit DenseStorage(internal::constructor_without_unaligned_array_assert) : m_data(0), m_rows(0), m_cols(0) {} EIGEN_DEVICE_FUNC DenseStorage(Index size, Index rows, Index cols) : m_data(internal::conditional_aligned_new_auto(size)), m_rows(rows), m_cols(cols) { EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) eigen_internal_assert(size==rows*cols && rows>=0 && cols >=0); } EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) : m_data(internal::conditional_aligned_new_auto(other.m_rows*other.m_cols)) , m_rows(other.m_rows) , m_cols(other.m_cols) { EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = m_rows*m_cols) internal::smart_copy(other.m_data, other.m_data+other.m_rows*other.m_cols, m_data); } EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) { if (this != &other) { DenseStorage tmp(other); this->swap(tmp); } return *this; } #if EIGEN_HAS_RVALUE_REFERENCES EIGEN_DEVICE_FUNC DenseStorage(DenseStorage&& other) EIGEN_NOEXCEPT : m_data(std::move(other.m_data)) , m_rows(std::move(other.m_rows)) , m_cols(std::move(other.m_cols)) { other.m_data = nullptr; other.m_rows = 0; other.m_cols = 0; } EIGEN_DEVICE_FUNC DenseStorage& operator=(DenseStorage&& other) EIGEN_NOEXCEPT { numext::swap(m_data, other.m_data); numext::swap(m_rows, other.m_rows); numext::swap(m_cols, other.m_cols); return *this; } #endif EIGEN_DEVICE_FUNC ~DenseStorage() { internal::conditional_aligned_delete_auto(m_data, m_rows*m_cols); } EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { numext::swap(m_data,other.m_data); numext::swap(m_rows,other.m_rows); numext::swap(m_cols,other.m_cols); } EIGEN_DEVICE_FUNC Index rows(void) const EIGEN_NOEXCEPT {return m_rows;} EIGEN_DEVICE_FUNC Index cols(void) const EIGEN_NOEXCEPT {return m_cols;} void conservativeResize(Index size, Index rows, Index cols) { m_data = internal::conditional_aligned_realloc_new_auto(m_data, size, m_rows*m_cols); m_rows = rows; m_cols = cols; } EIGEN_DEVICE_FUNC void resize(Index size, Index rows, Index cols) { if(size != m_rows*m_cols) { internal::conditional_aligned_delete_auto(m_data, m_rows*m_cols); if (size>0) // >0 and not simply !=0 to let the compiler knows that size cannot be negative m_data = internal::conditional_aligned_new_auto(size); else m_data = 0; EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) } m_rows = rows; m_cols = cols; } EIGEN_DEVICE_FUNC const T *data() const { return m_data; } EIGEN_DEVICE_FUNC T *data() { return m_data; } }; // matrix with dynamic width and fixed height (so that matrix has dynamic size). template class DenseStorage { T *m_data; Index m_cols; public: EIGEN_DEVICE_FUNC DenseStorage() : m_data(0), m_cols(0) {} explicit DenseStorage(internal::constructor_without_unaligned_array_assert) : m_data(0), m_cols(0) {} EIGEN_DEVICE_FUNC DenseStorage(Index size, Index rows, Index cols) : m_data(internal::conditional_aligned_new_auto(size)), m_cols(cols) { EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) eigen_internal_assert(size==rows*cols && rows==_Rows && cols >=0); EIGEN_UNUSED_VARIABLE(rows); } EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) : m_data(internal::conditional_aligned_new_auto(_Rows*other.m_cols)) , m_cols(other.m_cols) { EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = m_cols*_Rows) internal::smart_copy(other.m_data, other.m_data+_Rows*m_cols, m_data); } EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) { if (this != &other) { DenseStorage tmp(other); this->swap(tmp); } return *this; } #if EIGEN_HAS_RVALUE_REFERENCES EIGEN_DEVICE_FUNC DenseStorage(DenseStorage&& other) EIGEN_NOEXCEPT : m_data(std::move(other.m_data)) , m_cols(std::move(other.m_cols)) { other.m_data = nullptr; other.m_cols = 0; } EIGEN_DEVICE_FUNC DenseStorage& operator=(DenseStorage&& other) EIGEN_NOEXCEPT { numext::swap(m_data, other.m_data); numext::swap(m_cols, other.m_cols); return *this; } #endif EIGEN_DEVICE_FUNC ~DenseStorage() { internal::conditional_aligned_delete_auto(m_data, _Rows*m_cols); } EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { numext::swap(m_data,other.m_data); numext::swap(m_cols,other.m_cols); } EIGEN_DEVICE_FUNC static EIGEN_CONSTEXPR Index rows(void) EIGEN_NOEXCEPT {return _Rows;} EIGEN_DEVICE_FUNC Index cols(void) const EIGEN_NOEXCEPT {return m_cols;} EIGEN_DEVICE_FUNC void conservativeResize(Index size, Index, Index cols) { m_data = internal::conditional_aligned_realloc_new_auto(m_data, size, _Rows*m_cols); m_cols = cols; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void resize(Index size, Index, Index cols) { if(size != _Rows*m_cols) { internal::conditional_aligned_delete_auto(m_data, _Rows*m_cols); if (size>0) // >0 and not simply !=0 to let the compiler knows that size cannot be negative m_data = internal::conditional_aligned_new_auto(size); else m_data = 0; EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) } m_cols = cols; } EIGEN_DEVICE_FUNC const T *data() const { return m_data; } EIGEN_DEVICE_FUNC T *data() { return m_data; } }; // matrix with dynamic height and fixed width (so that matrix has dynamic size). template class DenseStorage { T *m_data; Index m_rows; public: EIGEN_DEVICE_FUNC DenseStorage() : m_data(0), m_rows(0) {} explicit DenseStorage(internal::constructor_without_unaligned_array_assert) : m_data(0), m_rows(0) {} EIGEN_DEVICE_FUNC DenseStorage(Index size, Index rows, Index cols) : m_data(internal::conditional_aligned_new_auto(size)), m_rows(rows) { EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) eigen_internal_assert(size==rows*cols && rows>=0 && cols == _Cols); EIGEN_UNUSED_VARIABLE(cols); } EIGEN_DEVICE_FUNC DenseStorage(const DenseStorage& other) : m_data(internal::conditional_aligned_new_auto(other.m_rows*_Cols)) , m_rows(other.m_rows) { EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = m_rows*_Cols) internal::smart_copy(other.m_data, other.m_data+other.m_rows*_Cols, m_data); } EIGEN_DEVICE_FUNC DenseStorage& operator=(const DenseStorage& other) { if (this != &other) { DenseStorage tmp(other); this->swap(tmp); } return *this; } #if EIGEN_HAS_RVALUE_REFERENCES EIGEN_DEVICE_FUNC DenseStorage(DenseStorage&& other) EIGEN_NOEXCEPT : m_data(std::move(other.m_data)) , m_rows(std::move(other.m_rows)) { other.m_data = nullptr; other.m_rows = 0; } EIGEN_DEVICE_FUNC DenseStorage& operator=(DenseStorage&& other) EIGEN_NOEXCEPT { numext::swap(m_data, other.m_data); numext::swap(m_rows, other.m_rows); return *this; } #endif EIGEN_DEVICE_FUNC ~DenseStorage() { internal::conditional_aligned_delete_auto(m_data, _Cols*m_rows); } EIGEN_DEVICE_FUNC void swap(DenseStorage& other) { numext::swap(m_data,other.m_data); numext::swap(m_rows,other.m_rows); } EIGEN_DEVICE_FUNC Index rows(void) const EIGEN_NOEXCEPT {return m_rows;} EIGEN_DEVICE_FUNC static EIGEN_CONSTEXPR Index cols(void) {return _Cols;} void conservativeResize(Index size, Index rows, Index) { m_data = internal::conditional_aligned_realloc_new_auto(m_data, size, m_rows*_Cols); m_rows = rows; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void resize(Index size, Index rows, Index) { if(size != m_rows*_Cols) { internal::conditional_aligned_delete_auto(m_data, _Cols*m_rows); if (size>0) // >0 and not simply !=0 to let the compiler knows that size cannot be negative m_data = internal::conditional_aligned_new_auto(size); else m_data = 0; EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({}) } m_rows = rows; } EIGEN_DEVICE_FUNC const T *data() const { return m_data; } EIGEN_DEVICE_FUNC T *data() { return m_data; } }; } // end namespace Eigen #endif // EIGEN_MATRIX_H RcppEigen/inst/include/Eigen/src/Core/Select.h0000644000176200001440000001377714567757725020674 0ustar liggesusers// This file is part of Eigen, a lightweight C++ template library // for linear algebra. // // Copyright (C) 2008-2010 Gael Guennebaud // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. #ifndef EIGEN_SELECT_H #define EIGEN_SELECT_H namespace Eigen { /** \class Select * \ingroup Core_Module * * \brief Expression of a coefficient wise version of the C++ ternary operator ?: * * \param ConditionMatrixType the type of the \em condition expression which must be a boolean matrix * \param ThenMatrixType the type of the \em then expression * \param ElseMatrixType the type of the \em else expression * * This class represents an expression of a coefficient wise version of the C++ ternary operator ?:. * It is the return type of DenseBase::select() and most of the time this is the only way it is used. * * \sa DenseBase::select(const DenseBase&, const DenseBase&) const */ namespace internal { template struct traits > : traits { typedef typename traits::Scalar Scalar; typedef Dense StorageKind; typedef typename traits::XprKind XprKind; typedef typename ConditionMatrixType::Nested ConditionMatrixNested; typedef typename ThenMatrixType::Nested ThenMatrixNested; typedef typename ElseMatrixType::Nested ElseMatrixNested; enum { RowsAtCompileTime = ConditionMatrixType::RowsAtCompileTime, ColsAtCompileTime = ConditionMatrixType::ColsAtCompileTime, MaxRowsAtCompileTime = ConditionMatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = ConditionMatrixType::MaxColsAtCompileTime, Flags = (unsigned int)ThenMatrixType::Flags & ElseMatrixType::Flags & RowMajorBit }; }; } template class Select : public internal::dense_xpr_base< Select >::type, internal::no_assignment_operator { public: typedef typename internal::dense_xpr_base